Merge pull request #7778 from SomberNight/202204_invoice_recalc_ids

wallet_db upgrade: recalc keys of outgoing on-chain invoices

electrum/gui/kivy/main_window.py

@@ -19,7 +19,7 @@ from electrum import util
 from electrum.util import (profiler, InvalidPassword, send_exception_to_crash_reporter,
                            format_satoshis, format_satoshis_plain, format_fee_satoshis,
                            maybe_extract_bolt11_invoice, parse_max_spend)
-from electrum.invoices import PR_PAID, PR_FAILED
+from electrum.invoices import PR_PAID, PR_FAILED, Invoice
 from electrum import blockchain
 from electrum.network import Network, TxBroadcastError, BestEffortRequestFailed
 from electrum.interface import PREFERRED_NETWORK_PROTOCOL, ServerAddr
@@ -447,8 +447,7 @@ class ElectrumWindow(App, Logger):
             self.show_error(_('No wallet loaded.'))
             return
         if pr.verify(self.wallet.contacts):
-            key = pr.get_id()
+            invoice = Invoice.from_bip70_payreq(pr, height=0)
-            invoice = self.wallet.get_invoice(key)  # FIXME wrong key...
             if invoice and self.wallet.get_invoice_status(invoice) == PR_PAID:
                 self.show_error("invoice already paid")
                 self.send_screen.do_clear()

electrum/gui/qt/main_window.py

@@ -2094,9 +2094,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         pr = self.payment_request
         if not pr:
             return
-        key = pr.get_id()
+        invoice = Invoice.from_bip70_payreq(pr, height=0)
-        invoice = self.wallet.get_invoice(key)
+        if self.wallet.get_invoice_status(invoice) == PR_PAID:
-        if invoice and self.wallet.get_invoice_status(invoice) == PR_PAID:
             self.show_message("invoice already paid")
             self.do_clear()
             self.payment_request = None
@@ -2335,8 +2334,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
             grid.addWidget(QLabel(_("Signature") + ':'), 6, 0)
             grid.addWidget(QLabel(pr.get_verify_status()), 6, 1)
             def do_export():
-                key = pr.get_id()
+                name = pr.get_name_for_export() or "payment_request"
-                name = str(key) + '.bip70'
+                name = f"{name}.bip70"
                 fn = getSaveFileName(
                     parent=self,
                     title=_("Save invoice to file"),

electrum/invoices.py

@@ -12,6 +12,7 @@ from . import constants
 from .bitcoin import COIN, TOTAL_COIN_SUPPLY_LIMIT_IN_BTC
 from .bitcoin import address_to_script
 from .transaction import PartialTxOutput
+from .crypto import sha256d
 
 if TYPE_CHECKING:
     from .paymentrequest import PaymentRequest
@@ -65,7 +66,7 @@ pr_expiration_values = {
 assert PR_DEFAULT_EXPIRATION_WHEN_CREATING in pr_expiration_values
 
 
-def _decode_outputs(outputs) -> List[PartialTxOutput]:
+def _decode_outputs(outputs) -> Optional[List[PartialTxOutput]]:
     if outputs is None:
         return None
     ret = []
@@ -98,13 +99,13 @@ class Invoice(StoredObject):
     # an invoice (outgoing) is constructed from a source: bip21, bip70, lnaddr
 
     # onchain only
-    outputs = attr.ib(kw_only=True, converter=_decode_outputs)  # type: List[PartialTxOutput]
+    outputs = attr.ib(kw_only=True, converter=_decode_outputs)  # type: Optional[List[PartialTxOutput]]
     height = attr.ib(type=int, kw_only=True, validator=attr.validators.instance_of(int)) # only for receiving
     bip70 = attr.ib(type=str, kw_only=True)  # type: Optional[str]
     #bip70_requestor = attr.ib(type=str, kw_only=True)  # type: Optional[str]
 
     # lightning only
-    lightning_invoice = attr.ib(type=str, kw_only=True)
+    lightning_invoice = attr.ib(type=str, kw_only=True)  # type: Optional[str]
 
     __lnaddr = None
 
@@ -227,7 +228,7 @@ class Invoice(StoredObject):
         )
 
     @classmethod
-    def from_bip70_payreq(cls, pr: 'PaymentRequest', height:int) -> 'Invoice':
+    def from_bip70_payreq(cls, pr: 'PaymentRequest', *, height: int = 0) -> 'Invoice':
         return Invoice(
             amount_msat=pr.get_amount()*1000,
             message=pr.get_memo(),
@@ -251,3 +252,14 @@ class Invoice(StoredObject):
             # 'tags': str(lnaddr.tags),
         })
         return d
+
+    def get_id(self) -> str:
+        if self.is_lightning():
+            return self.rhash
+        else:  # on-chain
+            return get_id_from_onchain_outputs(outputs=self.get_outputs(), timestamp=self.time)
+
+
+def get_id_from_onchain_outputs(outputs: List[PartialTxOutput], *, timestamp: int) -> str:
+    outputs_str = "\n".join(f"{txout.scriptpubkey.hex()}, {txout.value}" for txout in outputs)
+    return sha256d(outputs_str + "%d" % timestamp).hex()[0:10]

electrum/paymentrequest.py

@@ -41,7 +41,7 @@ except ImportError:
 
 from . import bitcoin, constants, ecc, util, transaction, x509, rsakey
 from .util import bh2u, bfh, make_aiohttp_session
-from .invoices import Invoice
+from .invoices import Invoice, get_id_from_onchain_outputs
 from .crypto import sha256
 from .bitcoin import address_to_script
 from .transaction import PartialTxOutput
@@ -121,7 +121,7 @@ async def get_payment_request(url: str) -> 'PaymentRequest':
 
 class PaymentRequest:
 
-    def __init__(self, data, *, error=None):
+    def __init__(self, data: bytes, *, error=None):
         self.raw = data
         self.error = error  # FIXME overloaded and also used when 'verify' succeeds
         self.parse(data)
@@ -131,11 +131,10 @@ class PaymentRequest:
     def __str__(self):
         return str(self.raw)
 
-    def parse(self, r):
+    def parse(self, r: bytes):
         self.outputs = []  # type: List[PartialTxOutput]
         if self.error:
             return
-        self.id = bh2u(sha256(r)[0:16])
         try:
             self.data = pb2.PaymentRequest()
             self.data.ParseFromString(r)
@@ -275,8 +274,10 @@ class PaymentRequest:
     def get_memo(self):
         return self.memo
 
-    def get_id(self):
+    def get_name_for_export(self) -> Optional[str]:
-        return self.id if self.requestor else self.get_address()
+        if not hasattr(self, 'details'):
+            return None
+        return get_id_from_onchain_outputs(self.outputs, timestamp=self.get_time())
 
     def get_outputs(self):
         return self.outputs[:]

electrum/wallet.py

@@ -773,9 +773,9 @@ class Abstract_Wallet(AddressSynchronizer, ABC):
             }
 
     def create_invoice(self, *, outputs: List[PartialTxOutput], message, pr, URI) -> Invoice:
-        height=self.get_local_height()
+        height = self.get_local_height()
         if pr:
-            return Invoice.from_bip70_payreq(pr, height)
+            return Invoice.from_bip70_payreq(pr, height=height)
         amount_msat = 0
         for x in outputs:
             if parse_max_spend(x.value):
@@ -2380,11 +2380,7 @@ class Abstract_Wallet(AddressSynchronizer, ABC):
     @classmethod
     def get_key_for_outgoing_invoice(cls, invoice: Invoice) -> str:
         """Return the key to use for this invoice in self.invoices."""
-        if invoice.is_lightning():
+        return invoice.get_id()
-            key = invoice.rhash
-        else:
-            key = bh2u(sha256d(repr(invoice.get_outputs()) + "%d"%invoice.time))[0:10]
-        return key
 
     def get_key_for_receive_request(self, req: Invoice, *, sanity_checks: bool = False) -> str:
         """Return the key to use for this invoice in self.receive_requests."""

electrum/wallet_db.py

@@ -42,7 +42,6 @@ from .lnutil import ImportedChannelBackupStorage, OnchainChannelBackupStorage
 from .lnutil import ChannelConstraints, Outpoint, ShachainElement
 from .json_db import StoredDict, JsonDB, locked, modifier
 from .plugin import run_hook, plugin_loaders
-from .paymentrequest import PaymentRequest
 from .submarine_swaps import SwapData
 
 if TYPE_CHECKING:
@@ -53,7 +52,7 @@ if TYPE_CHECKING:
 
 OLD_SEED_VERSION = 4        # electrum versions < 2.0
 NEW_SEED_VERSION = 11       # electrum versions >= 2.0
-FINAL_SEED_VERSION = 45     # electrum >= 2.7 will set this to prevent
+FINAL_SEED_VERSION = 46     # electrum >= 2.7 will set this to prevent
                             # old versions from overwriting new format
 
 
@@ -194,6 +193,7 @@ class WalletDB(JsonDB):
         self._convert_version_43()
         self._convert_version_44()
         self._convert_version_45()
+        self._convert_version_46()
         self.put('seed_version', FINAL_SEED_VERSION)  # just to be sure
 
         self._after_upgrade_tasks()
@@ -558,6 +558,7 @@ class WalletDB(JsonDB):
         self.data['seed_version'] = 24
 
     def _convert_version_25(self):
+        from .crypto import sha256
         if not self._is_upgrade_method_needed(24, 24):
             return
         # add 'type' field to onchain requests
@@ -574,25 +575,15 @@ class WalletDB(JsonDB):
                     'time': r.get('time'),
                     'type': PR_TYPE_ONCHAIN,
                 }
-        # convert bip70 invoices
+        # delete bip70 invoices
+        # note: this upgrade was changed ~2 years after-the-fact to delete instead of converting
         invoices = self.data.get('invoices', {})
         for k, r in list(invoices.items()):
             data = r.get("hex")
-            if data:
+            pr_id = sha256(bytes.fromhex(data))[0:16].hex()
-                pr = PaymentRequest(bytes.fromhex(data))
+            if pr_id != k:
-                if pr.id != k:
+                continue
-                    continue
+            del invoices[k]
-                invoices[k] = {
-                    'type': PR_TYPE_ONCHAIN,
-                    'amount': pr.get_amount(),
-                    'bip70': data,
-                    'exp': pr.get_expiration_date() - pr.get_time(),
-                    'id': pr.id,
-                    'message': pr.get_memo(),
-                    'outputs': [x.to_legacy_tuple() for x in pr.get_outputs()],
-                    'time': pr.get_time(),
-                    'requestor': pr.get_requestor(),
-                }
         self.data['seed_version'] = 25
 
     def _convert_version_26(self):
@@ -908,6 +899,30 @@ class WalletDB(JsonDB):
                 }
         self.data['seed_version'] = 45
 
+    def _convert_version_46(self):
+        from .crypto import sha256d
+        if not self._is_upgrade_method_needed(45, 45):
+            return
+        # recalc keys of outgoing on-chain invoices
+        def get_id_from_onchain_outputs(raw_outputs, timestamp):
+            outputs = [PartialTxOutput.from_legacy_tuple(*output) for output in raw_outputs]
+            outputs_str = "\n".join(f"{txout.scriptpubkey.hex()}, {txout.value}" for txout in outputs)
+            return sha256d(outputs_str + "%d" % timestamp).hex()[0:10]
+
+        invoices = self.data.get('invoices', {})
+        for key, item in list(invoices.items()):
+            is_lightning = item['lightning_invoice'] is not None
+            if is_lightning:
+                continue
+            outputs_raw = item['outputs']
+            assert outputs_raw, outputs_raw
+            timestamp = item['time']
+            newkey = get_id_from_onchain_outputs(outputs_raw, timestamp)
+            if newkey != key:
+                invoices[newkey] = item
+                del invoices[key]
+        self.data['seed_version'] = 46
+
     def _convert_imported(self):
         if not self._is_upgrade_method_needed(0, 13):
             return