From adfe542fae126a3ab0dd2d8ad89c2d314fbb954d Mon Sep 17 00:00:00 2001 From: SomberNight Date: Fri, 22 Apr 2022 19:53:55 +0200 Subject: [PATCH 1/2] wallet_db upgrade: recalc keys of outgoing on-chain invoices closes https://github.com/spesmilo/electrum/issues/7777 --- electrum/gui/kivy/main_window.py | 5 ++--- electrum/gui/qt/main_window.py | 9 ++++----- electrum/invoices.py | 20 ++++++++++++++++---- electrum/paymentrequest.py | 9 +++++---- electrum/wallet.py | 10 +++------- electrum/wallet_db.py | 27 ++++++++++++++++++++++++++- 6 files changed, 56 insertions(+), 24 deletions(-) diff --git a/electrum/gui/kivy/main_window.py b/electrum/gui/kivy/main_window.py index ae9f82f0a..ce8d1ca72 100644 --- a/electrum/gui/kivy/main_window.py +++ b/electrum/gui/kivy/main_window.py @@ -19,7 +19,7 @@ from electrum import util from electrum.util import (profiler, InvalidPassword, send_exception_to_crash_reporter, format_satoshis, format_satoshis_plain, format_fee_satoshis, maybe_extract_bolt11_invoice, parse_max_spend) -from electrum.invoices import PR_PAID, PR_FAILED +from electrum.invoices import PR_PAID, PR_FAILED, Invoice from electrum import blockchain from electrum.network import Network, TxBroadcastError, BestEffortRequestFailed from electrum.interface import PREFERRED_NETWORK_PROTOCOL, ServerAddr @@ -447,8 +447,7 @@ class ElectrumWindow(App, Logger): self.show_error(_('No wallet loaded.')) return if pr.verify(self.wallet.contacts): - key = pr.get_id() - invoice = self.wallet.get_invoice(key) # FIXME wrong key... + invoice = Invoice.from_bip70_payreq(pr, height=0) if invoice and self.wallet.get_invoice_status(invoice) == PR_PAID: self.show_error("invoice already paid") self.send_screen.do_clear() diff --git a/electrum/gui/qt/main_window.py b/electrum/gui/qt/main_window.py index 93de73f80..1c9f72240 100644 --- a/electrum/gui/qt/main_window.py +++ b/electrum/gui/qt/main_window.py @@ -2092,9 +2092,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger): pr = self.payment_request if not pr: return - key = pr.get_id() - invoice = self.wallet.get_invoice(key) - if invoice and self.wallet.get_invoice_status(invoice) == PR_PAID: + invoice = Invoice.from_bip70_payreq(pr, height=0) + if self.wallet.get_invoice_status(invoice) == PR_PAID: self.show_message("invoice already paid") self.do_clear() self.payment_request = None @@ -2333,8 +2332,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger): grid.addWidget(QLabel(_("Signature") + ':'), 6, 0) grid.addWidget(QLabel(pr.get_verify_status()), 6, 1) def do_export(): - key = pr.get_id() - name = str(key) + '.bip70' + name = pr.get_name_for_export() or "payment_request" + name = f"{name}.bip70" fn = getSaveFileName( parent=self, title=_("Save invoice to file"), diff --git a/electrum/invoices.py b/electrum/invoices.py index 7bf2eee6a..f9e13538d 100644 --- a/electrum/invoices.py +++ b/electrum/invoices.py @@ -12,6 +12,7 @@ from . import constants from .bitcoin import COIN, TOTAL_COIN_SUPPLY_LIMIT_IN_BTC from .bitcoin import address_to_script from .transaction import PartialTxOutput +from .crypto import sha256d if TYPE_CHECKING: from .paymentrequest import PaymentRequest @@ -65,7 +66,7 @@ pr_expiration_values = { assert PR_DEFAULT_EXPIRATION_WHEN_CREATING in pr_expiration_values -def _decode_outputs(outputs) -> List[PartialTxOutput]: +def _decode_outputs(outputs) -> Optional[List[PartialTxOutput]]: if outputs is None: return None ret = [] @@ -98,13 +99,13 @@ class Invoice(StoredObject): # an invoice (outgoing) is constructed from a source: bip21, bip70, lnaddr # onchain only - outputs = attr.ib(kw_only=True, converter=_decode_outputs) # type: List[PartialTxOutput] + outputs = attr.ib(kw_only=True, converter=_decode_outputs) # type: Optional[List[PartialTxOutput]] height = attr.ib(type=int, kw_only=True, validator=attr.validators.instance_of(int)) # only for receiving bip70 = attr.ib(type=str, kw_only=True) # type: Optional[str] #bip70_requestor = attr.ib(type=str, kw_only=True) # type: Optional[str] # lightning only - lightning_invoice = attr.ib(type=str, kw_only=True) + lightning_invoice = attr.ib(type=str, kw_only=True) # type: Optional[str] __lnaddr = None @@ -227,7 +228,7 @@ class Invoice(StoredObject): ) @classmethod - def from_bip70_payreq(cls, pr: 'PaymentRequest', height:int) -> 'Invoice': + def from_bip70_payreq(cls, pr: 'PaymentRequest', *, height: int = 0) -> 'Invoice': return Invoice( amount_msat=pr.get_amount()*1000, message=pr.get_memo(), @@ -251,3 +252,14 @@ class Invoice(StoredObject): # 'tags': str(lnaddr.tags), }) return d + + def get_id(self) -> str: + if self.is_lightning(): + return self.rhash + else: # on-chain + return get_id_from_onchain_outputs(outputs=self.get_outputs(), timestamp=self.time) + + +def get_id_from_onchain_outputs(outputs: List[PartialTxOutput], *, timestamp: int) -> str: + outputs_str = "\n".join(f"{txout.scriptpubkey.hex()}, {txout.value}" for txout in outputs) + return sha256d(outputs_str + "%d" % timestamp).hex()[0:10] diff --git a/electrum/paymentrequest.py b/electrum/paymentrequest.py index d6ac9ca15..a839e22b7 100644 --- a/electrum/paymentrequest.py +++ b/electrum/paymentrequest.py @@ -41,7 +41,7 @@ except ImportError: from . import bitcoin, constants, ecc, util, transaction, x509, rsakey from .util import bh2u, bfh, make_aiohttp_session -from .invoices import Invoice +from .invoices import Invoice, get_id_from_onchain_outputs from .crypto import sha256 from .bitcoin import address_to_script from .transaction import PartialTxOutput @@ -135,7 +135,6 @@ class PaymentRequest: self.outputs = [] # type: List[PartialTxOutput] if self.error: return - self.id = bh2u(sha256(r)[0:16]) try: self.data = pb2.PaymentRequest() self.data.ParseFromString(r) @@ -275,8 +274,10 @@ class PaymentRequest: def get_memo(self): return self.memo - def get_id(self): - return self.id if self.requestor else self.get_address() + def get_name_for_export(self) -> Optional[str]: + if not hasattr(self, 'details'): + return None + return get_id_from_onchain_outputs(self.outputs, timestamp=self.get_time()) def get_outputs(self): return self.outputs[:] diff --git a/electrum/wallet.py b/electrum/wallet.py index b0c78c48e..f6867a324 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -773,9 +773,9 @@ class Abstract_Wallet(AddressSynchronizer, ABC): } def create_invoice(self, *, outputs: List[PartialTxOutput], message, pr, URI) -> Invoice: - height=self.get_local_height() + height = self.get_local_height() if pr: - return Invoice.from_bip70_payreq(pr, height) + return Invoice.from_bip70_payreq(pr, height=height) amount_msat = 0 for x in outputs: if parse_max_spend(x.value): @@ -2380,11 +2380,7 @@ class Abstract_Wallet(AddressSynchronizer, ABC): @classmethod def get_key_for_outgoing_invoice(cls, invoice: Invoice) -> str: """Return the key to use for this invoice in self.invoices.""" - if invoice.is_lightning(): - key = invoice.rhash - else: - key = bh2u(sha256d(repr(invoice.get_outputs()) + "%d"%invoice.time))[0:10] - return key + return invoice.get_id() def get_key_for_receive_request(self, req: Invoice, *, sanity_checks: bool = False) -> str: """Return the key to use for this invoice in self.receive_requests.""" diff --git a/electrum/wallet_db.py b/electrum/wallet_db.py index 6c84c7291..1db2f933a 100644 --- a/electrum/wallet_db.py +++ b/electrum/wallet_db.py @@ -53,7 +53,7 @@ if TYPE_CHECKING: OLD_SEED_VERSION = 4 # electrum versions < 2.0 NEW_SEED_VERSION = 11 # electrum versions >= 2.0 -FINAL_SEED_VERSION = 45 # electrum >= 2.7 will set this to prevent +FINAL_SEED_VERSION = 46 # electrum >= 2.7 will set this to prevent # old versions from overwriting new format @@ -194,6 +194,7 @@ class WalletDB(JsonDB): self._convert_version_43() self._convert_version_44() self._convert_version_45() + self._convert_version_46() self.put('seed_version', FINAL_SEED_VERSION) # just to be sure self._after_upgrade_tasks() @@ -908,6 +909,30 @@ class WalletDB(JsonDB): } self.data['seed_version'] = 45 + def _convert_version_46(self): + from .crypto import sha256d + if not self._is_upgrade_method_needed(45, 45): + return + # recalc keys of outgoing on-chain invoices + def get_id_from_onchain_outputs(raw_outputs, timestamp): + outputs = [PartialTxOutput.from_legacy_tuple(*output) for output in raw_outputs] + outputs_str = "\n".join(f"{txout.scriptpubkey.hex()}, {txout.value}" for txout in outputs) + return sha256d(outputs_str + "%d" % timestamp).hex()[0:10] + + invoices = self.data.get('invoices', {}) + for key, item in list(invoices.items()): + is_lightning = item['lightning_invoice'] is not None + if is_lightning: + continue + outputs_raw = item['outputs'] + assert outputs_raw, outputs_raw + timestamp = item['time'] + newkey = get_id_from_onchain_outputs(outputs_raw, timestamp) + if newkey != key: + invoices[newkey] = item + del invoices[key] + self.data['seed_version'] = 46 + def _convert_imported(self): if not self._is_upgrade_method_needed(0, 13): return From cfa6b91f22ff8bdf632ad1a8a61729ed746e6354 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 23 Apr 2022 20:15:10 +0200 Subject: [PATCH 2/2] wallet_db: rm dependence on PaymentRequest class in convert_version_25 Change convert_version_25 to delete invoices instead of converting them. convert_version_25 was released ~2 years ago. Wallet files not opened since will have old bip70 invoices deleted upon upgrading. In general it is ~unsafe for convert_version_* to depend on other modules of the code. (using e.g. sha256 is fine as its API will never change, but using e.g. PaymentRequest is dangerous as its API might change over time) --- electrum/paymentrequest.py | 4 ++-- electrum/wallet_db.py | 24 +++++++----------------- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/electrum/paymentrequest.py b/electrum/paymentrequest.py index a839e22b7..550797a56 100644 --- a/electrum/paymentrequest.py +++ b/electrum/paymentrequest.py @@ -121,7 +121,7 @@ async def get_payment_request(url: str) -> 'PaymentRequest': class PaymentRequest: - def __init__(self, data, *, error=None): + def __init__(self, data: bytes, *, error=None): self.raw = data self.error = error # FIXME overloaded and also used when 'verify' succeeds self.parse(data) @@ -131,7 +131,7 @@ class PaymentRequest: def __str__(self): return str(self.raw) - def parse(self, r): + def parse(self, r: bytes): self.outputs = [] # type: List[PartialTxOutput] if self.error: return diff --git a/electrum/wallet_db.py b/electrum/wallet_db.py index 1db2f933a..c9c25d04e 100644 --- a/electrum/wallet_db.py +++ b/electrum/wallet_db.py @@ -42,7 +42,6 @@ from .lnutil import ImportedChannelBackupStorage, OnchainChannelBackupStorage from .lnutil import ChannelConstraints, Outpoint, ShachainElement from .json_db import StoredDict, JsonDB, locked, modifier from .plugin import run_hook, plugin_loaders -from .paymentrequest import PaymentRequest from .submarine_swaps import SwapData if TYPE_CHECKING: @@ -559,6 +558,7 @@ class WalletDB(JsonDB): self.data['seed_version'] = 24 def _convert_version_25(self): + from .crypto import sha256 if not self._is_upgrade_method_needed(24, 24): return # add 'type' field to onchain requests @@ -575,25 +575,15 @@ class WalletDB(JsonDB): 'time': r.get('time'), 'type': PR_TYPE_ONCHAIN, } - # convert bip70 invoices + # delete bip70 invoices + # note: this upgrade was changed ~2 years after-the-fact to delete instead of converting invoices = self.data.get('invoices', {}) for k, r in list(invoices.items()): data = r.get("hex") - if data: - pr = PaymentRequest(bytes.fromhex(data)) - if pr.id != k: - continue - invoices[k] = { - 'type': PR_TYPE_ONCHAIN, - 'amount': pr.get_amount(), - 'bip70': data, - 'exp': pr.get_expiration_date() - pr.get_time(), - 'id': pr.id, - 'message': pr.get_memo(), - 'outputs': [x.to_legacy_tuple() for x in pr.get_outputs()], - 'time': pr.get_time(), - 'requestor': pr.get_requestor(), - } + pr_id = sha256(bytes.fromhex(data))[0:16].hex() + if pr_id != k: + continue + del invoices[k] self.data['seed_version'] = 25 def _convert_version_26(self):