From 2c53af1664ffef137621511ffd14436f5ba0b398 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 11 Aug 2022 18:42:21 +0200 Subject: [PATCH 1/4] wallet: load lnworker earlier --- electrum/wallet.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/electrum/wallet.py b/electrum/wallet.py index 521ff735a..c3b40b1e1 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -339,6 +339,8 @@ class Abstract_Wallet(ABC, Logger, EventListener): self._freeze_lock = threading.RLock() # for mutating/iterating frozen_{addresses,coins} + self.load_keystore() + self._init_lnworker() self._init_requests_rhash_index() self._prepare_onchain_invoice_paid_detection() self.calc_unused_change_addresses() @@ -352,11 +354,12 @@ class Abstract_Wallet(ABC, Logger, EventListener): # to-be-generated (HD) addresses are also considered here (gap-limit-roll-forward) self._up_to_date = False - self.lnworker = None - self.load_keystore() self.test_addresses_sanity() self.register_callbacks() + def _init_lnworker(self): + self.lnworker = None + @ignore_exceptions # don't kill outer taskgroup async def main_loop(self): self.logger.info("starting taskgroup.") @@ -3170,6 +3173,8 @@ class Deterministic_Wallet(Abstract_Wallet): # generate addresses now. note that without libsecp this might block # for a few seconds! self.synchronize() + + def _init_lnworker(self): # lightning_privkey2 is not deterministic (legacy wallets, bip39) ln_xprv = self.db.get('lightning_xprv') or self.db.get('lightning_privkey2') # lnworker can only be initialized once receiving addresses are available From a64aa45e859874e156c65e00deb9a0c0da53f22d Mon Sep 17 00:00:00 2001 From: ThomasV Date: Thu, 11 Aug 2022 19:16:18 +0200 Subject: [PATCH 2/4] get default label for txid based on invoices --- electrum/gui/qt/history_list.py | 2 +- electrum/wallet.py | 51 ++++++++++++--------------------- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/electrum/gui/qt/history_list.py b/electrum/gui/qt/history_list.py index 662d68400..914068ef6 100644 --- a/electrum/gui/qt/history_list.py +++ b/electrum/gui/qt/history_list.py @@ -748,7 +748,7 @@ class HistoryList(MyTreeView, AcceptFileDragDrop): menu.addAction(_("Child pays for parent"), lambda: self.parent.cpfp_dialog(tx)) if tx_details.can_dscancel: menu.addAction(_("Cancel (double-spend)"), lambda: self.parent.dscancel_dialog(tx)) - invoices = self.wallet.get_relevant_invoices_for_tx(tx) + invoices = self.wallet.get_relevant_invoices_for_tx(tx_hash) if len(invoices) == 1: menu.addAction(_("View invoice"), lambda inv=invoices[0]: self.parent.show_onchain_invoice(inv)) elif len(invoices) > 1: diff --git a/electrum/wallet.py b/electrum/wallet.py index c3b40b1e1..0a02ff5fa 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -486,7 +486,6 @@ class Abstract_Wallet(ABC, Logger, EventListener): is_mine |= any([self.is_mine(self.adb.get_txin_address(txin)) for txin in tx.inputs()]) if not is_mine: return - self._maybe_set_tx_label_based_on_invoices(tx) if self.lnworker: self.lnworker.maybe_add_backup_from_tx(tx) self._update_request_statuses_touched_by_tx(tx_hash) @@ -993,9 +992,6 @@ class Abstract_Wallet(ABC, Logger, EventListener): if not invoice.is_lightning(): if self.is_onchain_invoice_paid(invoice)[0]: _logger.info("saving invoice... but it is already paid!") - with self.transaction_lock: - for txout in invoice.get_outputs(): - self._invoices_from_scriptpubkey_map[txout.scriptpubkey].add(key) self._invoices[key] = invoice self.save_db() @@ -1039,18 +1035,8 @@ class Abstract_Wallet(ABC, Logger, EventListener): def export_invoices(self, path): write_json_file(path, list(self._invoices.values())) - def _get_relevant_invoice_keys_for_tx(self, tx: Transaction) -> Set[str]: - relevant_invoice_keys = set() - with self.transaction_lock: - for txout in tx.outputs(): - for invoice_key in self._invoices_from_scriptpubkey_map.get(txout.scriptpubkey, set()): - # note: the invoice might have been deleted since, so check now: - if invoice_key in self._invoices: - relevant_invoice_keys.add(invoice_key) - return relevant_invoice_keys - - def get_relevant_invoices_for_tx(self, tx: Transaction) -> Sequence[Invoice]: - invoice_keys = self._get_relevant_invoice_keys_for_tx(tx) + def get_relevant_invoices_for_tx(self, tx_hash) -> Sequence[Invoice]: + invoice_keys = self._invoices_from_txid_map.get(tx_hash, set()) invoices = [self.get_invoice(key) for key in invoice_keys] invoices = [inv for inv in invoices if inv] # filter out None for inv in invoices: @@ -1064,13 +1050,16 @@ class Abstract_Wallet(ABC, Logger, EventListener): self._requests_addr_to_rhash[addr] = req.rhash def _prepare_onchain_invoice_paid_detection(self): - # scriptpubkey -> list(invoice_keys) - self._invoices_from_scriptpubkey_map = defaultdict(set) # type: Dict[bytes, Set[str]] + self._invoices_from_txid_map = defaultdict(set) # type: Dict[bytes, Set[str]] for invoice_key, invoice in self._invoices.items(): if invoice.is_lightning() and not invoice.get_address(): continue - for txout in invoice.get_outputs(): - self._invoices_from_scriptpubkey_map[txout.scriptpubkey].add(invoice_key) + if invoice.is_lightning() and self.lnworker and self.lnworker.get_invoice_status(invoice) == PR_PAID: + continue + is_paid, conf_needed, relevant_txs = self._is_onchain_invoice_paid(invoice) + if is_paid: + for txid in relevant_txs: + self._invoices_from_txid_map[txid].add(invoice_key) def _is_onchain_invoice_paid(self, invoice: Invoice) -> Tuple[bool, Optional[int], Sequence[str]]: """Returns whether on-chain invoice/request is satisfied, num confs required txs have, @@ -1110,19 +1099,12 @@ class Abstract_Wallet(ABC, Logger, EventListener): def is_onchain_invoice_paid(self, invoice: Invoice) -> Tuple[bool, Optional[int]]: is_paid, conf_needed, relevant_txs = self._is_onchain_invoice_paid(invoice) + if is_paid: + key = self.get_key_for_outgoing_invoice(invoice) + for txid in relevant_txs: + self._invoices_from_txid_map[txid].add(key) return is_paid, conf_needed - def _maybe_set_tx_label_based_on_invoices(self, tx: Transaction) -> bool: - # note: this is not done in 'get_default_label' as that would require deserializing each tx - tx_hash = tx.txid() - labels = [] - for invoice in self.get_relevant_invoices_for_tx(tx): - if invoice.message: - labels.append(invoice.message) - if labels and not self._labels.get(tx_hash, ''): - self.set_label(tx_hash, "; ".join(labels)) - return bool(labels) - @profiler def get_full_history(self, fx=None, *, onchain_domain=None, include_lightning=True): transactions_tmp = OrderedDictWithIndex() @@ -1385,7 +1367,12 @@ class Abstract_Wallet(ABC, Logger, EventListener): if label: labels.append(label) return ', '.join(labels) - return '' + else: + labels = [] + for invoice in self.get_relevant_invoices_for_tx(tx_hash): + if invoice.message: + labels.append(invoice.message) + return ', '.join(labels) def _get_default_label_for_rhash(self, rhash: str) -> str: req = self.get_request(rhash) From 24145f1f527772d5ef2986a387cf3c5dfcd35db4 Mon Sep 17 00:00:00 2001 From: ThomasV Date: Tue, 16 Aug 2022 10:14:47 +0200 Subject: [PATCH 3/4] detect paid invoices in on_event_adb_added_tx --- electrum/wallet.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/electrum/wallet.py b/electrum/wallet.py index 0a02ff5fa..8375e65e5 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -489,6 +489,7 @@ class Abstract_Wallet(ABC, Logger, EventListener): if self.lnworker: self.lnworker.maybe_add_backup_from_tx(tx) self._update_request_statuses_touched_by_tx(tx_hash) + self._detect_paid_invoices() if notify_GUI: util.trigger_callback('new_transaction', self, tx) @@ -1050,7 +1051,10 @@ class Abstract_Wallet(ABC, Logger, EventListener): self._requests_addr_to_rhash[addr] = req.rhash def _prepare_onchain_invoice_paid_detection(self): - self._invoices_from_txid_map = defaultdict(set) # type: Dict[bytes, Set[str]] + self._invoices_from_txid_map = defaultdict(set) # type: Dict[str, Set[str]] + self._detect_paid_invoices() + + def _detect_paid_invoices(self): for invoice_key, invoice in self._invoices.items(): if invoice.is_lightning() and not invoice.get_address(): continue @@ -1099,10 +1103,6 @@ class Abstract_Wallet(ABC, Logger, EventListener): def is_onchain_invoice_paid(self, invoice: Invoice) -> Tuple[bool, Optional[int]]: is_paid, conf_needed, relevant_txs = self._is_onchain_invoice_paid(invoice) - if is_paid: - key = self.get_key_for_outgoing_invoice(invoice) - for txid in relevant_txs: - self._invoices_from_txid_map[txid].add(key) return is_paid, conf_needed @profiler From d1c15fe5e9d1dd764e4588b55c935e4c169b12ba Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 16 Aug 2022 12:14:44 +0000 Subject: [PATCH 4/4] wallet: (perf) avoid iterating over all invoices in add_transaction --- electrum/wallet.py | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/electrum/wallet.py b/electrum/wallet.py index 8375e65e5..a47af2075 100644 --- a/electrum/wallet.py +++ b/electrum/wallet.py @@ -488,8 +488,7 @@ class Abstract_Wallet(ABC, Logger, EventListener): return if self.lnworker: self.lnworker.maybe_add_backup_from_tx(tx) - self._update_request_statuses_touched_by_tx(tx_hash) - self._detect_paid_invoices() + self._update_invoices_and_reqs_touched_by_tx(tx_hash) if notify_GUI: util.trigger_callback('new_transaction', self, tx) @@ -497,7 +496,7 @@ class Abstract_Wallet(ABC, Logger, EventListener): def on_event_adb_added_verified_tx(self, adb, tx_hash): if adb != self.adb: return - self._update_request_statuses_touched_by_tx(tx_hash) + self._update_invoices_and_reqs_touched_by_tx(tx_hash) tx_mined_status = self.adb.get_tx_height(tx_hash) util.trigger_callback('verified', self, tx_hash, tx_mined_status) @@ -505,7 +504,7 @@ class Abstract_Wallet(ABC, Logger, EventListener): def on_event_adb_removed_verified_tx(self, adb, tx_hash): if adb != self.adb: return - self._update_request_statuses_touched_by_tx(tx_hash) + self._update_invoices_and_reqs_touched_by_tx(tx_hash) def clear_history(self): self.adb.clear_history() @@ -993,6 +992,9 @@ class Abstract_Wallet(ABC, Logger, EventListener): if not invoice.is_lightning(): if self.is_onchain_invoice_paid(invoice)[0]: _logger.info("saving invoice... but it is already paid!") + with self.transaction_lock: + for txout in invoice.get_outputs(): + self._invoices_from_scriptpubkey_map[txout.scriptpubkey].add(key) self._invoices[key] = invoice self.save_db() @@ -1052,10 +1054,14 @@ class Abstract_Wallet(ABC, Logger, EventListener): def _prepare_onchain_invoice_paid_detection(self): self._invoices_from_txid_map = defaultdict(set) # type: Dict[str, Set[str]] - self._detect_paid_invoices() + self._invoices_from_scriptpubkey_map = defaultdict(set) # type: Dict[bytes, Set[str]] + self._update_onchain_invoice_paid_detection(self._invoices.keys()) - def _detect_paid_invoices(self): - for invoice_key, invoice in self._invoices.items(): + def _update_onchain_invoice_paid_detection(self, invoice_keys: Iterable[str]) -> None: + for invoice_key in invoice_keys: + invoice = self._invoices.get(invoice_key) + if not invoice: + continue if invoice.is_lightning() and not invoice.get_address(): continue if invoice.is_lightning() and self.lnworker and self.lnworker.get_invoice_status(invoice) == PR_PAID: @@ -1064,6 +1070,8 @@ class Abstract_Wallet(ABC, Logger, EventListener): if is_paid: for txid in relevant_txs: self._invoices_from_txid_map[txid].add(invoice_key) + for txout in invoice.get_outputs(): + self._invoices_from_scriptpubkey_map[txout.scriptpubkey].add(invoice_key) def _is_onchain_invoice_paid(self, invoice: Invoice) -> Tuple[bool, Optional[int], Sequence[str]]: """Returns whether on-chain invoice/request is satisfied, num confs required txs have, @@ -1359,8 +1367,9 @@ class Abstract_Wallet(ABC, Logger, EventListener): return self._labels.get(tx_hash) or self._get_default_label_for_txid(tx_hash) def _get_default_label_for_txid(self, tx_hash: str) -> str: - # if no inputs are ismine, concat labels of output addresses + # note: we don't deserialize tx as the history calls us for every tx, and that would be slow if not self.db.get_txi_addresses(tx_hash): + # no inputs are ismine -> likely incoming payment -> concat labels of output addresses labels = [] for addr in self.db.get_txo_addresses(tx_hash): label = self.get_label_for_address(addr) @@ -1368,6 +1377,7 @@ class Abstract_Wallet(ABC, Logger, EventListener): labels.append(label) return ', '.join(labels) else: + # some inputs are ismine -> likely outgoing payment labels = [] for invoice in self.get_relevant_invoices_for_tx(tx_hash): if invoice.message: @@ -2430,18 +2440,23 @@ class Abstract_Wallet(ABC, Logger, EventListener): d['bip70'] = x.bip70 return d - def _update_request_statuses_touched_by_tx(self, tx_hash: str) -> None: + def _update_invoices_and_reqs_touched_by_tx(self, tx_hash: str) -> None: # FIXME in some cases if tx2 replaces unconfirmed tx1 in the mempool, we are not called. # For a given receive request, if tx1 touches it but tx2 does not, then # we were called when tx1 was added, but we will not get called when tx2 replaces tx1. tx = self.db.get_transaction(tx_hash) if tx is None: return - for txo in tx.outputs(): - addr = txo.address - if self.get_request(addr): - status = self.get_request_status(addr) - util.trigger_callback('request_status', self, addr, status) + relevant_invoice_keys = set() + with self.transaction_lock: + for txo in tx.outputs(): + addr = txo.address + if self.get_request(addr): + status = self.get_request_status(addr) + util.trigger_callback('request_status', self, addr, status) + for invoice_key in self._invoices_from_scriptpubkey_map.get(txo.scriptpubkey, set()): + relevant_invoice_keys.add(invoice_key) + self._update_onchain_invoice_paid_detection(relevant_invoice_keys) def create_request(self, amount_sat: int, message: str, exp_delay: int, address: Optional[str]): # for receiving