wallet: try detecting internal address corruption

electrum/gui/kivy/main_window.py

@@ -9,9 +9,9 @@ import threading
 
 from electrum.bitcoin import TYPE_ADDRESS
 from electrum.storage import WalletStorage
-from electrum.wallet import Wallet
+from electrum.wallet import Wallet, InternalAddressCorruption
 from electrum.paymentrequest import InvoiceStore
-from electrum.util import profiler, InvalidPassword
+from electrum.util import profiler, InvalidPassword, send_exception_to_crash_reporter
 from electrum.plugin import run_hook
 from electrum.util import format_satoshis, format_satoshis_plain
 from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED
@@ -712,6 +712,11 @@ class ElectrumWindow(App):
             self.receive_screen.clear()
         self.update_tabs()
         run_hook('load_wallet', wallet, self)
+        try:
+            wallet.try_detecting_internal_addresses_corruption()
+        except InternalAddressCorruption as e:
+            self.show_error(str(e))
+            send_exception_to_crash_reporter(e)
 
     def update_status(self, *dt):
         self.num_blocks = self.network.get_local_height()
@@ -754,6 +759,10 @@ class ElectrumWindow(App):
             return ''
         except NotEnoughFunds:
             return ''
+        except InternalAddressCorruption as e:
+            self.show_error(str(e))
+            send_exception_to_crash_reporter(e)
+            return ''
         amount = tx.output_value()
         __, x_fee_amount = run_hook('get_tx_extra_fee', self.wallet, tx) or (None, 0)
         amount_after_all_fees = amount - x_fee_amount

electrum/gui/kivy/uix/screens.py

@@ -21,9 +21,10 @@ from kivy.utils import platform
 from electrum.util import profiler, parse_URI, format_time, InvalidPassword, NotEnoughFunds, Fiat
 from electrum import bitcoin
 from electrum.transaction import TxOutput
-from electrum.util import timestamp_to_datetime
+from electrum.util import send_exception_to_crash_reporter
 from electrum.paymentrequest import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED
 from electrum.plugin import run_hook
+from electrum.wallet import InternalAddressCorruption
 
 from .context_menu import ContextMenu
 
@@ -331,18 +332,24 @@ class ReceiveScreen(CScreen):
         self.screen.amount = ''
         self.screen.message = ''
 
-    def get_new_address(self):
+    def get_new_address(self) -> bool:
+        """Sets the address field, and returns whether the set address
+        is unused."""
         if not self.app.wallet:
             return False
         self.clear()
-        addr = self.app.wallet.get_unused_address()
+        unused = True
-        if addr is None:
+        try:
-            addr = self.app.wallet.get_receiving_address() or ''
+            addr = self.app.wallet.get_unused_address()
-            b = False
+            if addr is None:
-        else:
+                addr = self.app.wallet.get_receiving_address() or ''
-            b = True
+                unused = False
+        except InternalAddressCorruption as e:
+            addr = ''
+            self.app.show_error(str(e))
+            send_exception_to_crash_reporter(e)
         self.screen.address = addr
-        return b
+        return unused
 
     def on_address(self, addr):
         req = self.app.wallet.get_payment_request(addr, self.app.electrum_config)
@@ -401,8 +408,8 @@ class ReceiveScreen(CScreen):
         Clock.schedule_once(lambda dt: self.update_qr())
 
     def do_new(self):
-        addr = self.get_new_address()
+        is_unused = self.get_new_address()
-        if not addr:
+        if not is_unused:
             self.app.show_info(_('Please use the existing requests first.'))
 
     def do_save(self):

electrum/gui/qt/address_list.py

@@ -28,6 +28,7 @@ from electrum.i18n import _
 from electrum.util import block_explorer_URL
 from electrum.plugin import run_hook
 from electrum.bitcoin import is_address
+from electrum.wallet import InternalAddressCorruption
 
 from .util import *
 
@@ -168,7 +169,7 @@ class AddressList(MyTreeView):
 
             column_title = self.model().horizontalHeaderItem(col).text()
             copy_text = self.model().itemFromIndex(idx).text()
-            menu.addAction(_("Copy {}").format(column_title), lambda: self.parent.app.clipboard().setText(copy_text))
+            menu.addAction(_("Copy {}").format(column_title), lambda: self.place_text_on_clipboard(copy_text))
             menu.addAction(_('Details'), lambda: self.parent.show_address(addr))
             persistent = QPersistentModelIndex(addr_idx)
             menu.addAction(_("Edit {}").format(addr_column_title), lambda p=persistent: self.edit(QModelIndex(p)))
@@ -195,3 +196,12 @@ class AddressList(MyTreeView):
 
         run_hook('receive_menu', menu, addrs, self.wallet)
         menu.exec_(self.viewport().mapToGlobal(position))
+
+    def place_text_on_clipboard(self, text):
+        if is_address(text):
+            try:
+                self.wallet.raise_if_cannot_rederive_address(text)
+            except InternalAddressCorruption as e:
+                self.parent.show_error(str(e))
+                raise
+        self.parent.app.clipboard().setText(text)

electrum/gui/qt/main_window.py

@@ -56,11 +56,11 @@ from electrum.util import (format_time, format_satoshis, format_fee_satoshis,
                            base_units, base_units_list, base_unit_name_to_decimal_point,
                            decimal_point_to_base_unit_name, quantize_feerate,
                            UnknownBaseUnit, DECIMAL_POINT_DEFAULT, UserFacingException,
-                           get_new_wallet_name)
+                           get_new_wallet_name, send_exception_to_crash_reporter)
 from electrum.transaction import Transaction, TxOutput
 from electrum.address_synchronizer import AddTransactionException
 from electrum.wallet import (Multisig_Wallet, CannotBumpFee, Abstract_Wallet,
-                             sweep_preparations)
+                             sweep_preparations, InternalAddressCorruption)
 from electrum.version import ELECTRUM_VERSION
 from electrum.network import Network
 from electrum.exchange_rate import FxThread
@@ -399,6 +399,11 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             self.show()
         self.watching_only_changed()
         run_hook('load_wallet', wallet, self)
+        try:
+            wallet.try_detecting_internal_addresses_corruption()
+        except InternalAddressCorruption as e:
+            self.show_error(str(e))
+            send_exception_to_crash_reporter(e)
 
     def init_geometry(self):
         winpos = self.wallet.storage.get("winpos-qt")
@@ -1030,7 +1035,11 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         self.receive_amount_e.setAmount(None)
 
     def clear_receive_tab(self):
-        addr = self.wallet.get_receiving_address() or ''
+        try:
+            addr = self.wallet.get_receiving_address() or ''
+        except InternalAddressCorruption as e:
+            self.show_error(str(e))
+            addr = ''
         self.receive_address_e.setText(addr)
         self.receive_message_e.setText('')
         self.receive_amount_e.setAmount(None)
@@ -1557,6 +1566,9 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
         except (NotEnoughFunds, NoDynamicFeeEstimates) as e:
             self.show_message(str(e))
             return
+        except InternalAddressCorruption as e:
+            self.show_error(str(e))
+            raise
         except BaseException as e:
             traceback.print_exc(file=sys.stdout)
             self.show_message(str(e))
@@ -2600,11 +2612,24 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, PrintError):
             text = str(keys_e.toPlainText())
             return keystore.get_private_keys(text)
 
+        def on_address(text):
+            # set text color
+            addr = get_address()
+            ss = (ColorScheme.DEFAULT if addr else ColorScheme.RED).as_stylesheet()
+            address_e.setStyleSheet(ss)
+            # if addr looks to be ours, make sure we can re-derive it
+            if addr and self.wallet.is_mine(addr):
+                try:
+                    self.wallet.raise_if_cannot_rederive_address(addr)
+                except InternalAddressCorruption as e:
+                    self.show_error(str(e))
+                    raise
+
         f = lambda: button.setEnabled(get_address() is not None and get_pk() is not None)
-        on_address = lambda text: address_e.setStyleSheet((ColorScheme.DEFAULT if get_address() else ColorScheme.RED).as_stylesheet())
         keys_e.textChanged.connect(f)
         address_e.textChanged.connect(f)
         address_e.textChanged.connect(on_address)
+        on_address(str(address_e.text()))
         if not d.exec_():
             return
         # user pressed "sweep"

electrum/gui/qt/request_list.py

@@ -31,6 +31,7 @@ from electrum.i18n import _
 from electrum.util import format_time, age
 from electrum.plugin import run_hook
 from electrum.paymentrequest import PR_UNKNOWN
+from electrum.wallet import InternalAddressCorruption
 
 from .util import MyTreeView, pr_tooltips, pr_icons
 
@@ -78,7 +79,11 @@ class RequestList(MyTreeView):
         # update the receive address if necessary
         current_address = self.parent.receive_address_e.text()
         domain = self.wallet.get_receiving_addresses()
-        addr = self.wallet.get_unused_address()
+        try:
+            addr = self.wallet.get_unused_address()
+        except InternalAddressCorruption as e:
+            self.parent.show_error(str(e))
+            addr = ''
         if not current_address in domain and addr:
             self.parent.set_receive_address(addr)
         self.parent.new_request_button.setEnabled(addr != current_address)

electrum/util.py

@@ -835,6 +835,10 @@ def setup_thread_excepthook():
     threading.Thread.__init__ = init
 
 
+def send_exception_to_crash_reporter(e: BaseException):
+    sys.excepthook(type(e), e, e.__traceback__)
+
+
 def versiontuple(v):
     return tuple(map(int, (v.split("."))))
 

electrum/wallet.py

@@ -61,6 +61,7 @@ from .paymentrequest import (PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED,
                              InvoiceStore)
 from .contacts import Contacts
 from .interface import RequestTimedOut
+from .ecc_fast import is_using_fast_ecc
 
 if TYPE_CHECKING:
     from .network import Network
@@ -149,6 +150,11 @@ def sweep(privkeys, network: 'Network', config: 'SimpleConfig', recipient, fee=N
 class CannotBumpFee(Exception): pass
 
 
+class InternalAddressCorruption(Exception):
+    def __str__(self):
+        return _("Internal address database inconsistency detected. "
+                 "You should restore from seed.")
+
 
 
 class Abstract_Wallet(AddressSynchronizer):
@@ -632,6 +638,10 @@ class Abstract_Wallet(AddressSynchronizer):
                 # if there are none, take one randomly from the last few
                 addrs = self.get_change_addresses()[-self.gap_limit_for_change:]
                 change_addrs = [random.choice(addrs)] if addrs else []
+        for addr in change_addrs:
+            # note that change addresses are not necessarily ismine
+            # in which case this is a no-op
+            self.raise_if_cannot_rederive_address(addr)
 
         # Fee estimator
         if fixed_fee is None:
@@ -887,17 +897,33 @@ class Abstract_Wallet(AddressSynchronizer):
                 continue
         return tx
 
+    @profiler
+    def try_detecting_internal_addresses_corruption(self):
+        pass
+
+    def raise_if_cannot_rederive_address(self, addr):
+        pass
+
+    def try_rederiving_returned_address(func):
+        def wrapper(self, *args, **kwargs):
+            addr = func(self, *args, **kwargs)
+            self.raise_if_cannot_rederive_address(addr)
+            return addr
+        return wrapper
+
     def get_unused_addresses(self):
         # fixme: use slots from expired requests
         domain = self.get_receiving_addresses()
         return [addr for addr in domain if not self.history.get(addr)
                 and addr not in self.receive_requests.keys()]
 
+    @try_rederiving_returned_address
     def get_unused_address(self):
         addrs = self.get_unused_addresses()
         if addrs:
             return addrs[0]
 
+    @try_rederiving_returned_address
     def get_receiving_address(self):
         # always return an address
         domain = self.get_receiving_addresses()
@@ -1462,6 +1488,29 @@ class Deterministic_Wallet(Abstract_Wallet):
     def get_change_addresses(self):
         return self.change_addresses
 
+    @profiler
+    def try_detecting_internal_addresses_corruption(self):
+        if not is_using_fast_ecc():
+            self.print_error("internal address corruption test skipped due to missing libsecp256k1")
+            return
+        addresses_all = self.get_addresses()
+        # sample 1: first few
+        addresses_sample1 = addresses_all[:10]
+        # sample2: a few more randomly selected
+        addresses_rand = addresses_all[10:]
+        addresses_sample2 = random.sample(addresses_rand, min(len(addresses_rand), 10))
+        for addr_found in addresses_sample1 + addresses_sample2:
+            self.raise_if_cannot_rederive_address(addr_found)
+
+    def raise_if_cannot_rederive_address(self, addr):
+        if not addr:
+            return
+        if not self.is_mine(addr):
+            return
+        addr_derived = self.derive_address(*self.get_address_index(addr))
+        if addr != addr_derived:
+            raise InternalAddressCorruption()
+
     def get_seed(self, password):
         return self.keystore.get_seed(password)
 
@@ -1515,13 +1564,17 @@ class Deterministic_Wallet(Abstract_Wallet):
         for i, addr in enumerate(self.change_addresses):
             self._addr_to_addr_index[addr] = (True, i)
 
+    def derive_address(self, for_change, n):
+        x = self.derive_pubkeys(for_change, n)
+        address = self.pubkeys_to_address(x)
+        return address
+
     def create_new_address(self, for_change=False):
         assert type(for_change) is bool
         with self.lock:
             addr_list = self.change_addresses if for_change else self.receiving_addresses
             n = len(addr_list)
-            x = self.derive_pubkeys(for_change, n)
+            address = self.derive_address(for_change, n)
-            address = self.pubkeys_to_address(x)
             addr_list.append(address)
             self._addr_to_addr_index[address] = (for_change, n)
             self.save_addresses()