psbt: put fake xpubs into globals. keystores handle xfp/der_prefix missing

electrum/base_wizard.py

@@ -394,7 +394,7 @@ class BaseWizard(Logger):
             # For segwit, a custom path is used, as there is no standard at all.
             default_choice_idx = 2
             choices = [
-                ('standard',   'legacy multisig (p2sh)',            "m/45'/0"),
+                ('standard',   'legacy multisig (p2sh)',            normalize_bip32_derivation("m/45'/0")),
                 ('p2wsh-p2sh', 'p2sh-segwit multisig (p2wsh-p2sh)', purpose48_derivation(0, xtype='p2wsh-p2sh')),
                 ('p2wsh',      'native segwit multisig (p2wsh)',    purpose48_derivation(0, xtype='p2wsh')),
             ]

electrum/bip32.py

@@ -3,7 +3,7 @@
 # file LICENCE or http://www.opensource.org/licenses/mit-license.php
 
 import hashlib
-from typing import List, Tuple, NamedTuple, Union, Iterable
+from typing import List, Tuple, NamedTuple, Union, Iterable, Sequence, Optional
 
 from .util import bfh, bh2u, BitcoinException
 from . import constants
@@ -335,7 +335,7 @@ def convert_bip32_path_to_list_of_uint32(n: str) -> List[int]:
     return path
 
 
-def convert_bip32_intpath_to_strpath(path: List[int]) -> str:
+def convert_bip32_intpath_to_strpath(path: Sequence[int]) -> str:
     s = "m/"
     for child_index in path:
         if not isinstance(child_index, int):
@@ -363,8 +363,28 @@ def is_bip32_derivation(s: str) -> bool:
         return True
 
 
-def normalize_bip32_derivation(s: str) -> str:
+def normalize_bip32_derivation(s: Optional[str]) -> Optional[str]:
+    if s is None:
+        return None
     if not is_bip32_derivation(s):
         raise ValueError(f"invalid bip32 derivation: {s}")
     ints = convert_bip32_path_to_list_of_uint32(s)
     return convert_bip32_intpath_to_strpath(ints)
+
+
+def root_fp_and_der_prefix_from_xkey(xkey: str) -> Tuple[Optional[str], Optional[str]]:
+    """Returns the root bip32 fingerprint and the derivation path from the
+    root to the given xkey, if they can be determined. Otherwise (None, None).
+    """
+    node = BIP32Node.from_xkey(xkey)
+    derivation_prefix = None
+    root_fingerprint = None
+    assert node.depth >= 0, node.depth
+    if node.depth == 0:
+        derivation_prefix = 'm'
+        root_fingerprint = node.calc_fingerprint_of_this_node().hex().lower()
+    elif node.depth == 1:
+        child_number_int = int.from_bytes(node.child_number, 'big')
+        derivation_prefix = convert_bip32_intpath_to_strpath([child_number_int])
+        root_fingerprint = node.fingerprint.hex()
+    return root_fingerprint, derivation_prefix

electrum/json_db.py

@@ -446,28 +446,39 @@ class JsonDB(Logger):
         self.put('seed_version', 19)
 
     def _convert_version_20(self):
-        # store 'derivation' (prefix) and 'root_fingerprint' in all xpub-based keystores
+        # store 'derivation' (prefix) and 'root_fingerprint' in all xpub-based keystores.
+        # store explicit None values if we cannot retroactively determine them
         if not self._is_upgrade_method_needed(19, 19):
             return
 
-        from .bip32 import BIP32Node
+        from .bip32 import BIP32Node, convert_bip32_intpath_to_strpath
+        # note: This upgrade method reimplements bip32.root_fp_and_der_prefix_from_xkey.
+        #       This is done deliberately, to avoid introducing that method as a dependency to this upgrade.
         for ks_name in ('keystore', *['x{}/'.format(i) for i in range(1, 16)]):
             ks = self.get(ks_name, None)
             if ks is None: continue
             xpub = ks.get('xpub', None)
             if xpub is None: continue
+            bip32node = BIP32Node.from_xkey(xpub)
             # derivation prefix
-            derivation_prefix = ks.get('derivation', 'm')
-            ks['derivation'] = derivation_prefix
+            derivation_prefix = ks.get('derivation', None)
+            if derivation_prefix is None:
+                assert bip32node.depth >= 0, bip32node.depth
+                if bip32node.depth == 0:
+                    derivation_prefix = 'm'
+                elif bip32node.depth == 1:
+                    child_number_int = int.from_bytes(bip32node.child_number, 'big')
+                    derivation_prefix = convert_bip32_intpath_to_strpath([child_number_int])
+                ks['derivation'] = derivation_prefix
             # root fingerprint
             root_fingerprint = ks.get('ckcc_xfp', None)
             if root_fingerprint is not None:
                 root_fingerprint = root_fingerprint.to_bytes(4, byteorder="little", signed=False).hex().lower()
             if root_fingerprint is None:
-                # if we don't have prior data, we set it to the fp of the xpub
-                # EVEN IF there was already a derivation prefix saved different than 'm'
-                node = BIP32Node.from_xkey(xpub)
-                root_fingerprint = node.calc_fingerprint_of_this_node().hex().lower()
+                if bip32node.depth == 0:
+                    root_fingerprint = bip32node.calc_fingerprint_of_this_node().hex().lower()
+                elif bip32node.depth == 1:
+                    root_fingerprint = bip32node.fingerprint.hex()
             ks['root_fingerprint'] = root_fingerprint
             ks.pop('ckcc_xfp', None)
             self.put(ks_name, ks)

electrum/keystore.py

@@ -26,12 +26,14 @@
 
 from unicodedata import normalize
 import hashlib
-from typing import Tuple, TYPE_CHECKING, Union, Sequence, Optional, Dict, List
+import re
+from typing import Tuple, TYPE_CHECKING, Union, Sequence, Optional, Dict, List, NamedTuple
 
 from . import bitcoin, ecc, constants, bip32
 from .bitcoin import deserialize_privkey, serialize_privkey
 from .bip32 import (convert_bip32_path_to_list_of_uint32, BIP32_PRIME,
-                    is_xpub, is_xprv, BIP32Node, normalize_bip32_derivation)
+                    is_xpub, is_xprv, BIP32Node, normalize_bip32_derivation,
+                    convert_bip32_intpath_to_strpath)
 from .ecc import string_to_number, number_to_string
 from .crypto import (pw_decode, pw_encode, sha256, sha256d, PW_HASH_VERSION_LATEST,
                      SUPPORTED_PW_HASH_VERSIONS, UnsupportedPasswordHashVersion, hash_160)
@@ -49,6 +51,7 @@ class KeyStore(Logger):
 
     def __init__(self):
         Logger.__init__(self)
+        self.is_requesting_to_be_rewritten_to_wallet_file = False  # type: bool
 
     def has_seed(self):
         return False
@@ -325,30 +328,60 @@ class Xpub:
         self.xpub_receive = None
         self.xpub_change = None
 
-        # if these are None now, then it is the responsibility of the caller to
-        # also call self.add_derivation_prefix_and_root_fingerprint:
-        self._derivation_prefix = derivation_prefix  # note: subclass should persist this
-        self._root_fingerprint = root_fingerprint  # note: subclass should persist this
+        # "key origin" info (subclass should persist these):
+        self._derivation_prefix = derivation_prefix  # type: Optional[str]
+        self._root_fingerprint = root_fingerprint  # type: Optional[str]
 
     def get_master_public_key(self):
         return self.xpub
 
-    def get_derivation_prefix(self) -> str:
-        """Returns to bip32 path from some root node to self.xpub"""
-        assert self._derivation_prefix is not None, 'derivation_prefix should have been set already'
+    def get_derivation_prefix(self) -> Optional[str]:
+        """Returns to bip32 path from some root node to self.xpub
+        Note that the return value might be None; if it is unknown.
+        """
         return self._derivation_prefix
 
-    def get_root_fingerprint(self) -> str:
+    def get_root_fingerprint(self) -> Optional[str]:
         """Returns the bip32 fingerprint of the top level node.
         This top level node is the node at the beginning of the derivation prefix,
         i.e. applying the derivation prefix to it will result self.xpub
+        Note that the return value might be None; if it is unknown.
         """
-        assert self._root_fingerprint is not None, 'root_fingerprint should have been set already'
         return self._root_fingerprint
 
-    def add_derivation_prefix_and_root_fingerprint(self, *, derivation_prefix: str, root_node: BIP32Node):
+    def get_fp_and_derivation_to_be_used_in_partial_tx(self, der_suffix: Sequence[int]) -> Tuple[bytes, Sequence[int]]:
+        """Returns fingerprint and 'full' derivation path corresponding to a derivation suffix.
+        The fingerprint is either the root fp or the intermediate fp, depending on what is available,
+        and the 'full' derivation path is adjusted accordingly.
+        """
+        fingerprint_hex = self.get_root_fingerprint()
+        der_prefix_str = self.get_derivation_prefix()
+        if fingerprint_hex is not None and der_prefix_str is not None:
+            # use root fp, and true full path
+            fingerprint_bytes = bfh(fingerprint_hex)
+            der_prefix_ints = convert_bip32_path_to_list_of_uint32(der_prefix_str)
+        else:
+            # use intermediate fp, and claim der suffix is the full path
+            fingerprint_bytes = BIP32Node.from_xkey(self.xpub).calc_fingerprint_of_this_node()
+            der_prefix_ints = convert_bip32_path_to_list_of_uint32('m')
+        der_full = der_prefix_ints + list(der_suffix)
+        return fingerprint_bytes, der_full
+
+    def get_xpub_to_be_used_in_partial_tx(self) -> str:
+        assert self.xpub
+        fp_bytes, der_full = self.get_fp_and_derivation_to_be_used_in_partial_tx(der_suffix=[])
+        bip32node = BIP32Node.from_xkey(self.xpub)
+        depth = len(der_full)
+        child_number_int = der_full[-1] if len(der_full) >= 1 else 0
+        child_number_bytes = child_number_int.to_bytes(length=4, byteorder="big")
+        fingerprint = bytes(4) if depth == 0 else bip32node.fingerprint
+        bip32node = bip32node._replace(depth=depth,
+                                       fingerprint=fingerprint,
+                                       child_number=child_number_bytes)
+        return bip32node.to_xpub()
+
+    def add_key_origin_from_root_node(self, *, derivation_prefix: str, root_node: BIP32Node):
         assert self.xpub
-        derivation_prefix = normalize_bip32_derivation(derivation_prefix)
         # try to derive ourselves from what we were given
         child_node1 = root_node.subkey_at_private_derivation(derivation_prefix)
         child_pubkey_bytes1 = child_node1.eckey.get_public_key_bytes(compressed=True)
@@ -356,14 +389,13 @@ class Xpub:
         child_pubkey_bytes2 = child_node2.eckey.get_public_key_bytes(compressed=True)
         if child_pubkey_bytes1 != child_pubkey_bytes2:
             raise Exception("(xpub, derivation_prefix, root_node) inconsistency")
-        # store
-        self._root_fingerprint = root_node.calc_fingerprint_of_this_node().hex().lower()
-        self._derivation_prefix = derivation_prefix
+        self.add_key_origin(derivation_prefix=derivation_prefix,
+                            root_fingerprint=root_node.calc_fingerprint_of_this_node().hex().lower())
 
-    def reset_derivation_prefix(self):
+    def add_key_origin(self, *, derivation_prefix: Optional[str], root_fingerprint: Optional[str]):
         assert self.xpub
-        self._derivation_prefix = 'm'
-        self._root_fingerprint = BIP32Node.from_xkey(self.xpub).calc_fingerprint_of_this_node().hex().lower()
+        self._root_fingerprint = root_fingerprint
+        self._derivation_prefix = normalize_bip32_derivation(derivation_prefix)
 
     def derive_pubkey(self, for_change, n) -> str:
         for_change = int(for_change)
@@ -431,20 +463,22 @@ class BIP32_KeyStore(Deterministic_KeyStore, Xpub):
     def is_watching_only(self):
         return self.xprv is None
 
-    def add_xpub(self, xpub, *, default_der_prefix=True):
+    def add_xpub(self, xpub):
+        assert is_xpub(xpub)
         self.xpub = xpub
-        if default_der_prefix:
-            self.reset_derivation_prefix()
+        root_fingerprint, derivation_prefix = bip32.root_fp_and_der_prefix_from_xkey(xpub)
+        self.add_key_origin(derivation_prefix=derivation_prefix, root_fingerprint=root_fingerprint)
 
-    def add_xprv(self, xprv, *, default_der_prefix=True):
+    def add_xprv(self, xprv):
+        assert is_xprv(xprv)
         self.xprv = xprv
-        self.add_xpub(bip32.xpub_from_xprv(xprv), default_der_prefix=default_der_prefix)
+        self.add_xpub(bip32.xpub_from_xprv(xprv))
 
     def add_xprv_from_seed(self, bip32_seed, xtype, derivation):
         rootnode = BIP32Node.from_rootseed(bip32_seed, xtype=xtype)
         node = rootnode.subkey_at_private_derivation(derivation)
-        self.add_xprv(node.to_xprv(), default_der_prefix=False)
-        self.add_derivation_prefix_and_root_fingerprint(derivation_prefix=derivation, root_node=rootnode)
+        self.add_xprv(node.to_xprv())
+        self.add_key_origin_from_root_node(derivation_prefix=derivation, root_node=rootnode)
 
     def get_private_key(self, sequence, password):
         xprv = self.get_master_private_key(password)
@@ -568,6 +602,15 @@ class Old_KeyStore(Deterministic_KeyStore):
             self._root_fingerprint = xfp.hex().lower()
         return self._root_fingerprint
 
+    # TODO Old_KeyStore and Xpub could share a common baseclass?
+    def get_fp_and_derivation_to_be_used_in_partial_tx(self, der_suffix: Sequence[int]) -> Tuple[bytes, Sequence[int]]:
+        fingerprint_hex = self.get_root_fingerprint()
+        der_prefix_str = self.get_derivation_prefix()
+        fingerprint_bytes = bfh(fingerprint_hex)
+        der_prefix_ints = convert_bip32_path_to_list_of_uint32(der_prefix_str)
+        der_full = der_prefix_ints + list(der_suffix)
+        return fingerprint_bytes, der_full
+
     def update_password(self, old_password, new_password):
         self.check_password(old_password)
         if new_password == '':
@@ -658,6 +701,14 @@ class Hardware_KeyStore(KeyStore, Xpub):
     def ready_to_sign(self):
         return super().ready_to_sign() and self.has_usable_connection_with_device()
 
+    def opportunistically_fill_in_missing_info_from_device(self, client):
+        assert client is not None
+        if self._root_fingerprint is None:
+            root_xpub = client.get_xpub('m', xtype='standard')
+            root_fingerprint = BIP32Node.from_xkey(root_xpub).calc_fingerprint_of_this_node().hex().lower()
+            self._root_fingerprint = root_fingerprint
+            self.is_requesting_to_be_rewritten_to_wallet_file = True
+
 
 def bip39_normalize_passphrase(passphrase):
     return normalize('NFKD', passphrase or '')
@@ -718,16 +769,17 @@ PURPOSE48_SCRIPT_TYPES_INV = inv_dict(PURPOSE48_SCRIPT_TYPES)
 
 def xtype_from_derivation(derivation: str) -> str:
     """Returns the script type to be used for this derivation."""
-    if derivation.startswith("m/84'"):
-        return 'p2wpkh'
-    elif derivation.startswith("m/49'"):
-        return 'p2wpkh-p2sh'
-    elif derivation.startswith("m/44'"):
-        return 'standard'
-    elif derivation.startswith("m/45'"):
-        return 'standard'
-
     bip32_indices = convert_bip32_path_to_list_of_uint32(derivation)
+    if len(bip32_indices) >= 1:
+        if bip32_indices[0] == 84 + BIP32_PRIME:
+            return 'p2wpkh'
+        elif bip32_indices[0] == 49 + BIP32_PRIME:
+            return 'p2wpkh-p2sh'
+        elif bip32_indices[0] == 44 + BIP32_PRIME:
+            return 'standard'
+        elif bip32_indices[0] == 45 + BIP32_PRIME:
+            return 'standard'
+
     if len(bip32_indices) >= 4:
         if bip32_indices[0] == 48 + BIP32_PRIME:
             # m / purpose' / coin_type' / account' / script_type' / change / address_index
@@ -770,7 +822,7 @@ def load_keystore(storage, name) -> KeyStore:
 
 def is_old_mpk(mpk: str) -> bool:
     try:
-        int(mpk, 16)
+        int(mpk, 16)  # test if hex string
     except:
         return False
     if len(mpk) != 128:
@@ -804,16 +856,18 @@ def is_private_key_list(text, *, allow_spaces_inside_key=True, raise_on_error=Fa
                                  raise_on_error=raise_on_error))
 
 
-is_mpk = lambda x: is_old_mpk(x) or is_xpub(x)
-is_private = lambda x: is_seed(x) or is_xprv(x) or is_private_key_list(x)
-is_master_key = lambda x: is_old_mpk(x) or is_xprv(x) or is_xpub(x)
-is_private_key = lambda x: is_xprv(x) or is_private_key_list(x)
-is_bip32_key = lambda x: is_xprv(x) or is_xpub(x)
+def is_master_key(x):
+    return is_old_mpk(x) or is_bip32_key(x)
+
+
+def is_bip32_key(x):
+    return is_xprv(x) or is_xpub(x)
 
 
 def bip44_derivation(account_id, bip43_purpose=44):
     coin = constants.net.BIP44_COIN_TYPE
-    return "m/%d'/%d'/%d'" % (bip43_purpose, coin, int(account_id))
+    der = "m/%d'/%d'/%d'" % (bip43_purpose, coin, int(account_id))
+    return normalize_bip32_derivation(der)
 
 
 def purpose48_derivation(account_id: int, xtype: str) -> str:
@@ -824,7 +878,8 @@ def purpose48_derivation(account_id: int, xtype: str) -> str:
     script_type_int = PURPOSE48_SCRIPT_TYPES.get(xtype)
     if script_type_int is None:
         raise Exception('unknown xtype: {}'.format(xtype))
-    return "m/%d'/%d'/%d'/%d'" % (bip43_purpose, coin, account_id, script_type_int)
+    der = "m/%d'/%d'/%d'/%d'" % (bip43_purpose, coin, account_id, script_type_int)
+    return normalize_bip32_derivation(der)
 
 
 def from_seed(seed, passphrase, is_p2sh=False):

electrum/plugin.py

@@ -39,6 +39,7 @@ from .logging import get_logger, Logger
 
 if TYPE_CHECKING:
     from .plugins.hw_wallet import HW_PluginBase
+    from .keystore import Hardware_KeyStore
 
 
 _logger = get_logger(__name__)
@@ -442,7 +443,7 @@ class DeviceMgr(ThreadJob):
         self.scan_devices()
         return self.client_lookup(id_)
 
-    def client_for_keystore(self, plugin, handler, keystore, force_pair):
+    def client_for_keystore(self, plugin, handler, keystore: 'Hardware_KeyStore', force_pair):
         self.logger.info("getting client for keystore")
         if handler is None:
             raise Exception(_("Handler not found for") + ' ' + plugin.name + '\n' + _("A library is probably missing."))
@@ -450,12 +451,15 @@ class DeviceMgr(ThreadJob):
         devices = self.scan_devices()
         xpub = keystore.xpub
         derivation = keystore.get_derivation_prefix()
+        assert derivation is not None
         client = self.client_by_xpub(plugin, xpub, handler, devices)
         if client is None and force_pair:
             info = self.select_device(plugin, handler, keystore, devices)
             client = self.force_pair_xpub(plugin, handler, info, xpub, derivation, devices)
         if client:
             handler.update_status(True)
+        if client:
+            keystore.opportunistically_fill_in_missing_info_from_device(client)
         self.logger.info("end client for keystore")
         return client
 

electrum/plugins/coldcard/coldcard.py

@@ -266,13 +266,18 @@ class Coldcard_KeyStore(Hardware_KeyStore):
         d['ckcc_xpub'] = self.ckcc_xpub
         return d
 
+    def get_xfp_int(self) -> int:
+        xfp = self.get_root_fingerprint()
+        assert xfp is not None
+        return xfp_int_from_xfp_bytes(bfh(xfp))
+
     def get_client(self):
         # called when user tries to do something like view address, sign somthing.
         # - not called during probing/setup
         # - will fail if indicated device can't produce the xpub (at derivation) expected
         rv = self.plugin.get_client(self)
         if rv:
-            xfp_int = xfp_int_for_keystore(self)
+            xfp_int = self.get_xfp_int()
             rv.verify_connection(xfp_int, self.ckcc_xpub)
 
         return rv
@@ -363,7 +368,7 @@ class Coldcard_KeyStore(Hardware_KeyStore):
 
         client = self.get_client()
 
-        assert client.dev.master_fingerprint == xfp_int_for_keystore(self)
+        assert client.dev.master_fingerprint == self.get_xfp_int()
 
         raw_psbt = tx.serialize_as_bytes()
 
@@ -570,9 +575,11 @@ class ColdcardPlugin(HW_PluginBase):
         xpubs = []
         derivs = set()
         for xpub, ks in zip(wallet.get_master_public_keys(), wallet.get_keystores()):
-            der_prefix = ks.get_derivation_prefix()
-            xpubs.append( (ks.get_root_fingerprint(), xpub, der_prefix) )
-            derivs.add(der_prefix)
+            fp_bytes, der_full = ks.get_fp_and_derivation_to_be_used_in_partial_tx(der_suffix=[])
+            fp_hex = fp_bytes.hex().upper()
+            der_prefix_str = bip32.convert_bip32_intpath_to_strpath(der_full)
+            xpubs.append( (fp_hex, xpub, der_prefix_str) )
+            derivs.add(der_prefix_str)
 
         # Derivation doesn't matter too much to the Coldcard, since it
         # uses key path data from PSBT or USB request as needed. However,
@@ -613,10 +620,9 @@ class ColdcardPlugin(HW_PluginBase):
             xfp_paths = []
             for pubkey_hex in pubkey_deriv_info:
                 ks, der_suffix = pubkey_deriv_info[pubkey_hex]
-                xfp_int = xfp_int_for_keystore(ks)
-                der_prefix = bip32.convert_bip32_path_to_list_of_uint32(ks.get_derivation_prefix())
-                der_full = der_prefix + list(der_suffix)
-                xfp_paths.append([xfp_int] + der_full)
+                fp_bytes, der_full = ks.get_fp_and_derivation_to_be_used_in_partial_tx(der_suffix)
+                xfp_int = xfp_int_from_xfp_bytes(fp_bytes)
+                xfp_paths.append([xfp_int] + list(der_full))
 
             script = bfh(wallet.pubkeys_to_scriptcode(pubkeys))
 
@@ -627,9 +633,8 @@ class ColdcardPlugin(HW_PluginBase):
             return
 
 
-def xfp_int_for_keystore(keystore: Xpub) -> int:
-    xfp = keystore.get_root_fingerprint()
-    return int.from_bytes(bfh(xfp), byteorder="little", signed=False)
+def xfp_int_from_xfp_bytes(fp_bytes: bytes) -> int:
+    return int.from_bytes(fp_bytes, byteorder="little", signed=False)
 
 
 def xfp2str(xfp: int) -> str:

electrum/plugins/hw_wallet/plugin.py

@@ -72,6 +72,9 @@ class HW_PluginBase(BasePlugin):
         """
         raise NotImplementedError()
 
+    def get_client(self, keystore: 'Hardware_KeyStore', force_pair: bool = True):
+        raise NotImplementedError()
+
     def show_address(self, wallet: 'Abstract_Wallet', address, keystore: 'Hardware_KeyStore' = None):
         pass  # implemented in child classes
 

electrum/tests/test_transaction.py

@@ -772,7 +772,7 @@ class TestLegacyPartialTxFormat(TestCaseForTestnet):
 
         wallet = WalletIntegrityHelper.create_multisig_wallet([ks1, ks2, ks3], '2of3', config=self.config)
 
-        tx = tx_from_any('cHNidP8BAJQCAAAAAcqqxrXrkW4wZ9AiT5QvszHOHc+0Axz7R555Qdz5XkCYAQAAAAD9////A6CGAQAAAAAAFgAU+fBLRlKk9v89xVEm2xJ0kG1wcvNMCwMAAAAAABepFPKffLiXEB3Gmv1Y35uy5bTUM59Nh0ANAwAAAAAAGXapFPriyJZefiOenIisUU3nDewLDxYIiKwSKxgATwEENYfPAAAAAAAAAAAAnOMnCVq57ruCJ7c38H6PtmrwS48+kcQJPEh70w/ofCQCDSEN062A0pw2JKkYltX2G3th8zLexPfEVDGu74BeD6cEcH3xxE8BBDWHzwGCB4l2gAAAAJOfYJjOAH6kksFOokIboP3+8Gwhlzlxhl5uY7zokvfcAmGy8e8txy0wkx69/TgZFOMe1aZc2g1HCwrRQ9M9+Ph7BLoE1bNPAQQ1h88BggeJdoAAAAFvoSi9wKWkb8evGv0gXbYgcZHUxAUbbtvQZBPNwLGi3wNqTky+e8Rm3WU3hMhrN3Sb7D6CgCnOo0wVaQlW/WFXbwQqQERnAAEA3wIAAAABnCh8O3p5TIeiImyJBHikJS+aVEdsCr+hgtTU3eimPIIBAAAAakcwRAIgTR7BvR+c9dHhx7CDnuJBXb52St/BOycfgpq7teEnUP4CIFp4DWI/xfKhwIZHZVPgYGOZLQC9jHiFKKiCSl7nXBUTASEDVPOeil/J5isfPp2yEcI6UQL8jFq6CRs/hegA8M2L/xj9////ApydBwAAAAAAGXapFLEFQG/gWw3IvkTHg2ulgS2/Z0zoiKwgoQcAAAAAABepFCwWF9JPk25A0UsN8pTst7uq11M3hxIrGAAiAgP+Qtq1hxjqBBP3yN5pPN7uIs4Zsdw0wLvdekgkVGXFokgwRQIhANA8NcspOwHae+DXcc7a+oke1dcKZ3z8zWlnE1b2vr7cAiALKldsTNHGQkgHVs4f1mCsrwulJhk6MJnh+BzdAa0gkwEBBGlSIQIJHwtNirMAFqXRwIgkngKIP62BYPBvpTWIrYWYZQo+YiEDXy+CY7s2CNbMTuA71MuNZcTXCvcQSfBfv+5JeIMqH9IhA/5C2rWHGOoEE/fI3mk83u4izhmx3DTAu916SCRUZcWiU64iBgP+Qtq1hxjqBBP3yN5pPN7uIs4Zsdw0wLvdekgkVGXFogy6BNWzAAAAAAAAAAAiBgIJHwtNirMAFqXRwIgkngKIP62BYPBvpTWIrYWYZQo+YgwqQERnAAAAAAAAAAAiBgNfL4JjuzYI1sxO4DvUy41lxNcK9xBJ8F+/7kl4gyof0gxwffHEAAAAAAAAAAAAAAEAaVIhAiq2ef2i4zfECrvAggPT1x0qlv2784yQj3uS5TfBxO//IQNM0UcWdnXGYoZFDMVpQBP0N4OTY115ZKuKVzzD0EqNTiEDWOhaggFFh96oM+tf+5PjmFaQ7kumHvMtWyitWqFhc39TriICA0zRRxZ2dcZihkUMxWlAE/Q3g5NjXXlkq4pXPMPQSo1ODLoE1bMBAAAAAAAAACICAiq2ef2i4zfECrvAggPT1x0qlv2784yQj3uS5TfBxO//DCpARGcBAAAAAAAAACICA1joWoIBRYfeqDPrX/uT45hWkO5Lph7zLVsorVqhYXN/DHB98cQBAAAAAAAAAAAA')
+        tx = tx_from_any('cHNidP8BAJQCAAAAAcqqxrXrkW4wZ9AiT5QvszHOHc+0Axz7R555Qdz5XkCYAQAAAAD9////A6CGAQAAAAAAFgAU+fBLRlKk9v89xVEm2xJ0kG1wcvNMCwMAAAAAABepFPKffLiXEB3Gmv1Y35uy5bTUM59Nh0ANAwAAAAAAGXapFPriyJZefiOenIisUU3nDewLDxYIiKwSKxgATwEENYfPAAAAAAAAAAAAnOMnCVq57ruCJ7c38H6PtmrwS48+kcQJPEh70w/ofCQCDSEN062A0pw2JKkYltX2G3th8zLexPfEVDGu74BeD6cEcH3xxE8BBDWHzwGCB4l2gAAAAJOfYJjOAH6kksFOokIboP3+8Gwhlzlxhl5uY7zokvfcAmGy8e8txy0wkx69/TgZFOMe1aZc2g1HCwrRQ9M9+Ph7CIIHiXYAAACATwEENYfPAYIHiXaAAAABb6EovcClpG/Hrxr9IF22IHGR1MQFG27b0GQTzcCxot8Dak5MvnvEZt1lN4TIazd0m+w+goApzqNMFWkJVv1hV28IggeJdgEAAIAAAQDfAgAAAAGcKHw7enlMh6IibIkEeKQlL5pUR2wKv6GC1NTd6KY8ggEAAABqRzBEAiBNHsG9H5z10eHHsIOe4kFdvnZK38E7Jx+Cmru14SdQ/gIgWngNYj/F8qHAhkdlU+BgY5ktAL2MeIUoqIJKXudcFRMBIQNU856KX8nmKx8+nbIRwjpRAvyMWroJGz+F6ADwzYv/GP3///8CnJ0HAAAAAAAZdqkUsQVAb+BbDci+RMeDa6WBLb9nTOiIrCChBwAAAAAAF6kULBYX0k+TbkDRSw3ylOy3u6rXUzeHEisYACICA/5C2rWHGOoEE/fI3mk83u4izhmx3DTAu916SCRUZcWiSDBFAiEA0Dw1yyk7Adp74Ndxztr6iR7V1wpnfPzNaWcTVva+vtwCIAsqV2xM0cZCSAdWzh/WYKyvC6UmGTowmeH4HN0BrSCTAQEEaVIhAgkfC02KswAWpdHAiCSeAog/rYFg8G+lNYithZhlCj5iIQNfL4JjuzYI1sxO4DvUy41lxNcK9xBJ8F+/7kl4gyof0iED/kLatYcY6gQT98jeaTze7iLOGbHcNMC73XpIJFRlxaJTriIGA/5C2rWHGOoEE/fI3mk83u4izhmx3DTAu916SCRUZcWiEIIHiXYAAACAAAAAAAAAAAAiBgIJHwtNirMAFqXRwIgkngKIP62BYPBvpTWIrYWYZQo+YhCCB4l2AQAAgAAAAAAAAAAAIgYDXy+CY7s2CNbMTuA71MuNZcTXCvcQSfBfv+5JeIMqH9IMcH3xxAAAAAAAAAAAAAABAGlSIQIqtnn9ouM3xAq7wIID09cdKpb9u/OMkI97kuU3wcTv/yEDTNFHFnZ1xmKGRQzFaUAT9DeDk2NdeWSrilc8w9BKjU4hA1joWoIBRYfeqDPrX/uT45hWkO5Lph7zLVsorVqhYXN/U64iAgNM0UcWdnXGYoZFDMVpQBP0N4OTY115ZKuKVzzD0EqNThCCB4l2AAAAgAEAAAAAAAAAIgICKrZ5/aLjN8QKu8CCA9PXHSqW/bvzjJCPe5LlN8HE7/8QggeJdgEAAIABAAAAAAAAACICA1joWoIBRYfeqDPrX/uT45hWkO5Lph7zLVsorVqhYXN/DHB98cQBAAAAAAAAAAAA')
         tx.add_info_from_wallet(wallet)
         raw_tx = serialize_tx_in_legacy_format(tx, wallet=wallet)
         self.assertEqual('45505446ff000200000001caaac6b5eb916e3067d0224f942fb331ce1dcfb4031cfb479e7941dcf95e409801000000fd53010001ff01ff483045022100d03c35cb293b01da7be0d771cedafa891ed5d70a677cfccd69671356f6bebedc02200b2a576c4cd1c642480756ce1fd660acaf0ba526193a3099e1f81cdd01ad2093014d0201524c53ff043587cf0182078976800000016fa128bdc0a5a46fc7af1afd205db6207191d4c4051b6edbd06413cdc0b1a2df036a4e4cbe7bc466dd653784c86b37749bec3e828029cea34c15690956fd61576f000000004c53ff043587cf0000000000000000009ce327095ab9eebb8227b737f07e8fb66af04b8f3e91c4093c487bd30fe87c24020d210dd3ad80d29c3624a91896d5f61b7b61f332dec4f7c45431aeef805e0fa7000000004c53ff043587cf018207897680000000939f6098ce007ea492c14ea2421ba0fdfef06c21973971865e6e63bce892f7dc0261b2f1ef2dc72d30931ebdfd381914e31ed5a65cda0d470b0ad143d33df8f87b0000000053aefdffffff03a086010000000000160014f9f04b4652a4f6ff3dc55126db1274906d7072f34c0b03000000000017a914f29f7cb897101dc69afd58df9bb2e5b4d4339f4d87400d0300000000001976a914fae2c8965e7e239e9c88ac514de70dec0b0f160888ac122b1800',
@@ -795,7 +795,7 @@ class TestLegacyPartialTxFormat(TestCaseForTestnet):
 
         wallet = WalletIntegrityHelper.create_multisig_wallet([ks1, ks2, ks3], '2of3', config=self.config)
 
-        tx = tx_from_any('cHNidP8BAJ8CAAAAAYfEZGymkLOX41eyOyE3AwaRqQoGimaQg000C0voSs1qAQAAAAD9////A6CGAQAAAAAAFgAUi8nZR+TQrdwvTDS4NxA060ez0wUUDAMAAAAAACIAIKYIfE+EpV3DkBRymhjgiVUTnUOEVZ0f0qSKHZXXRqQlQA0DAAAAAAAZdqkUwT/WKU0b57lBClU49LTvEPxZTueIrBMrGABPAQJXVIMAAAAAAAAAAADWRNzdekrLQyNV4BCsSl+VWUDIKpdncxt9idxC6zzaxAJy+qL5i3bMnWVe8oHAes2nXDCpkNw6Unts+SqPWmuKgARL8hIJTwECV1SDAdJM1ZGAAAABmcTWyJP6Gt3sawEhGBE34lw4GUMzuMVyFbPPHm1+evECoj3a5pi7YJW4uANb3R6UR59mwpZ52Bkx4P4HqkNhSe4E5U2yWk8BAldUgwHSTNWRgAAAALYKhQia0IUKb/6FkKPhUGTmPu/QIIE98uSmsyCc499fAsixTykX1VfNSClwwRTD40V+CdJWOXgCJ3ouTRUZq5+MBAHZMVwAAQEqIKEHAAAAAAAAIKlI1/pqu7l+MXea5UODAStBPVOCHH/TlJAPa0Q8Yd7uIgIDB6PEHQftl21l4hPoI9AoQJN0decJtBJT6F6XDjyxZnRHMEQCIC975frTmPGjV2KTM58idNInrxd5hpCqGtAP+D2WclzFAiAAyy6f/1viOoYnGMZlpQNfFyeD6bMVjq6DZz9sWa3DwAEBBWlSIQKp3LVw6CgMdB8JAywVgJW3qjsM5AGtoDDy1HuZnwIGBiEDB6PEHQftl21l4hPoI9AoQJN0decJtBJT6F6XDjyxZnQhA1IbCkXgQvCMzQOvR/2IuyB7VBTg4wu4eZ/KMRoGMjoZU64iBgMHo8QdB+2XbWXiE+gj0ChAk3R15wm0ElPoXpcOPLFmdAwB2TFcAAAAAAEAAAAiBgNSGwpF4ELwjM0Dr0f9iLsge1QU4OMLuHmfyjEaBjI6GQzlTbJaAAAAAAEAAAAiBgKp3LVw6CgMdB8JAywVgJW3qjsM5AGtoDDy1HuZnwIGBgxL8hIJAAAAAAEAAAAAAAEBaVIhAgWCn5UiV3EiypypvrZ/lM8v4K0NF+BSoRBwHPQSjTOcIQIsyigs/dnMHzh9MJhmHWi8nIo5rGvXLDDbV5p4V9CFmyEC2Q48iXOES5zAs4SUxIUVoxnuw6wkiflvb1XmqmkSpghTriICAtkOPIlzhEucwLOElMSFFaMZ7sOsJIn5b29V5qppEqYIDAHZMVwBAAAAAAAAACICAizKKCz92cwfOH0wmGYdaLycijmsa9csMNtXmnhX0IWbDOVNsloBAAAAAAAAACICAgWCn5UiV3EiypypvrZ/lM8v4K0NF+BSoRBwHPQSjTOcDEvyEgkBAAAAAAAAAAAA')
+        tx = tx_from_any('cHNidP8BAJ8CAAAAAYfEZGymkLOX41eyOyE3AwaRqQoGimaQg000C0voSs1qAQAAAAD9////A6CGAQAAAAAAFgAUi8nZR+TQrdwvTDS4NxA060ez0wUUDAMAAAAAACIAIKYIfE+EpV3DkBRymhjgiVUTnUOEVZ0f0qSKHZXXRqQlQA0DAAAAAAAZdqkUwT/WKU0b57lBClU49LTvEPxZTueIrBMrGABPAQJXVIMAAAAAAAAAAADWRNzdekrLQyNV4BCsSl+VWUDIKpdncxt9idxC6zzaxAJy+qL5i3bMnWVe8oHAes2nXDCpkNw6Unts+SqPWmuKgARL8hIJTwECV1SDAdJM1ZGAAAABmcTWyJP6Gt3sawEhGBE34lw4GUMzuMVyFbPPHm1+evECoj3a5pi7YJW4uANb3R6UR59mwpZ52Bkx4P4HqkNhSe4I0kzVkQEAAIBPAQJXVIMB0kzVkYAAAAC2CoUImtCFCm/+hZCj4VBk5j7v0CCBPfLkprMgnOPfXwLIsU8pF9VXzUgpcMEUw+NFfgnSVjl4Aid6Lk0VGaufjAjSTNWRAAAAgAABASogoQcAAAAAAAAgqUjX+mq7uX4xd5rlQ4MBK0E9U4Icf9OUkA9rRDxh3u4iAgMHo8QdB+2XbWXiE+gj0ChAk3R15wm0ElPoXpcOPLFmdEcwRAIgL3vl+tOY8aNXYpMznyJ00ievF3mGkKoa0A/4PZZyXMUCIADLLp//W+I6hicYxmWlA18XJ4PpsxWOroNnP2xZrcPAAQEFaVIhAqnctXDoKAx0HwkDLBWAlbeqOwzkAa2gMPLUe5mfAgYGIQMHo8QdB+2XbWXiE+gj0ChAk3R15wm0ElPoXpcOPLFmdCEDUhsKReBC8IzNA69H/Yi7IHtUFODjC7h5n8oxGgYyOhlTriIGAwejxB0H7ZdtZeIT6CPQKECTdHXnCbQSU+helw48sWZ0ENJM1ZEAAACAAAAAAAEAAAAiBgNSGwpF4ELwjM0Dr0f9iLsge1QU4OMLuHmfyjEaBjI6GRDSTNWRAQAAgAAAAAABAAAAIgYCqdy1cOgoDHQfCQMsFYCVt6o7DOQBraAw8tR7mZ8CBgYMS/ISCQAAAAABAAAAAAABAWlSIQIFgp+VIldxIsqcqb62f5TPL+CtDRfgUqEQcBz0Eo0znCECLMooLP3ZzB84fTCYZh1ovJyKOaxr1yww21eaeFfQhZshAtkOPIlzhEucwLOElMSFFaMZ7sOsJIn5b29V5qppEqYIU64iAgLZDjyJc4RLnMCzhJTEhRWjGe7DrCSJ+W9vVeaqaRKmCBDSTNWRAAAAgAEAAAAAAAAAIgICLMooLP3ZzB84fTCYZh1ovJyKOaxr1yww21eaeFfQhZsQ0kzVkQEAAIABAAAAAAAAACICAgWCn5UiV3EiypypvrZ/lM8v4K0NF+BSoRBwHPQSjTOcDEvyEgkBAAAAAAAAAAAA')
         tx.add_info_from_wallet(wallet)
         raw_tx = serialize_tx_in_legacy_format(tx, wallet=wallet)
         self.assertEqual('45505446ff000200000000010187c4646ca690b397e357b23b2137030691a90a068a6690834d340b4be84acd6a0100000000fdffffff03a0860100000000001600148bc9d947e4d0addc2f4c34b8371034eb47b3d305140c030000000000220020a6087c4f84a55dc39014729a18e08955139d4384559d1fd2a48a1d95d746a425400d0300000000001976a914c13fd6294d1be7b9410a5538f4b4ef10fc594ee788acfeffffffff20a10700000000000000050001ff47304402202f7be5fad398f1a3576293339f2274d227af17798690aa1ad00ff83d96725cc5022000cb2e9fff5be23a862718c665a5035f172783e9b3158eae83673f6c59adc3c00101fffd0201524c53ff02575483000000000000000000d644dcdd7a4acb432355e010ac4a5f955940c82a9767731b7d89dc42eb3cdac40272faa2f98b76cc9d655ef281c07acda75c30a990dc3a527b6cf92a8f5a6b8a80000001004c53ff0257548301d24cd59180000000b60a85089ad0850a6ffe8590a3e15064e63eefd020813df2e4a6b3209ce3df5f02c8b14f2917d557cd482970c114c3e3457e09d256397802277a2e4d1519ab9f8c000001004c53ff0257548301d24cd5918000000199c4d6c893fa1addec6b0121181137e25c38194333b8c57215b3cf1e6d7e7af102a23ddae698bb6095b8b8035bdd1e94479f66c29679d81931e0fe07aa436149ee0000010053ae132b1800',

electrum/transaction.py

@@ -1453,7 +1453,8 @@ class PartialTransaction(Transaction):
                         raise SerializationError(f"duplicate key: {repr(kt)}")
                     xfp, path = unpack_bip32_root_fingerprint_and_int_path(val)
                     if bip32node.depth != len(path):
-                        raise SerializationError(f"PSBT global xpub has mismatching depth and derivation prefix len")
+                        raise SerializationError(f"PSBT global xpub has mismatching depth ({bip32node.depth}) "
+                                                 f"and derivation prefix len ({len(path)})")
                     child_number_of_xpub = int.from_bytes(bip32node.child_number, 'big')
                     if not ((bip32node.depth == 0 and child_number_of_xpub == 0)
                             or (bip32node.depth != 0 and child_number_of_xpub == path[-1])):
@@ -1736,10 +1737,9 @@ class PartialTransaction(Transaction):
         from .keystore import Xpub
         for ks in wallet.get_keystores():
             if isinstance(ks, Xpub):
-                bip32node = BIP32Node.from_xkey(ks.get_master_public_key())
-                xfp_bytes = bfh(ks.get_root_fingerprint())
-                der_prefix = bip32.convert_bip32_path_to_list_of_uint32(ks.get_derivation_prefix())
-                self.xpubs[bip32node] = (xfp_bytes, der_prefix)
+                fp_bytes, der_full = ks.get_fp_and_derivation_to_be_used_in_partial_tx(der_suffix=[])
+                bip32node = BIP32Node.from_xkey(ks.get_xpub_to_be_used_in_partial_tx())
+                self.xpubs[bip32node] = (fp_bytes, der_full)
         for txin in self.inputs():
             wallet.add_input_info(txin)
         for txout in self.outputs():

electrum/wallet.py

@@ -285,6 +285,8 @@ class Abstract_Wallet(AddressSynchronizer):
 
     def stop_threads(self):
         super().stop_threads()
+        if any([ks.is_requesting_to_be_rewritten_to_wallet_file for ks in self.get_keystores()]):
+            self.save_keystore()
         self.storage.write()
 
     def set_up_to_date(self, b):
@@ -318,6 +320,9 @@ class Abstract_Wallet(AddressSynchronizer):
     def get_master_public_key(self):
         return None
 
+    def get_master_public_keys(self):
+        return []
+
     def basename(self) -> str:
         return os.path.basename(self.storage.path)
 
@@ -1224,6 +1229,9 @@ class Abstract_Wallet(AddressSynchronizer):
     def _add_input_sig_info(self, txin: PartialTxInput, address: str) -> None:
         raise NotImplementedError()  # implemented by subclasses
 
+    def _add_txinout_derivation_info(self, txinout: Union[PartialTxInput, PartialTxOutput], address: str) -> None:
+        pass  # implemented by subclasses
+
     def _add_input_utxo_info(self, txin: PartialTxInput, address: str) -> None:
         if Transaction.is_segwit_input(txin):
             if txin.witness_utxo is None:
@@ -1312,16 +1320,7 @@ class Abstract_Wallet(AddressSynchronizer):
         txout.is_change = self.is_change(address)
         if isinstance(self, Multisig_Wallet):
             txout.num_sig = self.m
-        if isinstance(self, Deterministic_Wallet):
-            if not txout.pubkeys or len(txout.pubkeys) != len(txout.bip32_paths):
-                pubkey_deriv_info = self.get_public_keys_with_deriv_info(address)
-                txout.pubkeys = sorted([bfh(pk) for pk in list(pubkey_deriv_info)])
-                for pubkey_hex in pubkey_deriv_info:
-                    ks, der_suffix = pubkey_deriv_info[pubkey_hex]
-                    xfp_bytes = bfh(ks.get_root_fingerprint())
-                    der_prefix = bip32.convert_bip32_path_to_list_of_uint32(ks.get_derivation_prefix())
-                    der_full = der_prefix + list(der_suffix)
-                    txout.bip32_paths[bfh(pubkey_hex)] = (xfp_bytes, der_full)
+        self._add_txinout_derivation_info(txout, address)
         if txout.redeem_script is None:
             try:
                 redeem_script_hex = self.get_redeem_script(address)
@@ -1721,6 +1720,9 @@ class Abstract_Wallet(AddressSynchronizer):
     def get_keystores(self) -> Sequence[KeyStore]:
         return [self.keystore] if self.keystore else []
 
+    def save_keystore(self):
+        raise NotImplementedError()
+
 
 class Simple_Wallet(Abstract_Wallet):
     # wallet with a single keystore
@@ -1773,9 +1775,6 @@ class Imported_Wallet(Simple_Wallet):
     def is_change(self, address):
         return False
 
-    def get_master_public_keys(self):
-        return []
-
     def is_beyond_limit(self, address):
         return False
 
@@ -1903,7 +1902,8 @@ class Imported_Wallet(Simple_Wallet):
         return self.db.get_imported_address(address).get('type', 'address')
 
     def _add_input_sig_info(self, txin, address):
-        assert self.is_mine(address)
+        if not self.is_mine(address):
+            return
         if txin.script_type in ('unknown', 'address'):
             return
         elif txin.script_type in ('p2pkh', 'p2wpkh', 'p2wpkh-p2sh'):
@@ -2014,15 +2014,17 @@ class Deterministic_Wallet(Abstract_Wallet):
                 for k in self.get_keystores()}
 
     def _add_input_sig_info(self, txin, address):
-        assert self.is_mine(address)
+        self._add_txinout_derivation_info(txin, address)
+
+    def _add_txinout_derivation_info(self, txinout, address):
+        if not self.is_mine(address):
+            return
         pubkey_deriv_info = self.get_public_keys_with_deriv_info(address)
-        txin.pubkeys = sorted([bfh(pk) for pk in list(pubkey_deriv_info)])
+        txinout.pubkeys = sorted([bfh(pk) for pk in list(pubkey_deriv_info)])
         for pubkey_hex in pubkey_deriv_info:
             ks, der_suffix = pubkey_deriv_info[pubkey_hex]
-            xfp_bytes = bfh(ks.get_root_fingerprint())
-            der_prefix = bip32.convert_bip32_path_to_list_of_uint32(ks.get_derivation_prefix())
-            der_full = der_prefix + list(der_suffix)
-            txin.bip32_paths[bfh(pubkey_hex)] = (xfp_bytes, der_full)
+            fp_bytes, der_full = ks.get_fp_and_derivation_to_be_used_in_partial_tx(der_suffix)
+            txinout.bip32_paths[bfh(pubkey_hex)] = (fp_bytes, der_full)
 
     def create_new_address(self, for_change=False):
         assert type(for_change) is bool