small fixups

electrum/address_synchronizer.py

@@ -288,7 +288,7 @@ class AddressSynchronizer(PrintError):
                         continue
                     prevout_hash = txin['prevout_hash']
                     prevout_n = txin['prevout_n']
-                    self.spent_outpoints[prevout_hash].pop(prevout_n, None)
+                    self.spent_outpoints[prevout_hash].pop(prevout_n, None)  # FIXME
                     if not self.spent_outpoints[prevout_hash]:
                         self.spent_outpoints.pop(prevout_hash)
             else:  # expensive but always works
@@ -388,6 +388,7 @@ class AddressSynchronizer(PrintError):
         with self.lock:
             with self.transaction_lock:
                 self.db.clear_history()
+                self.storage.modified = True  # FIXME hack..
                 self.storage.write()
 
     def get_txpos(self, tx_hash):

electrum/base_wizard.py

@@ -242,7 +242,7 @@ class BaseWizard(object):
         k = keystore.from_master_key(text)
         self.on_keystore(k)
 
-    def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, storage=None):
+    def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, *, storage=None):
         title = _('Hardware Keystore')
         # check available plugins
         supported_plugins = self.plugins.get_hardware_support()
@@ -311,9 +311,10 @@ class BaseWizard(object):
             descr = f"{label} [{name}, {state}, {transport_str}]"
             choices.append(((name, info), descr))
         msg = _('Select a device') + ':'
-        self.choice_dialog(title=title, message=msg, choices=choices, run_next= lambda *args: self.on_device(*args, purpose=purpose))
+        self.choice_dialog(title=title, message=msg, choices=choices,
+                           run_next=lambda *args: self.on_device(*args, purpose=purpose, storage=storage))
 
-    def on_device(self, name, device_info, *, purpose):
+    def on_device(self, name, device_info, *, purpose, storage=None):
         self.plugin = self.plugins.get_plugin(name)
         try:
             self.plugin.setup_device(device_info, self, purpose)
@@ -550,7 +551,7 @@ class BaseWizard(object):
                 keys = self.keystores[0].dump()
                 self.data['keystore'] = keys
         else:
-            raise BaseException('Unknown wallet type')
+            raise Exception('Unknown wallet type')
         self.pw_args = password, encrypt_storage, storage_enc_version
         self.terminate()
 
@@ -559,11 +560,11 @@ class BaseWizard(object):
             return
         password, encrypt_storage, storage_enc_version = self.pw_args
         storage = WalletStorage(path)
-        for key, value in self.data.items():
-            storage.put(key, value)
-        storage.set_keystore_encryption(bool(password))# and encrypt_keystore)
+        storage.set_keystore_encryption(bool(password))  # and encrypt_keystore)
         if encrypt_storage:
             storage.set_password(password, enc_version=storage_enc_version)
+        for key, value in self.data.items():
+            storage.put(key, value)
         storage.write()
         return storage
 

electrum/gui/qt/__init__.py

@@ -229,6 +229,7 @@ class ElectrumGui(PrintError):
                 return
         if not wallet:
             wizard = InstallWizard(self.config, self.app, self.plugins)
+            storage = None
             try:
                 path, storage = wizard.select_storage(path, self.daemon.get_wallet)
                 # storage is None if file does not exist

electrum/gui/qt/installwizard.py

@@ -289,7 +289,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
                 else:
                     raise Exception('Unexpected encryption version')
 
-        return self.temp_storage.path, self.temp_storage if self.temp_storage.file_exists() else None
+        return self.temp_storage.path, (self.temp_storage if self.temp_storage.file_exists() else None)
 
     def run_upgrades(self, storage):
         path = storage.path

electrum/json_db.py

@@ -24,19 +24,14 @@
 # SOFTWARE.
 import os
 import ast
-import threading
 import json
 import copy
-import re
-import stat
-import hashlib
-import base64
-import zlib
 from collections import defaultdict
+from typing import Dict
 
-from . import util, bitcoin, ecc
-from .util import PrintError, profiler, InvalidPassword, WalletFileException, bfh, standardize_path, multisig_type, TxMinedInfo
-from .plugin import run_hook, plugin_loaders
+from . import util, bitcoin
+from .util import PrintError, profiler, WalletFileException, multisig_type, TxMinedInfo
+from .plugin import plugin_loaders
 from .keystore import bip44_derivation
 from .transaction import Transaction
 
@@ -48,10 +43,9 @@ FINAL_SEED_VERSION = 18     # electrum >= 2.7 will set this to prevent
                             # old versions from overwriting new format
 
 
-
 class JsonDB(PrintError):
 
-    def __init__(self, raw, manual_upgrades):
+    def __init__(self, raw, *, manual_upgrades):
         self.data = {}
         self.manual_upgrades = manual_upgrades
         if raw:
@@ -84,19 +78,6 @@ class JsonDB(PrintError):
             return True
         return False
 
-    def get_all_data(self) -> dict:
-        return copy.deepcopy(self.data)
-
-    def overwrite_all_data(self, data: dict) -> None:
-        try:
-            json.dumps(data, cls=util.MyEncoder)
-        except:
-            self.print_error(f"json error: cannot save {repr(data)}")
-            return
-        with self.db_lock:
-            self.modified = True
-            self.data = copy.deepcopy(data)
-
     def commit(self):
         pass
 
@@ -111,7 +92,7 @@ class JsonDB(PrintError):
                 d = ast.literal_eval(s)
                 labels = d.get('labels', {})
             except Exception as e:
-                raise IOError("Cannot read wallet file '%s'" % self.path)
+                raise IOError("Cannot read wallet file")
             self.data = {}
             for key, value in d.items():
                 try:
@@ -482,7 +463,6 @@ class JsonDB(PrintError):
 
     def raise_unsupported_version(self, seed_version):
         msg = "Your wallet has an unsupported seed version."
-        msg += '\n\nWallet file: %s' % os.path.abspath(self.path)
         if seed_version in [5, 7, 8, 9, 10, 14]:
             msg += "\n\nTo open this wallet, try 'git checkout seed_v%d'"%seed_version
         if seed_version == 6:
@@ -514,6 +494,7 @@ class JsonDB(PrintError):
             self.txi[tx_hash] = {}
         d = self.txi[tx_hash]
         if addr not in d:
+            # note that as this is a set, we can ignore "duplicates"
             d[addr] = set()
         d[addr].add((ser, v))
 
@@ -522,8 +503,9 @@ class JsonDB(PrintError):
             self.txo[tx_hash] = {}
         d = self.txo[tx_hash]
         if addr not in d:
-            d[addr] = []
-        d[addr].append((n, v, is_coinbase))
+            # note that as this is a set, we can ignore "duplicates"
+            d[addr] = set()
+        d[addr].add((n, v, is_coinbase))
 
     def get_txi_keys(self):
         return self.txi.keys()
@@ -613,7 +595,7 @@ class JsonDB(PrintError):
         self.txo = self.get_data_ref('txo')  # txid -> address -> (output_index, value, is_coinbase)
         self.transactions = self.get_data_ref('transactions')   # type: Dict[str, Transaction]
         self.spent_outpoints = self.get_data_ref('spent_outpoints')
-        self.history = self.get_data_ref('history')  # address -> list(txid, height)
+        self.history = self.get_data_ref('addr_history')  # address -> list(txid, height)
         self.verified_tx = self.get_data_ref('verified_tx3') # txid -> TxMinedInfo.  Access with self.lock.
         self.tx_fees = self.get_data_ref('tx_fees')
 

electrum/storage.py

@@ -24,18 +24,14 @@
 # SOFTWARE.
 import os
 import threading
-import copy
-import re
 import stat
 import hashlib
 import base64
 import zlib
-from collections import defaultdict
 
-from . import util, bitcoin, ecc
+from . import ecc
 from .util import PrintError, profiler, InvalidPassword, WalletFileException, bfh, standardize_path
-from .plugin import run_hook, plugin_loaders
-from .keystore import bip44_derivation
+from .plugin import run_hook
 
 from .json_db import JsonDB
 
@@ -52,7 +48,7 @@ STO_EV_PLAINTEXT, STO_EV_USER_PW, STO_EV_XPUB_PW = range(0, 3)
 
 class WalletStorage(PrintError):
 
-    def __init__(self, path, manual_upgrades=False):
+    def __init__(self, path, *, manual_upgrades=False):
         self.db_lock = threading.RLock()
         self.path = standardize_path(path)
         self._file_exists = self.path and os.path.exists(self.path)
@@ -67,11 +63,11 @@ class WalletStorage(PrintError):
                 self.raw = f.read()
             self._encryption_version = self._init_encryption_version()
             if not self.is_encrypted():
-                self.db = DB_Class(self.raw, manual_upgrades)
+                self.db = DB_Class(self.raw, manual_upgrades=manual_upgrades)
         else:
             self._encryption_version = STO_EV_PLAINTEXT
             # avoid new wallets getting 'upgraded'
-            self.db = DB_Class('', False)
+            self.db = DB_Class('', manual_upgrades=False)
 
     def put(self, key,value):
         with self.db_lock:
@@ -109,9 +105,6 @@ class WalletStorage(PrintError):
         self.print_error("saved", self.path)
         self.modified = False
 
-    def encrypt_before_writing(self, plaintext: str) -> str:
-        return plaintext
-
     def file_exists(self):
         return self._file_exists
 
@@ -181,7 +174,7 @@ class WalletStorage(PrintError):
             s = None
         self.pubkey = ec_key.get_public_key_hex()
         s = s.decode('utf8')
-        self.db = JsonDB(s, True)
+        self.db = JsonDB(s, manual_upgrades=True)
 
     def encrypt_before_writing(self, plaintext: str) -> str:
         s = plaintext