abstract database away from wallet and address_synchronizer

electrum/address_synchronizer.py

@@ -61,6 +61,7 @@ class AddressSynchronizer(PrintError):
 
     def __init__(self, storage: 'WalletStorage'):
         self.storage = storage
+        self.db = self.storage.db
         self.network = None  # type: Network
         # verifier (SPV) and synchronizer are started in start_network
         self.synchronizer = None  # type: Synchronizer
@@ -68,17 +69,6 @@ class AddressSynchronizer(PrintError):
         # locks: if you need to take multiple ones, acquire them in the order they are defined here!
         self.lock = threading.RLock()
         self.transaction_lock = threading.RLock()
-        # address -> list(txid, height)
-        self.history = storage.get('addr_history',{})
-        # Verified transactions.  txid -> TxMinedInfo.  Access with self.lock.
-        verified_tx = storage.get('verified_tx3', {})
-        self.verified_tx = {}  # type: Dict[str, TxMinedInfo]
-        for txid, (height, timestamp, txpos, header_hash) in verified_tx.items():
-            self.verified_tx[txid] = TxMinedInfo(height=height,
-                                                 conf=None,
-                                                 timestamp=timestamp,
-                                                 txpos=txpos,
-                                                 header_hash=header_hash)
         # Transactions pending verification.  txid -> tx_height. Access with self.lock.
         self.unverified_tx = defaultdict(int)
         # true when synchronized
@@ -95,17 +85,16 @@ class AddressSynchronizer(PrintError):
         return func_wrapper
 
     def load_and_cleanup(self):
-        self.load_transactions()
         self.load_local_history()
         self.check_history()
         self.load_unverified_transactions()
         self.remove_local_transactions_we_dont_have()
 
     def is_mine(self, address):
-        return address in self.history
+        return address in self.db.get_history()
 
     def get_addresses(self):
-        return sorted(self.history.keys())
+        return sorted(self.db.get_history())
 
     def get_address_history(self, addr):
         h = []
@@ -128,8 +117,8 @@ class AddressSynchronizer(PrintError):
             return addr
         prevout_hash = txi.get('prevout_hash')
         prevout_n = txi.get('prevout_n')
-        dd = self.txo.get(prevout_hash, {})
-        for addr, l in dd.items():
+        for addr in self.db.get_txo(prevout_hash):
+            l = self.db.get_txo_addr(prevout_hash, addr)
             for n, v, is_cb in l:
                 if n == prevout_n:
                     return addr
@@ -146,7 +135,8 @@ class AddressSynchronizer(PrintError):
 
     def load_unverified_transactions(self):
         # review transactions that are in the history
-        for addr, hist in self.history.items():
+        for addr in self.db.get_history():
+            hist = self.db.get_addr_history(addr)
             for tx_hash, tx_height in hist:
                 # add it in case it was previously unconfirmed
                 self.add_unverified_tx(tx_hash, tx_height)
@@ -167,13 +157,11 @@ class AddressSynchronizer(PrintError):
                 self.verifier = None
             self.storage.put('stored_height', self.get_local_height())
         if write_to_disk:
-            self.save_transactions()
-            self.save_verified_tx()
             self.storage.write()
 
     def add_address(self, address):
-        if address not in self.history:
-            self.history[address] = []
+        if address not in self.db.get_history():
+            self.db.history[address] = []
             self.set_up_to_date(False)
         if self.synchronizer:
             self.synchronizer.add(address)
@@ -191,11 +179,11 @@ class AddressSynchronizer(PrintError):
                     continue
                 prevout_hash = txin['prevout_hash']
                 prevout_n = txin['prevout_n']
-                spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n)
+                spending_tx_hash = self.db.get_spent_outpoint(prevout_hash, prevout_n)
                 if spending_tx_hash is None:
                     continue
                 # this outpoint has already been spent, by spending_tx
-                assert spending_tx_hash in self.transactions
+                assert spending_tx_hash in self.db.list_transactions()
                 conflicting_txns |= {spending_tx_hash}
             if tx_hash in conflicting_txns:
                 # this tx is already in history, so it conflicts with itself
@@ -256,49 +244,39 @@ class AddressSynchronizer(PrintError):
                     self.remove_transaction(tx_hash2)
             # add inputs
             def add_value_from_prev_output():
-                dd = self.txo.get(prevout_hash, {})
                 # note: this nested loop takes linear time in num is_mine outputs of prev_tx
-                for addr, outputs in dd.items():
+                for addr in self.db.get_txo(prevout_hash):
+                    outputs = self.db.get_txo_addr(prevout_hash, addr)
                     # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)}
                     for n, v, is_cb in outputs:
                         if n == prevout_n:
                             if addr and self.is_mine(addr):
-                                if d.get(addr) is None:
-                                    d[addr] = set()
-                                d[addr].add((ser, v))
+                                self.db.add_txi_addr(tx_hash, addr, ser, v)
                             return
-            self.txi[tx_hash] = d = {}
             for txi in tx.inputs():
                 if txi['type'] == 'coinbase':
                     continue
                 prevout_hash = txi['prevout_hash']
                 prevout_n = txi['prevout_n']
                 ser = prevout_hash + ':%d' % prevout_n
-                self.spent_outpoints[prevout_hash][prevout_n] = tx_hash
+                self.db.set_spent_outpoint(prevout_hash, prevout_n, tx_hash)
                 add_value_from_prev_output()
             # add outputs
-            self.txo[tx_hash] = d = {}
             for n, txo in enumerate(tx.outputs()):
                 v = txo[2]
                 ser = tx_hash + ':%d'%n
                 addr = self.get_txout_address(txo)
                 if addr and self.is_mine(addr):
-                    if d.get(addr) is None:
-                        d[addr] = []
-                    d[addr].append((n, v, is_coinbase))
+                    self.db.add_txo_addr(tx_hash, addr, n, v, is_coinbase)
                     # give v to txi that spends me
-                    next_tx = self.spent_outpoints[tx_hash].get(n)
+                    next_tx = self.db.get_spent_outpoint(tx_hash, n)
                     if next_tx is not None:
-                        dd = self.txi.get(next_tx, {})
-                        if dd.get(addr) is None:
-                            dd[addr] = set()
-                        if (ser, v) not in dd[addr]:
-                            dd[addr].add((ser, v))
+                        self.db.add_txi_addr(next_tx, addr, ser, v)
                         self._add_tx_to_local_history(next_tx)
             # add to local history
             self._add_tx_to_local_history(tx_hash)
             # save
-            self.transactions[tx_hash] = tx
+            self.db.add_transaction(tx_hash, tx)
             return True
 
     def remove_transaction(self, tx_hash):
@@ -328,17 +306,18 @@ class AddressSynchronizer(PrintError):
 
         with self.transaction_lock:
             self.print_error("removing tx from history", tx_hash)
-            tx = self.transactions.pop(tx_hash, None)
+            tx = self.db.remove_transaction(tx_hash)
             remove_from_spent_outpoints()
             self._remove_tx_from_local_history(tx_hash)
-            self.txi.pop(tx_hash, None)
-            self.txo.pop(tx_hash, None)
+            self.db.remove_txi(tx_hash)
+            self.db.remove_txo(tx_hash)
 
     def get_depending_transactions(self, tx_hash):
         """Returns all (grand-)children of tx_hash in this wallet."""
         with self.transaction_lock:
             children = set()
-            for other_hash in self.spent_outpoints[tx_hash].values():
+            for n in self.db.get_spent_outpoints(tx_hash):
+                other_hash = self.db.get_spent_outpoint(tx_hash, n)
                 children.add(other_hash)
                 children |= self.get_depending_transactions(other_hash)
             return children
@@ -354,129 +333,68 @@ class AddressSynchronizer(PrintError):
                 if (tx_hash, height) not in hist:
                     # make tx local
                     self.unverified_tx.pop(tx_hash, None)
-                    self.verified_tx.pop(tx_hash, None)
+                    self.db.remove_verified_tx(tx_hash)
                     if self.verifier:
                         self.verifier.remove_spv_proof_for_tx(tx_hash)
-            self.history[addr] = hist
+            self.db.set_addr_history(addr, hist)
 
         for tx_hash, tx_height in hist:
             # add it in case it was previously unconfirmed
             self.add_unverified_tx(tx_hash, tx_height)
             # if addr is new, we have to recompute txi and txo
-            tx = self.transactions.get(tx_hash)
+            tx = self.db.get_transaction(tx_hash)
             if tx is None:
                 continue
             self.add_transaction(tx_hash, tx, allow_unrelated=True)
 
         # Store fees
-        self.tx_fees.update(tx_fees)
+        self.db.update_tx_fees(tx_fees)
 
-    @profiler
-    def load_transactions(self):
-        # load txi, txo, tx_fees
-        # bookkeeping data of is_mine inputs of transactions
-        self.txi = self.storage.get('txi', {})  # txid -> address -> (prev_outpoint, value)
-        for txid, d in list(self.txi.items()):
-            for addr, lst in d.items():
-                self.txi[txid][addr] = set([tuple(x) for x in lst])
-        # bookkeeping data of is_mine outputs of transactions
-        self.txo = self.storage.get('txo', {})  # txid -> address -> (output_index, value, is_coinbase)
-        self.tx_fees = self.storage.get('tx_fees', {})
-        tx_list = self.storage.get('transactions', {})
-        # load transactions
-        self.transactions = {}
-        for tx_hash, raw in tx_list.items():
-            tx = Transaction(raw)
-            self.transactions[tx_hash] = tx
-            if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None:
-                self.print_error("removing unreferenced tx", tx_hash)
-                self.transactions.pop(tx_hash)
-        # load spent_outpoints
-        _spent_outpoints = self.storage.get('spent_outpoints', {})
-        self.spent_outpoints = defaultdict(dict)
-        for prevout_hash, d in _spent_outpoints.items():
-            for prevout_n_str, spending_txid in d.items():
-                prevout_n = int(prevout_n_str)
-                if spending_txid not in self.transactions:
-                    continue  # only care about txns we have
-                self.spent_outpoints[prevout_hash][prevout_n] = spending_txid
 
     @profiler
     def load_local_history(self):
         self._history_local = {}  # address -> set(txid)
         self._address_history_changed_events = defaultdict(asyncio.Event)  # address -> Event
-        for txid in itertools.chain(self.txi, self.txo):
+        for txid in itertools.chain(self.db.get_txi_keys(), self.db.get_txo_keys()):
             self._add_tx_to_local_history(txid)
 
     @profiler
     def check_history(self):
         save = False
-        hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.history.keys()))
-        hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.history.keys()))
+        hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.db.get_history()))
+        hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.db.get_history()))
         for addr in hist_addrs_not_mine:
-            self.history.pop(addr)
+            self.db.remove_addr_history(addr)
             save = True
         for addr in hist_addrs_mine:
-            hist = self.history[addr]
+            hist = self.db.get_addr_history(addr)
             for tx_hash, tx_height in hist:
-                if self.txi.get(tx_hash) or self.txo.get(tx_hash):
+                if self.db.get_txi(tx_hash) or self.db.get_txo(tx_hash):
                     continue
-                tx = self.transactions.get(tx_hash)
+                tx = self.db.get_transaction(tx_hash)
                 if tx is not None:
                     self.add_transaction(tx_hash, tx, allow_unrelated=True)
                     save = True
         if save:
-            self.save_transactions()
+            self.storage.write()
 
     def remove_local_transactions_we_dont_have(self):
-        txid_set = set(self.txi) | set(self.txo)
-        for txid in txid_set:
+        for txid in itertools.chain(list(self.db.get_txi_keys()), list(self.db.get_txo_keys())):
             tx_height = self.get_tx_height(txid).height
-            if tx_height == TX_HEIGHT_LOCAL and txid not in self.transactions:
+            if tx_height == TX_HEIGHT_LOCAL and txid not in self.db.list_transactions():
                 self.remove_transaction(txid)
 
-    @profiler
-    def save_transactions(self, write=False):
-        with self.transaction_lock:
-            tx = {}
-            for k,v in self.transactions.items():
-                tx[k] = str(v)
-            self.storage.put('transactions', tx)
-            self.storage.put('txi', self.txi)
-            self.storage.put('txo', self.txo)
-            self.storage.put('tx_fees', self.tx_fees)
-            self.storage.put('addr_history', self.history)
-            self.storage.put('spent_outpoints', self.spent_outpoints)
-            if write:
-                self.storage.write()
-
-    def save_verified_tx(self, write=False):
-        with self.lock:
-            verified_tx_to_save = {}
-            for txid, tx_info in self.verified_tx.items():
-                verified_tx_to_save[txid] = (tx_info.height, tx_info.timestamp,
-                                             tx_info.txpos, tx_info.header_hash)
-            self.storage.put('verified_tx3', verified_tx_to_save)
-            if write:
-                self.storage.write()
-
     def clear_history(self):
         with self.lock:
             with self.transaction_lock:
-                self.txi = {}
-                self.txo = {}
-                self.tx_fees = {}
-                self.spent_outpoints = defaultdict(dict)
-                self.history = {}
-                self.verified_tx = {}
-                self.transactions = {}  # type: Dict[str, Transaction]
-                self.save_transactions()
+                self.db.clear_history()
+                self.storage.write()
 
     def get_txpos(self, tx_hash):
         """Returns (height, txpos) tuple, even if the tx is unverified."""
         with self.lock:
-            if tx_hash in self.verified_tx:
-                info = self.verified_tx[tx_hash]
+            if tx_hash in self.db.list_verified_tx():
+                info = self.db.get_verified_tx(tx_hash)
                 return info.height, info.txpos
             elif tx_hash in self.unverified_tx:
                 height = self.unverified_tx[tx_hash]
@@ -540,7 +458,7 @@ class AddressSynchronizer(PrintError):
 
     def _add_tx_to_local_history(self, txid):
         with self.transaction_lock:
-            for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])):
+            for addr in itertools.chain(self.db.get_txi(txid), self.db.get_txo(txid)):
                 cur_hist = self._history_local.get(addr, set())
                 cur_hist.add(txid)
                 self._history_local[addr] = cur_hist
@@ -548,7 +466,7 @@ class AddressSynchronizer(PrintError):
 
     def _remove_tx_from_local_history(self, txid):
         with self.transaction_lock:
-            for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])):
+            for addr in itertools.chain(self.db.get_txi(txid), self.db.get_txo(txid)):
                 cur_hist = self._history_local.get(addr, set())
                 try:
                     cur_hist.remove(txid)
@@ -573,10 +491,10 @@ class AddressSynchronizer(PrintError):
         await self._address_history_changed_events[addr].wait()
 
     def add_unverified_tx(self, tx_hash, tx_height):
-        if tx_hash in self.verified_tx:
+        if tx_hash in self.db.list_verified_tx():
             if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT):
                 with self.lock:
-                    self.verified_tx.pop(tx_hash)
+                    self.db.remove_verified_tx(tx_hash)
                 if self.verifier:
                     self.verifier.remove_spv_proof_for_tx(tx_hash)
         else:
@@ -594,7 +512,7 @@ class AddressSynchronizer(PrintError):
         # Remove from the unverified map and add to the verified map
         with self.lock:
             self.unverified_tx.pop(tx_hash, None)
-            self.verified_tx[tx_hash] = info
+            self.db.add_verified_tx(tx_hash, info)
         tx_mined_status = self.get_tx_height(tx_hash)
         self.network.trigger_callback('verified', self, tx_hash, tx_mined_status)
 
@@ -607,12 +525,13 @@ class AddressSynchronizer(PrintError):
         '''Used by the verifier when a reorg has happened'''
         txs = set()
         with self.lock:
-            for tx_hash, info in list(self.verified_tx.items()):
+            for tx_hash in list(self.db.list_verified_tx()):
+                info = self.db.get_verified_tx(tx_hash)
                 tx_height = info.height
                 if tx_height >= height:
                     header = blockchain.read_header(tx_height)
                     if not header or hash_header(header) != info.header_hash:
-                        self.verified_tx.pop(tx_hash, None)
+                        self.db.remove_verified_tx(tx_hash)
                         # NOTE: we should add these txns to self.unverified_tx,
                         # but with what height?
                         # If on the new fork after the reorg, the txn is at the
@@ -635,8 +554,8 @@ class AddressSynchronizer(PrintError):
 
     def get_tx_height(self, tx_hash: str) -> TxMinedInfo:
         with self.lock:
-            if tx_hash in self.verified_tx:
-                info = self.verified_tx[tx_hash]
+            if tx_hash in self.db.list_verified_tx():
+                info = self.db.get_verified_tx(tx_hash)
                 conf = max(self.get_local_height() - info.height + 1, 0)
                 return info._replace(conf=conf)
             elif tx_hash in self.unverified_tx:
@@ -652,11 +571,7 @@ class AddressSynchronizer(PrintError):
         if self.network:
             self.network.notify('status')
         if up_to_date:
-            self.save_transactions(write=True)
-            # if the verifier is also up to date, persist that too;
-            # otherwise it will persist its results when it finishes
-            if self.verifier and self.verifier.is_up_to_date():
-                self.save_verified_tx(write=True)
+            self.storage.write()
 
     def is_up_to_date(self):
         with self.lock: return self.up_to_date
@@ -666,11 +581,11 @@ class AddressSynchronizer(PrintError):
         """effect of tx on address"""
         delta = 0
         # substract the value of coins sent from address
-        d = self.txi.get(tx_hash, {}).get(address, [])
+        d = self.db.get_txi_addr(tx_hash, address)
         for n, v in d:
             delta -= v
         # add the value of the coins received at address
-        d = self.txo.get(tx_hash, {}).get(address, [])
+        d = self.db.get_txo_addr(tx_hash, address)
         for n, v, cb in d:
             delta += v
         return delta
@@ -679,10 +594,12 @@ class AddressSynchronizer(PrintError):
     def get_tx_value(self, txid):
         """effect of tx on the entire domain"""
         delta = 0
-        for addr, d in self.txi.get(txid, {}).items():
+        for addr in self.db.get_txi(txid):
+            d = self.db.get_txi_addr(txid, addr)
             for n, v in d:
                 delta -= v
-        for addr, d in self.txo.get(txid, {}).items():
+        for addr in self.db.get_txo(txid):
+            d = self.db.get_txo_addr(txid, addr)
             for n, v, cb in d:
                 delta += v
         return delta
@@ -699,7 +616,7 @@ class AddressSynchronizer(PrintError):
             if self.is_mine(addr):
                 is_mine = True
                 is_relevant = True
-                d = self.txo.get(txin['prevout_hash'], {}).get(addr, [])
+                d = self.db.get_txo_addr(txin['prevout_hash'], addr)
                 for n, v, cb in d:
                     if n == txin['prevout_n']:
                         value = v
@@ -748,7 +665,7 @@ class AddressSynchronizer(PrintError):
             is_relevant, is_mine, v, fee = self.get_wallet_delta(tx)
             if fee is None:
                 txid = tx.txid()
-                fee = self.tx_fees.get(txid)
+                fee = self.db.get_tx_fee(txid)
             # only cache non-None, as None can still change while syncing
             if fee is not None:
                 tx._cached_fee = fee
@@ -760,11 +677,11 @@ class AddressSynchronizer(PrintError):
             received = {}
             sent = {}
             for tx_hash, height in h:
-                l = self.txo.get(tx_hash, {}).get(address, [])
+                l = self.db.get_txo_addr(tx_hash, address)
                 for n, v, is_cb in l:
                     received[tx_hash + ':%d'%n] = (height, v, is_cb)
             for tx_hash, height in h:
-                l = self.txi.get(tx_hash, {}).get(address, [])
+                l = self.db.get_txi_addr(tx_hash, address)
                 for txi, v in l:
                     sent[txi] = height
         return received, sent
@@ -849,7 +766,7 @@ class AddressSynchronizer(PrintError):
         return cc, uu, xx
 
     def is_used(self, address):
-        h = self.history.get(address,[])
+        h = self.db.get_addr_history(address)
         return len(h) != 0
 
     def is_empty(self, address):

electrum/json_db.py

@@ -35,10 +35,10 @@ import zlib
 from collections import defaultdict
 
 from . import util, bitcoin, ecc
-from .util import PrintError, profiler, InvalidPassword, WalletFileException, bfh, standardize_path, multisig_type
+from .util import PrintError, profiler, InvalidPassword, WalletFileException, bfh, standardize_path, multisig_type, TxMinedInfo
 from .plugin import run_hook, plugin_loaders
 from .keystore import bip44_derivation
-
+from .transaction import Transaction
 
 # seed_version is now used for the version of the wallet file
 
@@ -49,9 +49,6 @@ FINAL_SEED_VERSION = 18     # electrum >= 2.7 will set this to prevent
 
 
 
-
-
-
 class JsonDB(PrintError):
 
     def __init__(self, raw, manual_upgrades):
@@ -61,6 +58,7 @@ class JsonDB(PrintError):
             self.load_data(raw)
         else:
             self.put('seed_version', FINAL_SEED_VERSION)
+        self.load_transactions()
 
     def get(self, key, default=None):
         v = self.data.get(key)
@@ -498,3 +496,151 @@ class JsonDB(PrintError):
                 msg += "\nPlease open this file with Electrum 1.9.8, and move your coins to a new wallet."
         raise WalletFileException(msg)
 
+
+    def get_txi(self, tx_hash):
+        return self.txi.get(tx_hash, {}).keys()
+
+    def get_txo(self, tx_hash):
+        return self.txo.get(tx_hash, {}).keys()
+
+    def get_txi_addr(self, tx_hash, address):
+        return self.txi.get(tx_hash, {}).get(address, [])
+
+    def get_txo_addr(self, tx_hash, address):
+        return self.txo.get(tx_hash, {}).get(address, [])
+
+    def add_txi_addr(self, tx_hash, addr, ser, v):
+        if tx_hash not in self.txi:
+            self.txi[tx_hash] = {}
+        d = self.txi[tx_hash]
+        if addr not in d:
+            d[addr] = set()
+        d[addr].add((ser, v))
+
+    def add_txo_addr(self, tx_hash, addr, n, v, is_coinbase):
+        if tx_hash not in self.txo:
+            self.txo[tx_hash] = {}
+        d = self.txo[tx_hash]
+        if addr not in d:
+            d[addr] = []
+        d[addr].append((n, v, is_coinbase))
+
+    def get_txi_keys(self):
+        return self.txi.keys()
+
+    def get_txo_keys(self):
+        return self.txo.keys()
+
+    def remove_txi(self, tx_hash):
+        self.txi.pop(tx_hash, None)
+
+    def remove_txo(self, tx_hash):
+        self.txo.pop(tx_hash, None)
+
+    def get_spent_outpoints(self, prevout_hash):
+        return self.spent_outpoints.get(prevout_hash, {}).keys()
+
+    def get_spent_outpoint(self, prevout_hash, prevout_n):
+        return self.spent_outpoints.get(prevout_hash, {}).get(str(prevout_n))
+
+    def set_spent_outpoint(self, prevout_hash, prevout_n, tx_hash):
+        if prevout_hash not in self.spent_outpoints:
+            self.spent_outpoints[prevout_hash] = {}
+        self.spent_outpoints[prevout_hash][str(prevout_n)] = tx_hash
+
+    def add_transaction(self, tx_hash, tx):
+        self.transactions[tx_hash] = str(tx)
+
+    def remove_transaction(self, tx_hash):
+        self.transactions.pop(tx_hash, None)
+
+    def get_transaction(self, tx_hash):
+        tx = self.transactions.get(tx_hash)
+        return Transaction(tx) if tx else None
+
+    def list_transactions(self):
+        return self.transactions.keys()
+
+    def get_history(self):
+        return self.history.keys()
+
+    def get_addr_history(self, addr):
+        return self.history.get(addr, [])
+
+    def set_addr_history(self, addr, hist):
+        self.history[addr] = hist
+
+    def remove_addr_history(self, addr):
+        self.history.pop(addr, None)
+
+    def list_verified_tx(self):
+        return self.verified_tx.keys()
+
+    def get_verified_tx(self, txid):
+        if txid not in self.verified_tx:
+            return None
+        height, timestamp, txpos, header_hash = self.verified_tx[txid]
+        return TxMinedInfo(height=height,
+                           conf=None,
+                           timestamp=timestamp,
+                           txpos=txpos,
+                           header_hash=header_hash)
+
+    def add_verified_tx(self, txid, info):
+        self.verified_tx[txid] = (info.height, info.timestamp, info.txpos, info.header_hash)
+
+    def remove_verified_tx(self, txid):
+        self.verified_tx.pop(txid, None)
+
+    def update_tx_fees(self, d):
+        return self.tx_fees.update(d)
+
+    def get_tx_fee(self, txid):
+        return self.tx_fees.get(txid)
+
+    def remove_tx_fee(self, txid):
+        self.tx_fees.pop(txid, None)
+
+    def get_data_ref(self, name):
+        if name not in self.data:
+            self.data[name] = {}
+        return self.data[name]
+
+    @profiler
+    def load_transactions(self):
+        # references in self.data
+        self.txi = self.get_data_ref('txi')  # txid -> address -> (prev_outpoint, value)
+        self.txo = self.get_data_ref('txo')  # txid -> address -> (output_index, value, is_coinbase)
+        self.transactions = self.get_data_ref('transactions')   # type: Dict[str, Transaction]
+        self.spent_outpoints = self.get_data_ref('spent_outpoints')
+        self.history = self.get_data_ref('history')  # address -> list(txid, height)
+        self.verified_tx = self.get_data_ref('verified_tx3') # txid -> TxMinedInfo.  Access with self.lock.
+        self.tx_fees = self.get_data_ref('tx_fees')
+
+        # tuple to set
+        for txid, d in list(self.txi.items()):
+            for addr, lst in d.items():
+                self.txi[txid][addr] = set([tuple(x) for x in lst])
+
+        # remove unreferenced tx
+        for tx_hash in self.transactions:
+            if not self.get_txi(tx_hash) and not self.get_txo(tx_hash):
+                self.print_error("removing unreferenced tx", tx_hash)
+                self.transactions.pop(tx_hash)
+
+        # remove unreferenced outpoints
+        for prevout_hash in self.spent_outpoints.keys():
+            d = self.spent_outpoints[prevout_hash]
+            for prevout_n, spending_txid in list(d.items()):
+                if spending_txid not in self.transactions:
+                    self.print_error("removing unreferenced spent outpoint")
+                    d.pop(prevout_n)
+
+    def clear_history(self):
+        self.txi.clear()
+        self.txo.clear()
+        self.spent_outpoints.clear()
+        self.transactions.clear()
+        self.history.clear()
+        self.verified_tx.clear()
+        self.tx_fees.clear()

electrum/network.py

@@ -1184,8 +1184,6 @@ class Network(PrintError):
         coro = asyncio.run_coroutine_threadsafe(cls._send_http_on_proxy(method, url, **kwargs), loop)
         return coro.result(5)
 
-
-
     # methods used in scripts
     async def get_peers(self):
         while not self.is_connected():

electrum/synchronizer.py

@@ -139,7 +139,7 @@ class Synchronizer(SynchronizerBase):
                 and not self.requested_tx)
 
     async def _on_address_status(self, addr, status):
-        history = self.wallet.history.get(addr, [])
+        history = self.wallet.db.get_addr_history(addr)
         if history_status(history) == status:
             return
         if addr in self.requested_histories:
@@ -175,7 +175,7 @@ class Synchronizer(SynchronizerBase):
         for tx_hash, tx_height in hist:
             if tx_hash in self.requested_tx:
                 continue
-            if tx_hash in self.wallet.transactions:
+            if tx_hash in self.wallet.db.list_transactions():
                 continue
             transaction_hashes.append(tx_hash)
             self.requested_tx[tx_hash] = tx_height
@@ -216,7 +216,8 @@ class Synchronizer(SynchronizerBase):
     async def main(self):
         self.wallet.set_up_to_date(False)
         # request missing txns, if any
-        for history in self.wallet.history.values():
+        for addr in self.wallet.db.get_history():
+            history = self.wallet.db.get_addr_history(addr)
             # Old electrum servers returned ['*'] when all history for the address
             # was pruned. This no longer happens but may remain in old wallets.
             if history == ['*']: continue

electrum/verifier.py

@@ -135,8 +135,8 @@ class SPV(NetworkJobOnDefaultServer):
                               txpos=pos,
                               header_hash=header_hash)
         self.wallet.add_verified_tx(tx_hash, tx_info)
-        if self.is_up_to_date() and self.wallet.is_up_to_date():
-            self.wallet.save_verified_tx(write=True)
+        #if self.is_up_to_date() and self.wallet.is_up_to_date():
+        #    self.wallet.save_verified_tx(write=True)
 
     @classmethod
     def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int):

electrum/wallet.py

@@ -276,7 +276,7 @@ class Abstract_Wallet(AddressSynchronizer):
         return changed
 
     def set_fiat_value(self, txid, ccy, text, fx, value_sat):
-        if txid not in self.transactions:
+        if txid not in self.db.list_transactions():
             return
         # since fx is inserting the thousands separator,
         # and not util, also have fx remove it
@@ -346,7 +346,8 @@ class Abstract_Wallet(AddressSynchronizer):
         return [self.get_public_key(address)]
 
     def is_found(self):
-        return self.history.values() != [[]] * len(self.history)
+        return True
+        #return self.history.values() != [[]] * len(self.history)
 
     def get_tx_info(self, tx):
         is_relevant, is_mine, v, fee = self.get_wallet_delta(tx)
@@ -357,7 +358,7 @@ class Abstract_Wallet(AddressSynchronizer):
         height = conf = timestamp = None
         tx_hash = tx.txid()
         if tx.is_complete():
-            if tx_hash in self.transactions.keys():
+            if tx_hash in self.db.list_transactions():
                 label = self.get_label(tx_hash)
                 tx_mined_status = self.get_tx_height(tx_hash)
                 height, conf = tx_mined_status.height, tx_mined_status.conf
@@ -369,7 +370,7 @@ class Abstract_Wallet(AddressSynchronizer):
                 elif height in (TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED):
                     status = _('Unconfirmed')
                     if fee is None:
-                        fee = self.tx_fees.get(tx_hash)
+                        fee = self.db.get_tx_fee(tx_hash)
                     if fee and self.network and self.network.config.has_fee_mempool():
                         size = tx.estimated_size()
                         fee_per_byte = fee / size
@@ -447,7 +448,7 @@ class Abstract_Wallet(AddressSynchronizer):
                 continue
             if to_height is not None and height >= to_height:
                 continue
-            tx = self.transactions.get(tx_hash)
+            tx = self.db.get_transaction(tx_hash)
             item = {
                 'txid': tx_hash,
                 'height': height,
@@ -554,10 +555,9 @@ class Abstract_Wallet(AddressSynchronizer):
         return label
 
     def get_default_label(self, tx_hash):
-        if self.txi.get(tx_hash) == {}:
-            d = self.txo.get(tx_hash, {})
+        if not self.db.get_txi(tx_hash):
             labels = []
-            for addr in d.keys():
+            for addr in self.db.get_txo(tx_hash):
                 label = self.labels.get(addr)
                 if label:
                     labels.append(label)
@@ -570,7 +570,7 @@ class Abstract_Wallet(AddressSynchronizer):
         conf = tx_mined_info.conf
         timestamp = tx_mined_info.timestamp
         if conf == 0:
-            tx = self.transactions.get(tx_hash)
+            tx = self.db.get_transaction(tx_hash)
             if not tx:
                 return 2, 'unknown'
             is_final = tx and tx.is_final()
@@ -578,7 +578,7 @@ class Abstract_Wallet(AddressSynchronizer):
                 extra.append('rbf')
             fee = self.get_wallet_delta(tx)[3]
             if fee is None:
-                fee = self.tx_fees.get(tx_hash)
+                fee = self.db.get_tx_fee(tx_hash)
             if fee is not None:
                 size = tx.estimated_size()
                 fee_per_byte = fee / size
@@ -616,9 +616,11 @@ class Abstract_Wallet(AddressSynchronizer):
             # tx should not be mined yet
             if tx_mined_status.conf > 0: continue
             # tx should be "outgoing" from wallet
-            if delta >= 0: continue
-            tx = self.transactions.get(tx_hash)
-            if not tx: continue
+            if delta >= 0:
+                continue
+            tx = self.db.get_transaction(tx_hash)
+            if not tx:
+                continue
             # is_mine outputs should not be spent yet
             # to avoid cancelling our own dependent transactions
             txid = tx.txid()
@@ -787,7 +789,7 @@ class Abstract_Wallet(AddressSynchronizer):
 
     def address_is_old(self, address, age_limit=2):
         age = -1
-        h = self.history.get(address, [])
+        h = self.db.get_addr_history(address)
         for tx_hash, tx_height in h:
             if tx_height <= 0:
                 tx_age = 0
@@ -884,7 +886,7 @@ class Abstract_Wallet(AddressSynchronizer):
         # First look up an input transaction in the wallet where it
         # will likely be.  If co-signing a transaction it may not have
         # all the input txs, in which case we ask the network.
-        tx = self.transactions.get(tx_hash, None)
+        tx = self.db.get_transaction(tx_hash)
         if not tx and self.network:
             try:
                 raw_tx = self.network.run_from_another_thread(
@@ -950,7 +952,7 @@ class Abstract_Wallet(AddressSynchronizer):
     def get_unused_addresses(self):
         # fixme: use slots from expired requests
         domain = self.get_receiving_addresses()
-        return [addr for addr in domain if not self.history.get(addr)
+        return [addr for addr in domain if not self.db.get_addr_history(addr)
                 and addr not in self.receive_requests.keys()]
 
     @check_returned_address
@@ -967,7 +969,7 @@ class Abstract_Wallet(AddressSynchronizer):
             return
         choice = domain[0]
         for addr in domain:
-            if not self.history.get(addr):
+            if not self.db.get_addr_history(addr):
                 if addr not in self.receive_requests.keys():
                     return addr
                 else:
@@ -981,7 +983,7 @@ class Abstract_Wallet(AddressSynchronizer):
         for txo, x in received.items():
             h, v, is_cb = x
             txid, n = txo.split(':')
-            info = self.verified_tx.get(txid)
+            info = self.db.get_verified_tx(txid)
             if info:
                 conf = local_height - info.height
             else:
@@ -1214,7 +1216,8 @@ class Abstract_Wallet(AddressSynchronizer):
     def txin_value(self, txin):
         txid = txin['prevout_hash']
         prev_n = txin['prevout_n']
-        for address, d in self.txo.get(txid, {}).items():
+        for addr in self.db.get_txo(txid):
+            d = self.db.get_txo_addr(txid, addr)
             for n, v, cb in d:
                 if n == prev_n:
                     return v
@@ -1238,7 +1241,8 @@ class Abstract_Wallet(AddressSynchronizer):
         """ Average acquisition price of the inputs of a transaction """
         input_value = 0
         total_price = 0
-        for addr, d in self.txi.get(txid, {}).items():
+        for addr in self.db.get_txi(txid):
+            d = self.db.get_txi_addr(txid, addr)
             for ser, v in d:
                 input_value += v
                 total_price += self.coin_price(ser.split(':')[0], price_func, ccy, v)
@@ -1258,7 +1262,7 @@ class Abstract_Wallet(AddressSynchronizer):
         result = self._coin_price_cache.get(cache_key, None)
         if result is not None:
             return result
-        if self.txi.get(txid, {}) != {}:
+        if self.db.get_txi(txid):
             result = self.average_price(txid, price_func, ccy) * txin_value/Decimal(COIN)
             self._coin_price_cache[cache_key] = result
             return result
@@ -1398,7 +1402,8 @@ class Imported_Wallet(Simple_Wallet):
         transactions_to_remove = set()  # only referred to by this address
         transactions_new = set()  # txs that are not only referred to by address
         with self.lock:
-            for addr, details in self.history.items():
+            for addr in self.db.get_history():
+                details = self.db.get_addr_history(addr)
                 if addr == address:
                     for tx_hash, height in details:
                         transactions_to_remove.add(tx_hash)
@@ -1406,14 +1411,14 @@ class Imported_Wallet(Simple_Wallet):
                     for tx_hash, height in details:
                         transactions_new.add(tx_hash)
             transactions_to_remove -= transactions_new
-            self.history.pop(address, None)
+            self.db.remove_history(address)
 
             for tx_hash in transactions_to_remove:
                 self.remove_transaction(tx_hash)
-                self.tx_fees.pop(tx_hash, None)
-                self.verified_tx.pop(tx_hash, None)
+                self.db.remove_tx_fee(tx_hash)
+                self.db.remove_verified_tx(tx_hash)
                 self.unverified_tx.pop(tx_hash, None)
-                self.transactions.pop(tx_hash, None)
+                self.db.remove_transaction(tx_hash)
             self.save_verified_tx()
         self.save_transactions()
 
@@ -1465,7 +1470,8 @@ class Imported_Wallet(Simple_Wallet):
             self.add_address(addr)
         self.save_keystore()
         self.save_addresses()
-        self.save_transactions(write=write_to_disk)
+        if write_to_disk:
+            self.storage.write()
         return good_addr, bad_keys
 
     def import_private_key(self, key: str, password: Optional[str]) -> str:
@@ -1571,7 +1577,8 @@ class Deterministic_Wallet(Abstract_Wallet):
     def num_unused_trailing_addresses(self, addresses):
         k = 0
         for a in addresses[::-1]:
-            if self.history.get(a):break
+            if a in self.db.get_history():
+                break
             k = k + 1
         return k
 
@@ -1582,7 +1589,7 @@ class Deterministic_Wallet(Abstract_Wallet):
         addresses = self.get_receiving_addresses()
         k = self.num_unused_trailing_addresses(addresses)
         for a in addresses[0:-k]:
-            if self.history.get(a):
+            if a in self.db.get_history():
                 n = 0
             else:
                 n += 1
@@ -1641,7 +1648,7 @@ class Deterministic_Wallet(Abstract_Wallet):
             return False
         prev_addresses = addr_list[max(0, i - limit):max(0, i)]
         for addr in prev_addresses:
-            if self.history.get(addr):
+            if addr in self.db.get_history():
                 return False
         return True