improve wallet loading performance

3 years ago · 9faf036e4d
20 changed files with 129 additions and 57 deletions
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 8dca2ee3f0ba8dbebf88c3629b2a52c7eecf5b89
+Subproject commit fe61c633ae230c3425d52a0c9ac6264ea77e5041
--- a/src/main/java/com/sparrowwallet/sparrow/AppController.java
+++ b/src/main/java/com/sparrowwallet/sparrow/AppController.java
@ -880,8 +880,21 @@ public class AppController implements Initializable {
        try {
            Storage storage = new Storage(file);
            if(!storage.isEncrypted()) {
-                WalletBackupAndKey walletBackupAndKey = storage.loadUnencryptedWallet();
-                openWallet(storage, walletBackupAndKey, this, forceSameWindow);
+                Storage.LoadWalletService loadWalletService = new Storage.LoadWalletService(storage);
+                loadWalletService.setOnSucceeded(workerStateEvent -> {
+                    WalletBackupAndKey walletBackupAndKey = loadWalletService.getValue();
+                    openWallet(storage, walletBackupAndKey, this, forceSameWindow);
+                });
+                loadWalletService.setOnFailed(workerStateEvent -> {
+                    Throwable exception = workerStateEvent.getSource().getException();
+                    if(exception instanceof StorageException) {
+                        showErrorDialog("Error Opening Wallet", exception.getMessage());
+                    } else if(!attemptImportWallet(file, null)) {
+                        log.error("Error opening wallet", exception);
+                        showErrorDialog("Error Opening Wallet", exception.getMessage() == null || exception.getMessage().contains("Expected BEGIN_OBJECT") ? "Unsupported wallet file format." : exception.getMessage());
+                    }
+                });
+                loadWalletService.start();
            } else {
                WalletPasswordDialog dlg = new WalletPasswordDialog(storage.getWalletName(null), WalletPasswordDialog.PasswordRequirement.LOAD);
                Optional<SecureString> optionalPassword = dlg.showAndWait();
@ -905,9 +918,11 @@ public class AppController implements Initializable {
                            Platform.runLater(() -> openWalletFile(file, forceSameWindow));
                        }
                    } else {
-                        if(!attemptImportWallet(file, password)) {
+                        if(exception instanceof StorageException) {
+                            showErrorDialog("Error Opening Wallet", exception.getMessage());
+                        } else if(!attemptImportWallet(file, password)) {
                            log.error("Error Opening Wallet", exception);
-                            showErrorDialog("Error Opening Wallet", exception.getMessage() == null ? "Unsupported file format" : exception.getMessage());
+                            showErrorDialog("Error Opening Wallet", exception.getMessage() == null || exception.getMessage().contains("Expected BEGIN_OBJECT") ? "Unsupported wallet file format." : exception.getMessage());
                        }
                        password.clear();
                    }
@ -915,8 +930,6 @@ public class AppController implements Initializable {
                EventManager.get().post(new StorageEvent(storage.getWalletId(null), TimedEvent.Action.START, "Decrypting wallet..."));
                loadWalletService.start();
            }
-        } catch(StorageException e) {
-            showErrorDialog("Error Opening Wallet", e.getMessage());
        } catch(Exception e) {
            if(e instanceof IOException && e.getMessage().startsWith("The process cannot access the file because another process has locked")) {
                log.error("Error opening wallet", e);
--- a/src/main/java/com/sparrowwallet/sparrow/control/BalanceChart.java
+++ b/src/main/java/com/sparrowwallet/sparrow/control/BalanceChart.java
@ -1,20 +1,26 @@
 package com.sparrowwallet.sparrow.control;

+import com.google.common.collect.Lists;
 import com.sparrowwallet.drongo.BitcoinUnit;
 import com.sparrowwallet.drongo.wallet.Wallet;
 import com.sparrowwallet.sparrow.io.Config;
+import com.sparrowwallet.sparrow.wallet.Entry;
 import com.sparrowwallet.sparrow.wallet.TransactionEntry;
 import com.sparrowwallet.sparrow.wallet.WalletTransactionsEntry;
 import javafx.beans.NamedArg;
 import javafx.scene.Node;
 import javafx.scene.chart.*;

+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;

 public class BalanceChart extends LineChart<Number, Number> {
+    private static final int MAX_VALUES = 500;
+
    private XYChart.Series<Number, Number> balanceSeries;

    private TransactionEntry selectedEntry;
@ -37,7 +43,7 @@ public class BalanceChart extends LineChart<Number, Number> {
        setVisible(!walletTransactionsEntry.getChildren().isEmpty());
        balanceSeries.getData().clear();

-        List<Data<Number, Number>> balanceDataList = walletTransactionsEntry.getChildren().stream()
+        List<Data<Number, Number>> balanceDataList = getTransactionEntries(walletTransactionsEntry)
                .map(entry -> (TransactionEntry)entry)
                .filter(txEntry -> txEntry.getBlockTransaction().getHeight() > 0)
                .map(txEntry -> new XYChart.Data<>((Number)txEntry.getBlockTransaction().getDate().getTime(), (Number)txEntry.getBalance(), txEntry))
@ -74,6 +80,24 @@ public class BalanceChart extends LineChart<Number, Number> {
        }
    }

+    private Stream<Entry> getTransactionEntries(WalletTransactionsEntry walletTransactionsEntry) {
+        int total = walletTransactionsEntry.getChildren().size();
+        if(walletTransactionsEntry.getChildren().size() <= MAX_VALUES) {
+            return walletTransactionsEntry.getChildren().stream();
+        }
+
+        int bucketSize = total / MAX_VALUES;
+        List<List<Entry>> buckets = Lists.partition(walletTransactionsEntry.getChildren(), bucketSize);
+        List<Entry> reducedEntries = new ArrayList<>(MAX_VALUES);
+        for(List<Entry> bucket : buckets) {
+            long max = bucket.stream().mapToLong(entry -> Math.abs(entry.getValue())).max().orElse(0);
+            Entry bucketEntry = bucket.stream().filter(entry -> entry.getValue() == max || entry.getValue() == -max).findFirst().orElseThrow();
+            reducedEntries.add(bucketEntry);
+        }
+
+        return reducedEntries.stream();
+    }
+
    public void select(TransactionEntry transactionEntry) {
        Set<Node> selectedSymbols = lookupAll(".chart-line-symbol.selected");
        for(Node selectedSymbol : selectedSymbols) {
--- a/src/main/java/com/sparrowwallet/sparrow/control/FileImportPane.java
+++ b/src/main/java/com/sparrowwallet/sparrow/control/FileImportPane.java
@ -158,6 +158,7 @@ public abstract class FileImportPane extends TitledDescriptionPane {
                try {
                    importFile(importer.getName(), null, null);
                } catch(ImportException e) {
+                    log.error("Error importing QR", e);
                    setError("Import Error", e.getMessage());
                }
            } else if(result.outputDescriptor != null) {
@ -165,6 +166,7 @@ public abstract class FileImportPane extends TitledDescriptionPane {
                try {
                    importFile(importer.getName(), null, null);
                } catch(ImportException e) {
+                    log.error("Error importing QR", e);
                    setError("Import Error", e.getMessage());
                }
            } else if(result.payload != null) {
--- a/src/main/java/com/sparrowwallet/sparrow/io/CaravanMultisig.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/CaravanMultisig.java
@ -78,7 +78,6 @@ public class CaravanMultisig implements WalletImport, WalletExport {

            return wallet;
        } catch(Exception e) {
-            log.error("Error importing " + getName() + " wallet", e);
            throw new ImportException("Error importing " + getName() + " wallet", e);
        }
    }
--- a/src/main/java/com/sparrowwallet/sparrow/io/CoboVaultSinglesig.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/CoboVaultSinglesig.java
@ -57,7 +57,6 @@ public class CoboVaultSinglesig implements KeystoreFileImport, WalletImport {

            return keystore;
        } catch (Exception e) {
-            log.error("Error getting " + getName() + " keystore", e);
            throw new ImportException("Error getting " + getName() + " keystore", e);
        }
    }
--- a/src/main/java/com/sparrowwallet/sparrow/io/ColdcardMultisig.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/ColdcardMultisig.java
@ -154,7 +154,6 @@ public class ColdcardMultisig implements WalletImport, KeystoreFileImport, Walle

            return wallet;
        } catch(Exception e) {
-            log.error("Error importing " + getName() + " wallet", e);
            throw new ImportException("Error importing " + getName() + " wallet", e);
        }
    }
--- a/src/main/java/com/sparrowwallet/sparrow/io/ColdcardSinglesig.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/ColdcardSinglesig.java
@ -86,7 +86,6 @@ public class ColdcardSinglesig implements KeystoreFileImport, WalletImport {
                }
            }
        } catch (Exception e) {
-            log.error("Error getting " + getName() + " keystore", e);
            throw new ImportException("Error getting " + getName() + " keystore", e);
        }

--- a/src/main/java/com/sparrowwallet/sparrow/io/Electrum.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/Electrum.java
@ -264,7 +264,6 @@ public class Electrum implements KeystoreFileImport, WalletImport, WalletExport

            return wallet;
        } catch (Exception e) {
-            log.error("Error importing Electrum Wallet", e);
            throw new ImportException("Error importing Electrum Wallet", e);
        }
    }
--- a/src/main/java/com/sparrowwallet/sparrow/io/KeystoneSinglesig.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/KeystoneSinglesig.java
@ -62,10 +62,8 @@ public class KeystoneSinglesig implements KeystoreFileImport, WalletImport {

            return keystore;
        } catch (IllegalArgumentException e) {
-            log.error("Error getting " + getName() + " keystore - not an output descriptor");
-            throw new ImportException("Error getting " + getName() + " keystore", e);
+            throw new ImportException("Error getting " + getName() + " keystore - not an output descriptor", e);
        } catch (Exception e) {
-            log.error("Error getting " + getName() + " keystore", e);
            throw new ImportException("Error getting " + getName() + " keystore", e);
        }
    }
--- a/src/main/java/com/sparrowwallet/sparrow/io/Sparrow.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/Sparrow.java
@ -121,7 +121,6 @@ public class Sparrow implements WalletImport, WalletExport {

            return wallet;
        } catch(IOException | StorageException e) {
-            log.error("Error importing Sparrow wallet", e);
            throw new ImportException("Error importing Sparrow wallet", e);
        } finally {
            if(storage != null) {
--- a/src/main/java/com/sparrowwallet/sparrow/io/SpecterDIY.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/SpecterDIY.java
@ -35,7 +35,6 @@ public class SpecterDIY implements KeystoreFileImport, WalletExport {

            return keystore;
        } catch(IOException e) {
-            log.error("Error getting " + getName() + " keystore", e);
            throw new ImportException("Error getting " + getName() + " keystore", e);
        }
    }
--- a/src/main/java/com/sparrowwallet/sparrow/io/SpecterDesktop.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/SpecterDesktop.java
@ -93,7 +93,6 @@ public class SpecterDesktop implements WalletImport, WalletExport {
                return wallet;
            }
        } catch(Exception e) {
-            log.error("Error importing " + getName() + " wallet", e);
            throw new ImportException("Error importing " + getName() + " wallet", e);
        }

--- a/src/main/java/com/sparrowwallet/sparrow/io/Storage.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/Storage.java
@ -534,6 +534,11 @@ public class Storage {
        private final Storage storage;
        private final SecureString password;

+        public LoadWalletService(Storage storage) {
+            this.storage = storage;
+            this.password = null;
+        }
+
        public LoadWalletService(Storage storage, SecureString password) {
            this.storage = storage;
            this.password = password;
@ -543,8 +548,15 @@ public class Storage {
        protected Task<WalletBackupAndKey> createTask() {
            return new Task<>() {
                protected WalletBackupAndKey call() throws IOException, StorageException {
-                    WalletBackupAndKey walletBackupAndKey = storage.loadEncryptedWallet(password);
-                    password.clear();
+                    WalletBackupAndKey walletBackupAndKey;
+
+                    if(password != null) {
+                        walletBackupAndKey = storage.loadEncryptedWallet(password);
+                        password.clear();
+                    } else {
+                        walletBackupAndKey = storage.loadUnencryptedWallet();
+                    }
+
                    return walletBackupAndKey;
                }
            };
--- a/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java
+++ b/src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java
@ -578,7 +578,7 @@ public class DbPersistence implements Persistence {
        if(updateExecutor != null) {
            updateExecutor.shutdown();
            try {
-                if(!updateExecutor.awaitTermination(10, TimeUnit.SECONDS)) {
+                if(!updateExecutor.awaitTermination(1, TimeUnit.MINUTES)) {
                    updateExecutor.shutdownNow();
                }

--- a/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java
+++ b/src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java
@ -166,7 +166,7 @@ public class ElectrumServer {
        }
    }

-    public static void addCalculatedScriptHashes(Wallet wallet) {
+    private static void addCalculatedScriptHashes(Wallet wallet) {
        getCalculatedScriptHashes(wallet).forEach(retrievedScriptHashes::putIfAbsent);
    }

@ -1248,8 +1248,12 @@ public class ElectrumServer {
        protected Task<Boolean> createTask() {
            return new Task<>() {
                protected Boolean call() throws ServerException {
-                    walletSynchronizeLocks.putIfAbsent(wallet, new Object());
+                    boolean initial = (walletSynchronizeLocks.putIfAbsent(wallet, new Object()) == null);
                    synchronized(walletSynchronizeLocks.get(wallet)) {
+                        if(initial) {
+                            addCalculatedScriptHashes(wallet);
+                        }
+
                        if(isConnected()) {
                            ElectrumServer electrumServer = new ElectrumServer();
                            Map<String, String> previousScriptHashes = getCalculatedScriptHashes(wallet);
--- a/src/main/java/com/sparrowwallet/sparrow/wallet/TransactionEntry.java
+++ b/src/main/java/com/sparrowwallet/sparrow/wallet/TransactionEntry.java
@ -38,7 +38,18 @@ public class TransactionEntry extends Entry implements Comparable<TransactionEnt
            }
        });

-        setConfirmations(calculateConfirmations());
+        confirmations = new IntegerPropertyBase(calculateConfirmations()) {
+            @Override
+            public Object getBean() {
+                return TransactionEntry.this;
+            }
+
+            @Override
+            public String getName() {
+                return "confirmations";
+            }
+        };
+
        if(isFullyConfirming()) {
            EventManager.get().register(this);
        }
@ -174,33 +185,17 @@ public class TransactionEntry extends Entry implements Comparable<TransactionEnt
    /**
     * Defines the number of confirmations
     */
-    private IntegerProperty confirmations;
+    private final IntegerProperty confirmations;

    public final void setConfirmations(int value) {
-        if(confirmations != null || value != 0) {
-            confirmationsProperty().set(value);
-        }
+        confirmations.set(value);
    }

    public final int getConfirmations() {
-        return confirmations == null ? 0 : confirmations.get();
+        return confirmations.get();
    }

    public final IntegerProperty confirmationsProperty() {
-        if(confirmations == null) {
-            confirmations = new IntegerPropertyBase(0) {
-
-                @Override
-                public Object getBean() {
-                    return TransactionEntry.this;
-                }
-
-                @Override
-                public String getName() {
-                    return "confirmations";
-                }
-            };
-        }
        return confirmations;
    }

--- a/src/main/java/com/sparrowwallet/sparrow/wallet/TransactionsController.java
+++ b/src/main/java/com/sparrowwallet/sparrow/wallet/TransactionsController.java
@ -5,7 +5,6 @@ import com.google.common.eventbus.Subscribe;
 import com.sparrowwallet.drongo.BitcoinUnit;
 import com.sparrowwallet.drongo.protocol.Transaction;
 import com.sparrowwallet.sparrow.AppServices;
-import com.sparrowwallet.sparrow.CurrencyRate;
 import com.sparrowwallet.sparrow.EventManager;
 import com.sparrowwallet.sparrow.control.*;
 import com.sparrowwallet.sparrow.event.*;
--- a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java
+++ b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java
@ -59,7 +59,6 @@ public class WalletForm {
        savedPastWallet = backupWallet;

        if(refreshHistory && wallet.isValid()) {
-            ElectrumServer.addCalculatedScriptHashes(wallet);
            refreshHistory(AppServices.getCurrentBlockHeight(), backupWallet);
        }
    }
@ -252,11 +251,13 @@ public class WalletForm {
    }

    private List<WalletNode> getHistoryChangedNodes(Set<WalletNode> previousNodes, Set<WalletNode> currentNodes) {
+        Map<WalletNode, WalletNode> previousNodeMap = new HashMap<>(previousNodes.size());
+        previousNodes.forEach(walletNode -> previousNodeMap.put(walletNode, walletNode));
+
        List<WalletNode> changedNodes = new ArrayList<>();
        for(WalletNode currentNode : currentNodes) {
-            Optional<WalletNode> optPreviousNode = previousNodes.stream().filter(node -> node.equals(currentNode)).findFirst();
-            if(optPreviousNode.isPresent()) {
-                WalletNode previousNode = optPreviousNode.get();
+            WalletNode previousNode = previousNodeMap.get(currentNode);
+            if(previousNode != null) {
                if(!currentNode.getTransactionOutputs().equals(previousNode.getTransactionOutputs())) {
                    changedNodes.add(currentNode);
                }
--- a/src/main/java/com/sparrowwallet/sparrow/wallet/WalletTransactionsEntry.java
+++ b/src/main/java/com/sparrowwallet/sparrow/wallet/WalletTransactionsEntry.java
@ -22,7 +22,7 @@ public class WalletTransactionsEntry extends Entry {

    public WalletTransactionsEntry(Wallet wallet) {
        super(wallet, wallet.getName(), getWalletTransactions(wallet).stream().map(WalletTransaction::getTransactionEntry).collect(Collectors.toList()));
-        calculateBalances();
+        calculateBalances(false); //No need to resort
    }

    @Override
@ -30,12 +30,14 @@ public class WalletTransactionsEntry extends Entry {
        return getBalance();
    }

-    protected void calculateBalances() {
+    private void calculateBalances(boolean resort) {
        long balance = 0L;
        long mempoolBalance = 0L;

-        //Note transaction entries must be in ascending order. This sorting is ultimately done according to BlockTransactions' comparator
-        getChildren().sort(Comparator.comparing(TransactionEntry.class::cast));
+        if(resort) {
+            //Note transaction entries must be in ascending order. This sorting is ultimately done according to BlockTransactions' comparator
+            getChildren().sort(Comparator.comparing(TransactionEntry.class::cast));
+        }

        for(Entry entry : getChildren()) {
            TransactionEntry transactionEntry = (TransactionEntry)entry;
@ -66,7 +68,7 @@ public class WalletTransactionsEntry extends Entry {
        entriesRemoved.removeAll(current);
        getChildren().removeAll(entriesRemoved);

-        calculateBalances();
+        calculateBalances(true);

        Map<HashIndex, BlockTransactionHashIndex> walletTxos = getWallet().getWalletTxos().entrySet().stream()
                .collect(Collectors.toUnmodifiableMap(entry -> new HashIndex(entry.getKey().getHash(), entry.getKey().getIndex()), Map.Entry::getKey));
@ -86,12 +88,14 @@ public class WalletTransactionsEntry extends Entry {
    }

    private static Collection<WalletTransaction> getWalletTransactions(Wallet wallet) {
-        Map<BlockTransaction, WalletTransaction> walletTransactionMap = new TreeMap<>();
+        Map<BlockTransaction, WalletTransaction> walletTransactionMap = new HashMap<>(wallet.getTransactions().size());

        getWalletTransactions(wallet, walletTransactionMap, wallet.getNode(KeyPurpose.RECEIVE));
        getWalletTransactions(wallet, walletTransactionMap, wallet.getNode(KeyPurpose.CHANGE));

-        return new ArrayList<>(walletTransactionMap.values());
+        List<WalletTransaction> walletTransactions = new ArrayList<>(walletTransactionMap.values());
+        Collections.sort(walletTransactions);
+        return walletTransactions;
    }

    private static void getWalletTransactions(Wallet wallet, Map<BlockTransaction, WalletTransaction> walletTransactionMap, WalletNode purposeNode) {
@ -191,7 +195,7 @@ public class WalletTransactionsEntry extends Entry {
        return mempoolBalance;
    }

-    private static class WalletTransaction {
+    private static class WalletTransaction implements Comparable<WalletTransaction> {
        private final Wallet wallet;
        private final BlockTransaction blockTransaction;
        private final Map<BlockTransactionHashIndex, KeyPurpose> incoming = new TreeMap<>();
@ -205,5 +209,33 @@ public class WalletTransactionsEntry extends Entry {
        public TransactionEntry getTransactionEntry() {
            return new TransactionEntry(wallet, blockTransaction, incoming, outgoing);
        }
+
+        public long getValue() {
+            long value = 0L;
+            for(BlockTransactionHashIndex in : incoming.keySet()) {
+                value += in.getValue();
+            }
+            for(BlockTransactionHashIndex out : outgoing.keySet()) {
+                value -= out.getValue();
+            }
+
+            return value;
+        }
+
+        @Override
+        public int compareTo(WalletTransactionsEntry.WalletTransaction other) {
+            //This comparison must be identical to that of TransactionEntry so we can avoid a resort calculating balances when creating WalletTransactionsEntry
+            int blockOrder = blockTransaction.compareBlockOrder(other.blockTransaction);
+            if(blockOrder != 0) {
+                return blockOrder;
+            }
+
+            int valueOrder = Long.compare(other.getValue(), getValue());
+            if(valueOrder != 0) {
+                return valueOrder;
+            }
+
+            return blockTransaction.compareTo(other.blockTransaction);
+        }
    }
 }