lukechilds
/
sparrow
mirror of https://github.com/lukechilds/sparrow.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

improve performance on deep wallets by storing addresses

terminal
Craig Raw 3 years ago
parent
11cda40a40
commit
60aa20ac55
12 changed files with 73 additions and 28 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      drongo
  2. 17
      src/main/java/com/sparrowwallet/sparrow/control/AddressTreeTable.java
  3. 4
      src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryChangedEvent.java
  4. 2
      src/main/java/com/sparrowwallet/sparrow/io/IOUtils.java
  5. 22
      src/main/java/com/sparrowwallet/sparrow/io/JsonPersistence.java
  6. 18
      src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java
  7. 2
      src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java
  8. 15
      src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeDao.java
  9. 7
      src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeMapper.java
  10. 7
      src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java
  11. 4
      src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java
  12. 1
      src/main/resources/com/sparrowwallet/sparrow/sql/V7__AddressData.sql

2
drongo

@ -1 +1 @@
Subproject commit 9ae1f68dc42529085edcc8c10d9bcfdbf9639448
Subproject commit 5de3abd36230d545f11bad3b25a21d23ffbbe9cd

17
src/main/java/com/sparrowwallet/sparrow/control/AddressTreeTable.java
View File

@ -14,9 +14,7 @@ import javafx.collections.ListChangeListener;
import javafx.scene.control.*;
import javafx.scene.input.MouseButton;
import java.util.List;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.*;
public class AddressTreeTable extends CoinTreeTable {
public void initialize(NodeEntry rootEntry) {
@ -110,10 +108,15 @@ public class AddressTreeTable extends CoinTreeTable {
//We only ever add child nodes - never remove in order to keep a full sequence (unless hide empty used addresses is set)
NodeEntry rootEntry = (NodeEntry)getRoot().getValue();
Map<WalletNode, NodeEntry> childNodes = new HashMap<>();
for(Entry childEntry : rootEntry.getChildren()) {
NodeEntry nodeEntry = (NodeEntry)childEntry;
childNodes.put(nodeEntry.getNode(), nodeEntry);
}
for(WalletNode updatedNode : updatedNodes) {
Optional<Entry> optEntry = rootEntry.getChildren().stream().filter(childEntry -> ((NodeEntry)childEntry).getNode().equals(updatedNode)).findFirst();
if(optEntry.isPresent()) {
NodeEntry existingEntry = (NodeEntry)optEntry.get();
NodeEntry existingEntry = childNodes.get(updatedNode);
if(existingEntry != null) {
existingEntry.refreshChildren();
if(Config.get().isHideEmptyUsedAddresses() && existingEntry.getValue() == 0L) {
@ -125,7 +128,7 @@ public class AddressTreeTable extends CoinTreeTable {
if(Config.get().isHideEmptyUsedAddresses()) {
int index = 0;
for( ; index < rootEntry.getChildren().size(); index++) {
NodeEntry existingEntry = (NodeEntry)rootEntry.getChildren().get(index);
existingEntry = (NodeEntry)rootEntry.getChildren().get(index);
if(nodeEntry.compareTo(existingEntry) < 0) {
break;
}

4
src/main/java/com/sparrowwallet/sparrow/event/WalletHistoryChangedEvent.java
View File

@ -45,10 +45,10 @@ public class WalletHistoryChangedEvent extends WalletChangedEvent {
}
public List<WalletNode> getReceiveNodes() {
return getWallet().getNode(KeyPurpose.RECEIVE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList());
return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.RECEIVE).collect(Collectors.toList());
}
public List<WalletNode> getChangeNodes() {
return getWallet().getNode(KeyPurpose.CHANGE).getChildren().stream().filter(historyChangedNodes::contains).collect(Collectors.toList());
return historyChangedNodes.stream().filter(node -> node.getKeyPurpose() == KeyPurpose.CHANGE).collect(Collectors.toList());
}
}

2
src/main/java/com/sparrowwallet/sparrow/io/IOUtils.java
View File

@ -131,7 +131,7 @@ public class IOUtils {
if(file.exists()) {
long length = file.length();
SecureRandom random = new SecureRandom();
byte[] data = new byte[64];
byte[] data = new byte[1024*1024];
random.nextBytes(data);
try(RandomAccessFile raf = new RandomAccessFile(file, "rws")) {
raf.seek(0);

22
src/main/java/com/sparrowwallet/sparrow/io/JsonPersistence.java
View File

@ -3,6 +3,8 @@ package com.sparrowwallet.sparrow.io;
import com.google.gson.*;
import com.sparrowwallet.drongo.ExtendedKey;
import com.sparrowwallet.drongo.Utils;
import com.sparrowwallet.drongo.address.Address;
import com.sparrowwallet.drongo.address.InvalidAddressException;
import com.sparrowwallet.drongo.crypto.Argon2KeyDeriver;
import com.sparrowwallet.drongo.crypto.AsymmetricKeyDeriver;
import com.sparrowwallet.drongo.crypto.ECKey;
@ -331,6 +333,8 @@ public class JsonPersistence implements Persistence {
gsonBuilder.registerTypeAdapter(Date.class, new DateDeserializer());
gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionSerializer());
gsonBuilder.registerTypeAdapter(Transaction.class, new TransactionDeserializer());
gsonBuilder.registerTypeAdapter(Address.class, new AddressSerializer());
gsonBuilder.registerTypeAdapter(Address.class, new AddressDeserializer());
if(includeWalletSerializers) {
gsonBuilder.registerTypeAdapter(Keystore.class, new KeystoreSerializer());
gsonBuilder.registerTypeAdapter(WalletNode.class, new NodeSerializer());
@ -429,6 +433,24 @@ public class JsonPersistence implements Persistence {
}
}
private static class AddressSerializer implements JsonSerializer<Address> {
@Override
public JsonElement serialize(Address src, Type typeOfSrc, JsonSerializationContext context) {
return new JsonPrimitive(src.toString());
}
}
private static class AddressDeserializer implements JsonDeserializer<Address> {
@Override
public Address deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
try {
return Address.fromString(json.getAsJsonPrimitive().getAsString());
} catch(InvalidAddressException e) {
throw new IllegalStateException(e);
}
}
}
private static class KeystoreSerializer implements JsonSerializer<Keystore> {
@Override
public JsonElement serialize(Keystore keystore, Type typeOfSrc, JsonSerializationContext context) {

18
src/main/java/com/sparrowwallet/sparrow/io/db/DbPersistence.java
View File

@ -87,6 +87,10 @@ public class DbPersistence implements Persistence {
return walletDao.getMainWallet(MASTER_SCHEMA, getWalletName(storage.getWalletFile(), null));
});
if(masterWallet == null) {
throw new StorageException("The wallet file was corrupted. Check the backups folder for previous copies.");
}
Map<WalletAndKey, Storage> childWallets = loadChildWallets(storage, masterWallet, encryptionKey);
masterWallet.setChildWallets(childWallets.keySet().stream().map(WalletAndKey::getWallet).collect(Collectors.toList()));
@ -231,12 +235,14 @@ public class DbPersistence implements Persistence {
if(addressNode.getId() == null) {
WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose());
if(purposeNode.getId() == null) {
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null);
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null);
purposeNode.setId(purposeNodeId);
}
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId());
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData());
addressNode.setId(nodeId);
} else if(addressNode.getAddress() != null) {
walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData());
}
Set<BlockTransactionHashIndex> txos = addressNode.getTransactionOutputs().stream().flatMap(txo -> txo.isSpent() ? Stream.of(txo, txo.getSpentBy()) : Stream.of(txo)).collect(Collectors.toSet());
@ -285,12 +291,14 @@ public class DbPersistence implements Persistence {
if(addressNode.getId() == null) {
WalletNode purposeNode = wallet.getNode(addressNode.getKeyPurpose());
if(purposeNode.getId() == null) {
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null);
long purposeNodeId = walletNodeDao.insertWalletNode(purposeNode.getDerivationPath(), purposeNode.getLabel(), wallet.getId(), null, null);
purposeNode.setId(purposeNodeId);
}
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId());
long nodeId = walletNodeDao.insertWalletNode(addressNode.getDerivationPath(), addressNode.getLabel(), wallet.getId(), purposeNode.getId(), addressNode.getAddressData());
addressNode.setId(nodeId);
} else if(addressNode.getAddress() != null) {
walletNodeDao.updateNodeAddressData(addressNode.getId(), addressNode.getAddressData());
}
walletNodeDao.updateNodeLabel(addressNode.getId(), entry.getLabel());
@ -666,7 +674,7 @@ public class DbPersistence implements Persistence {
}
private String getUrl(File walletFile, String password) {
return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES");
return "jdbc:h2:" + walletFile.getAbsolutePath().replace("." + getType().getExtension(), "") + ";INIT=SET TRACE_LEVEL_FILE=4;TRACE_LEVEL_FILE=4;DEFRAG_ALWAYS=true;MAX_COMPACT_TIME=5000;DATABASE_TO_UPPER=false" + (password == null ? "" : ";CIPHER=AES");
}
private boolean persistsFor(Wallet wallet) {

2
src/main/java/com/sparrowwallet/sparrow/io/db/WalletDao.java
View File

@ -108,7 +108,7 @@ public interface WalletDao {
default void loadWallet(Wallet wallet) {
wallet.getKeystores().addAll(createKeystoreDao().getForWalletId(wallet.getId()));
List<WalletNode> walletNodes = createWalletNodeDao().getForWalletId(wallet.getId());
List<WalletNode> walletNodes = createWalletNodeDao().getForWalletId(wallet.getScriptType().ordinal(), wallet.getId());
wallet.getPurposeNodes().addAll(walletNodes.stream().filter(walletNode -> walletNode.getDerivation().size() == 1).collect(Collectors.toList()));
wallet.getPurposeNodes().forEach(walletNode -> walletNode.setWallet(wallet));

15
src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeDao.java
View File

@ -16,18 +16,18 @@ import java.util.Date;
import java.util.List;
public interface WalletNodeDao {
@SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, " +
@SqlQuery("select walletNode.id, walletNode.derivationPath, walletNode.label, walletNode.parent, walletNode.addressData, ?, " +
"blockTransactionHashIndex.id, blockTransactionHashIndex.hash, blockTransactionHashIndex.height, blockTransactionHashIndex.date, blockTransactionHashIndex.fee, blockTransactionHashIndex.label, " +
"blockTransactionHashIndex.index, blockTransactionHashIndex.outputValue, blockTransactionHashIndex.status, blockTransactionHashIndex.spentBy, blockTransactionHashIndex.node " +
"from walletNode left join blockTransactionHashIndex on walletNode.id = blockTransactionHashIndex.node where walletNode.wallet = ? order by walletNode.parent asc nulls first, blockTransactionHashIndex.spentBy asc nulls first")
@RegisterRowMapper(WalletNodeMapper.class)
@RegisterRowMapper(BlockTransactionHashIndexMapper.class)
@UseRowReducer(WalletNodeReducer.class)
List<WalletNode> getForWalletId(Long id);
List<WalletNode> getForWalletId(int scriptType, Long id);
@SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent) values (?, ?, ?, ?)")
@SqlUpdate("insert into walletNode (derivationPath, label, wallet, parent, addressData) values (?, ?, ?, ?, ?)")
@GetGeneratedKeys("id")
long insertWalletNode(String derivationPath, String label, long wallet, Long parent);
long insertWalletNode(String derivationPath, String label, long wallet, Long parent, byte[] addressData);
@SqlUpdate("insert into blockTransactionHashIndex (hash, height, date, fee, label, index, outputValue, status, spentBy, node) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
@GetGeneratedKeys("id")
@ -39,6 +39,9 @@ public interface WalletNodeDao {
@SqlUpdate("update walletNode set label = :label where id = :id")
void updateNodeLabel(@Bind("id") long id, @Bind("label") String label);
@SqlUpdate("update walletNode set addressData = :addressData where id = :id and addressData is null")
void updateNodeAddressData(@Bind("id") long id, @Bind("addressData") byte[] addressData);
@SqlUpdate("update blockTransactionHashIndex set label = :label where id = :id")
void updateTxoLabel(@Bind("id") long id, @Bind("label") String label);
@ -59,12 +62,12 @@ public interface WalletNodeDao {
default void addWalletNodes(Wallet wallet) {
for(WalletNode purposeNode : wallet.getPurposeNodes()) {
long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null);
long purposeNodeId = insertWalletNode(purposeNode.getDerivationPath(), truncate(purposeNode.getLabel()), wallet.getId(), null, null);
purposeNode.setId(purposeNodeId);
addTransactionOutputs(purposeNode);
List<WalletNode> childNodes = new ArrayList<>(purposeNode.getChildren());
for(WalletNode addressNode : childNodes) {
long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId);
long addressNodeId = insertWalletNode(addressNode.getDerivationPath(), truncate(addressNode.getLabel()), wallet.getId(), purposeNodeId, addressNode.getAddressData());
addressNode.setId(addressNodeId);
addTransactionOutputs(addressNode);
}

7
src/main/java/com/sparrowwallet/sparrow/io/db/WalletNodeMapper.java
View File

@ -1,5 +1,6 @@
package com.sparrowwallet.sparrow.io.db;
import com.sparrowwallet.drongo.protocol.ScriptType;
import com.sparrowwallet.drongo.wallet.WalletNode;
import org.jdbi.v3.core.mapper.RowMapper;
import org.jdbi.v3.core.statement.StatementContext;
@ -13,7 +14,11 @@ public class WalletNodeMapper implements RowMapper<WalletNode> {
WalletNode walletNode = new WalletNode(rs.getString("walletNode.derivationPath"));
walletNode.setId(rs.getLong("walletNode.id"));
walletNode.setLabel(rs.getString("walletNode.label"));
byte[] addressData = rs.getBytes("walletNode.addressData");
if(addressData != null) {
ScriptType scriptType = ScriptType.values()[rs.getInt(6)];
walletNode.setAddress(scriptType.getAddress(addressData));
}
return walletNode;
}
}

7
src/main/java/com/sparrowwallet/sparrow/net/ElectrumServer.java
View File

@ -432,6 +432,7 @@ public class ElectrumServer {
try {
Set<String> scriptHashes = new HashSet<>();
Map<String, String> pathScriptHashes = new LinkedHashMap<>();
Map<String, WalletNode> pathNodes = new HashMap<>();
for(WalletNode node : nodes) {
if(node == null) {
log.error("Null node for wallet " + wallet.getFullName() + " subscribing nodes " + nodes + " startIndex " + startIndex, new Throwable());
@ -448,6 +449,7 @@ public class ElectrumServer {
} else if(!subscribedScriptHashes.containsKey(scriptHash) && scriptHashes.add(scriptHash)) {
//Unique script hash we are not yet subscribed to
pathScriptHashes.put(node.getDerivationPath(), scriptHash);
pathNodes.put(node.getDerivationPath(), node);
}
}
}
@ -463,9 +465,8 @@ public class ElectrumServer {
for(String path : result.keySet()) {
String status = result.get(path);
Optional<WalletNode> optionalNode = nodes.stream().filter(n -> n.getDerivationPath().equals(path)).findFirst();
if(optionalNode.isPresent()) {
WalletNode node = optionalNode.get();
WalletNode node = pathNodes.computeIfAbsent(path, p -> nodes.stream().filter(n -> n.getDerivationPath().equals(p)).findFirst().orElse(null));
if(node != null) {
String scriptHash = getScriptHash(wallet, node);
//Check if there is history for this script hash, and if the history has changed since last fetched

4
src/main/java/com/sparrowwallet/sparrow/wallet/WalletForm.java
View File

@ -201,7 +201,9 @@ public class WalletForm {
}
}
}
EventManager.get().post(new ChildWalletsAddedEvent(storage, wallet, addedWallets));
if(!addedWallets.isEmpty()) {
EventManager.get().post(new ChildWalletsAddedEvent(storage, wallet, addedWallets));
}
});
paymentCodesService.setOnFailed(failedEvent -> {
log.error("Could not determine payment codes for wallet " + wallet.getFullName(), failedEvent.getSource().getException());

1
src/main/resources/com/sparrowwallet/sparrow/sql/V7__AddressData.sql
View File

@ -0,0 +1 @@
alter table walletNode add column addressData varbinary(32) after parent;
Write Preview
Loading…
Cancel
Save