From 38b7ea525a6ada2eda0afbef25796cfcf1d34c96 Mon Sep 17 00:00:00 2001 From: Overtorment Date: Fri, 3 Apr 2020 13:10:53 +0100 Subject: [PATCH] ADD: support for ElectrumPersonalServer (disable protocol batching) --- BlueElectrum.js | 154 +++++++++++++++++- class/abstract-hd-electrum-wallet.js | 43 +++++ .../hd-segwit-bech32-wallet.test.js | 9 + 3 files changed, 201 insertions(+), 5 deletions(-) diff --git a/BlueElectrum.js b/BlueElectrum.js index 31b5f0e0..5c3eb41d 100644 --- a/BlueElectrum.js +++ b/BlueElectrum.js @@ -1,6 +1,6 @@ import AsyncStorage from '@react-native-community/async-storage'; import { Platform } from 'react-native'; -import { AppStorage } from './class'; +import { AppStorage, LegacyWallet, SegwitBech32Wallet, SegwitP2SHWallet } from './class'; const bitcoin = require('bitcoinjs-lib'); const ElectrumClient = require('electrum-client'); let reverse = require('buffer-reverse'); @@ -27,6 +27,10 @@ const hardcodedPeers = [ let mainClient: ElectrumClient = false; let mainConnected = false; let wasConnectedAtLeastOnce = false; +let serverName = false; +let disableBatching = false; + +let txhashHeightCache = {}; async function connectMain() { let usingPeer = await getRandomHardcodedPeer(); @@ -54,8 +58,13 @@ async function connectMain() { const ver = await mainClient.initElectrum({ client: 'bluewallet', version: '1.4' }); if (ver && ver[0]) { console.log('connected to ', ver); + serverName = ver[0]; mainConnected = true; wasConnectedAtLeastOnce = true; + if (ver[0].startsWith('ElectrumPersonalServer') || ver[0].startsWith('electrs')) { + // TODO: once they release support for batching - disable batching only for lower versions + disableBatching = true; + } // AsyncStorage.setItem(storageKey, JSON.stringify(peers)); TODO: refactor } } catch (e) { @@ -154,6 +163,7 @@ module.exports.getTransactionsByAddress = async function(address) { let hash = bitcoin.crypto.sha256(script); let reversedHash = Buffer.from(reverse(hash)); let history = await mainClient.blockchainScripthash_getHistory(reversedHash.toString('hex')); + if (history.tx_hash) txhashHeightCache[history.tx_hash] = history.height; // cache tx height return history; }; @@ -224,7 +234,16 @@ module.exports.multiGetBalanceByAddress = async function(addresses, batchsize) { scripthash2addr[reversedHash] = addr; } - let balances = await mainClient.blockchainScripthash_getBalanceBatch(scripthashes); + let balances = []; + + if (disableBatching) { + for (let sh of scripthashes) { + let balance = await mainClient.blockchainScripthash_getBalance(sh); + balances.push({ result: balance, param: sh }); + } + } else { + balances = await mainClient.blockchainScripthash_getBalanceBatch(scripthashes); + } for (let bal of balances) { ret.balance += +bal.result.confirmed; @@ -254,7 +273,13 @@ module.exports.multiGetUtxoByAddress = async function(addresses, batchsize) { scripthash2addr[reversedHash] = addr; } - let results = await mainClient.blockchainScripthash_listunspentBatch(scripthashes); + let results = []; + + if (disableBatching) { + // ElectrumPersonalServer doesnt support `blockchain.scripthash.listunspent` + } else { + results = await mainClient.blockchainScripthash_listunspentBatch(scripthashes); + } for (let utxos of results) { ret[scripthash2addr[utxos.param]] = utxos.result; @@ -289,10 +314,20 @@ module.exports.multiGetHistoryByAddress = async function(addresses, batchsize) { scripthash2addr[reversedHash] = addr; } - let results = await mainClient.blockchainScripthash_getHistoryBatch(scripthashes); + let results = []; + + if (disableBatching) { + for (let sh of scripthashes) { + let history = await mainClient.blockchainScripthash_getHistory(sh); + results.push({ result: history, param: sh }); + } + } else { + results = await mainClient.blockchainScripthash_getHistoryBatch(scripthashes); + } for (let history of results) { ret[scripthash2addr[history.param]] = history.result; + if (history.result[0]) txhashHeightCache[history.result[0].tx_hash] = history.result[0].height; // cache tx height for (let hist of ret[scripthash2addr[history.param]]) { hist.address = scripthash2addr[history.param]; } @@ -309,10 +344,35 @@ module.exports.multiGetTransactionByTxid = async function(txids, batchsize, verb verbose = verbose !== false; if (!mainClient) throw new Error('Electrum client is not connected'); let ret = {}; + txids = [...new Set(txids)]; // deduplicate just for any case let chunks = splitIntoChunks(txids, batchsize); for (let chunk of chunks) { - let results = await mainClient.blockchainTransaction_getBatch(chunk, verbose); + let results = []; + + if (disableBatching) { + for (let txid of chunk) { + try { + // in case of ElectrumPersonalServer it might not track some transactions (like source transactions for our transactions) + // so we wrap it in try-catch + let tx = await mainClient.blockchainTransaction_get(txid, verbose); + if (typeof tx === 'string' && verbose) { + // apparently electrum server (EPS?) didnt recognize VERBOSE parameter, and sent us plain txhex instead of decoded tx. + // lets decode it manually on our end then: + tx = txhexToElectrumTransaction(tx); + if (txhashHeightCache[txid]) { + // got blockheight where this tx was confirmed + tx.confirmations = this.estimateCurrentBlockheight() - txhashHeightCache[txid]; + tx.time = this.calculateBlockTime(txhashHeightCache[txid]); + tx.blocktime = this.calculateBlockTime(txhashHeightCache[txid]); + } + } + results.push({ result: tx, param: txid }); + } catch (_) {} + } + } else { + results = await mainClient.blockchainTransaction_getBatch(chunk, verbose); + } for (let txdata of results) { ret[txdata.param] = txdata.result; @@ -400,6 +460,23 @@ module.exports.broadcastV2 = async function(hex) { return mainClient.blockchainTransaction_broadcast(hex); }; +module.exports.estimateCurrentBlockheight = function() { + const baseTs = 1585837504347; // uS + const baseHeight = 624083; + return Math.floor(baseHeight + (+new Date() - baseTs) / 1000 / 60 / 10); +}; + +/** + * + * @param height + * @returns {number} Timestamp in seconds + */ +module.exports.calculateBlockTime = function(height) { + const baseTs = 1585837504; // sec + const baseHeight = 624083; + return baseTs + (height - baseHeight) * 10 * 60; +}; + /** * * @param host @@ -434,3 +511,70 @@ let splitIntoChunks = function(arr, chunkSize) { } return groups; }; + +function txhexToElectrumTransaction(txhex) { + let tx = bitcoin.Transaction.fromHex(txhex); + + let ret = { + txid: tx.getId(), + hash: tx.getId(), + version: tx.version, + size: Math.ceil(txhex.length / 2), + vsize: tx.virtualSize(), + weight: tx.weight(), + locktime: tx.locktime, + vin: [], + vout: [], + hex: txhex, + blockhash: '', + confirmations: 0, + time: 0, + blocktime: 0, + }; + + for (let inn of tx.ins) { + let txinwitness = []; + if (inn.witness[0]) txinwitness.push(inn.witness[0].toString('hex')); + if (inn.witness[1]) txinwitness.push(inn.witness[1].toString('hex')); + + ret.vin.push({ + txid: reverse(inn.hash).toString('hex'), + vout: inn.index, + scriptSig: { hex: inn.script.toString('hex'), asm: '' }, + txinwitness, + sequence: inn.sequence, + }); + } + + let n = 0; + for (let out of tx.outs) { + let value = new BigNumber(out.value).dividedBy(100000000).toNumber(); + let address = false; + let type = false; + + if (SegwitBech32Wallet.scriptPubKeyToAddress(out.script.toString('hex'))) { + address = SegwitBech32Wallet.scriptPubKeyToAddress(out.script.toString('hex')); + type = 'witness_v0_keyhash'; + } else if (SegwitP2SHWallet.scriptPubKeyToAddress(out.script.toString('hex'))) { + address = SegwitP2SHWallet.scriptPubKeyToAddress(out.script.toString('hex')); + type = '???'; // TODO + } else if (LegacyWallet.scriptPubKeyToAddress(out.script.toString('hex'))) { + address = LegacyWallet.scriptPubKeyToAddress(out.script.toString('hex')); + type = '???'; // TODO + } + + ret.vout.push({ + value, + n, + scriptPubKey: { + asm: '', + hex: out.script.toString('hex'), + reqSigs: 1, // todo + type, + addresses: [address], + }, + }); + n++; + } + return ret; +} diff --git a/class/abstract-hd-electrum-wallet.js b/class/abstract-hd-electrum-wallet.js index 01f0212c..567b81c9 100644 --- a/class/abstract-hd-electrum-wallet.js +++ b/class/abstract-hd-electrum-wallet.js @@ -702,9 +702,52 @@ export class AbstractHDElectrumWallet extends AbstractHDWallet { * @returns {[]} */ getUtxo() { + if (this._utxo.length === 0) return this.getDerivedUtxoFromOurTransaction(); // oy vey, no stored utxo. lets attempt to derive it from stored transactions return this._utxo; } + getDerivedUtxoFromOurTransaction() { + let utxos = []; + for (let tx of this.getTransactions()) { + for (let output of tx.outputs) { + let address = false; + if (output.scriptPubKey && output.scriptPubKey.addresses && output.scriptPubKey.addresses[0]) { + address = output.scriptPubKey.addresses[0]; + } + if (this.weOwnAddress(address)) { + let value = new BigNumber(output.value).multipliedBy(100000000).toNumber(); + utxos.push({ + txid: tx.txid, + txId: tx.txid, + vout: output.n, + address, + value, + amount: value, + confirmations: tx.confirmations, + wif: this._getWifForAddress(address), + height: BlueElectrum.estimateCurrentBlockheight() - tx.confirmations, + }); + } + } + } + + // got all utxos we ever had. lets filter out the ones that are spent: + let ret = []; + for (let utxo of utxos) { + let spent = false; + for (let tx of this.getTransactions()) { + for (let input of tx.inputs) { + if (input.txid === utxo.txid && input.vout === utxo.vout) spent = true; + // utxo we got previously was actually spent right here ^^ + } + } + + if (!spent) ret.push(utxo); + } + + return ret; + } + _getDerivationPathByAddress(address) { const path = "m/84'/0'/0'"; for (let c = 0; c < this.next_free_address_index + this.gap_limit; c++) { diff --git a/tests/integration/hd-segwit-bech32-wallet.test.js b/tests/integration/hd-segwit-bech32-wallet.test.js index 31821dfa..3ef7ea2e 100644 --- a/tests/integration/hd-segwit-bech32-wallet.test.js +++ b/tests/integration/hd-segwit-bech32-wallet.test.js @@ -197,6 +197,15 @@ describe('Bech32 Segwit HD (BIP84)', () => { assert.strictEqual(txFound, 4); await hd.fetchUtxo(); + assert.strictEqual(hd.getUtxo().length, 2); + assert.strictEqual(hd.getDerivedUtxoFromOurTransaction().length, 2); + let u1 = hd.getUtxo()[0]; + let u2 = hd.getDerivedUtxoFromOurTransaction()[0]; + delete u1.confirmations; + delete u2.confirmations; + delete u1.height; + delete u2.height; + assert.deepStrictEqual(u1, u2); let changeAddress = await hd.getChangeAddressAsync(); assert.ok(changeAddress && changeAddress.startsWith('bc1'));