From 3e8b092e865e3b6dd1847c7aa6925db16da8e5ae Mon Sep 17 00:00:00 2001 From: Dan Janosik Date: Thu, 25 Apr 2019 14:30:02 -0400 Subject: [PATCH] Support for pluggable Address APIs (work on #118 and #119) Move closer to the original vision of supporting any address-querying implementation desired. Current options include ElectrumX (as before) and now blockchain.com and blockcypher.com since they were easy to support and are publicly/easily available (though ridiculously neither is yet to support bc1 addresses). - new env var option BTCEXP_ADDRESS_API, value can be electrumx, blockchain.com, blockcypher.com - update ElectrumX client connect to request v1.4 of API as all clients are now supposed to do - misc frontend improvements/cleanup for addresses --- .env-sample | 12 +- README.md | 2 +- app.js | 41 +++-- app/api/addressApi.js | 79 +++++++++ app/api/blockchainAddressApi.js | 116 ++++++++++++ app/api/blockcypherAddressApi.js | 60 +++++++ .../{electrumApi.js => electrumAddressApi.js} | 122 +++++++++---- app/config.js | 3 +- app/utils.js | 18 +- bin/cli.js | 3 +- routes/baseActionsRouter.js | 165 ++++++++---------- views/address.pug | 85 +++++---- 12 files changed, 519 insertions(+), 187 deletions(-) create mode 100644 app/api/addressApi.js create mode 100644 app/api/blockchainAddressApi.js create mode 100644 app/api/blockcypherAddressApi.js rename app/api/{electrumApi.js => electrumAddressApi.js} (60%) diff --git a/.env-sample b/.env-sample index 5a95240..18e2647 100644 --- a/.env-sample +++ b/.env-sample @@ -14,10 +14,18 @@ #BTCEXP_BITCOIND_COOKIE=/path/to/bitcoind/.cookie #BTCEXP_BITCOIND_RPC_TIMEOUT=5000 -# Optional ElectrumX Servers, used to display address transaction histories -# Ref: https://uasf.saltylemon.org/electrum +# Select optional "address API" to display address tx lists and balances +# Options: electrumx, blockchain.com, blockcypher.com +# If electrumx set, the BTCEXP_ELECTRUMX_SERVERS variable must also be +# set. Neither blockchain.com or blockcypher.com support native-segwit bc1... +# addresses, but are convenient for users who don't run their own ElectrumX. +BTCEXP_ADDRESS_API=(electrumx|blockchain.com|blockcypher.com) + +# Optional ElectrumX Servers. See BTCEXP_ADDRESS_API. This value is only +# used if BTCEXP_ADDRESS_API=electrumx #BTCEXP_ELECTRUMX_SERVERS=tls://electrumx.server.com:50002,tcp://127.0.0.1:50001,... + # Optional InfluxDB Credentials (URI -OR- HOST/PORT/DBNAME/USER/PASS) #BTCEXP_ENABLE_INFLUXDB=true #BTCEXP_INFLUXDB_URI=influx://username:password@127.0.0.1:8086 diff --git a/README.md b/README.md index cf6178b..18bff79 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Live demos are available at: * View transaction details, with navigation "backward" via spent transaction outputs * View JSON content used to generate most pages * Search by transaction ID, block hash/height, and address -* Optional transaction history for addresses by querying configurable ElectrumX servers +* Optional transaction history for addresses by querying from ElectrumX, blockchain.com, or blockcypher.com * Mempool summary, with fee, size, and age breakdowns * RPC command browser and terminal * Currently supports BTC, LTC (support for any Bitcoin-RPC-protocol-compliant coin can be added easily) diff --git a/app.js b/app.js index d92358f..753f480 100755 --- a/app.js +++ b/app.js @@ -37,7 +37,8 @@ var coreApi = require("./app/api/coreApi.js"); var coins = require("./app/coins.js"); var request = require("request"); var qrcode = require("qrcode"); -var electrumApi = require("./app/api/electrumApi.js"); +var addressApi = require("./app/api/addressApi.js"); +var electrumAddressApi = require("./app/api/electrumAddressApi.js"); var Influx = require("influx"); var coreApi = require("./app/api/coreApi.js"); var auth = require('./app/auth.js'); @@ -357,17 +358,27 @@ app.runOnStartup = function() { }); } - if (config.electrumXServers && config.electrumXServers.length > 0) { - electrumApi.connectToServers().then(function() { - console.log("Live with ElectrumX API."); + if (config.addressApi) { + var supportedAddressApis = addressApi.getSupportedAddressApis(); + if (!supportedAddressApis.includes(config.addressApi)) { + utils.logError("32907ghsd0ge", `Unrecognized value for BTCEXP_ADDRESS_API: '${config.addressApi}'. Valid options are: ${supportedAddressApis}`); + } - global.electrumApi = electrumApi; - - }).catch(function(err) { - console.log("Error 31207ugf4e0fed: " + err + ", while initializing ElectrumX API"); - }); + if (config.addressApi == "electrumx") { + if (config.electrumXServers && config.electrumXServers.length > 0) { + electrumAddressApi.connectToServers().then(function() { + global.electrumAddressApi = electrumAddressApi; + + }).catch(function(err) { + utils.logError("31207ugf4e0fed", err, {electrumXServers:config.electrumXServers}); + }); + } else { + utils.logError("327hs0gde", "You must set the 'BTCEXP_ELECTRUMX_SERVERS' environment variable when BTCEXP_ADDRESS_API=electrumx."); + } + } } + loadMiningPoolConfigs(); if (global.sourcecodeVersion == null && fs.existsSync('.git')) { @@ -470,18 +481,6 @@ app.use(function(req, res, next) { } } - // electrum trust warnings on address pages - if (!req.session.hideElectrumTrustWarnings) { - var cookieValue = req.cookies['user-setting-hideElectrumTrustWarnings']; - - if (cookieValue) { - req.session.hideElectrumTrustWarnings = cookieValue; - - } else { - req.session.hideElectrumTrustWarnings = "false"; - } - } - res.locals.currencyFormatType = req.session.currencyFormatType; diff --git a/app/api/addressApi.js b/app/api/addressApi.js new file mode 100644 index 0000000..1ce2446 --- /dev/null +++ b/app/api/addressApi.js @@ -0,0 +1,79 @@ +var config = require("./../config.js"); +var coins = require("../coins.js"); +var utils = require("../utils.js"); + +var coinConfig = coins[config.coin]; + +var electrumAddressApi = require("./electrumAddressApi.js"); +var blockchainAddressApi = require("./blockchainAddressApi.js"); +var blockcypherAddressApi = require("./blockcypherAddressApi.js"); + +function getSupportedAddressApis() { + return ["blockchain.com", "blockcypher.com", "electrumx"]; +} + +function getCurrentAddressApiFeatureSupport() { + if (config.addressApi == "blockchain.com") { + return { + pageNumbers: true, + sortDesc: true, + sortAsc: true + }; + + } else if (config.addressApi == "blockcypher.com") { + return { + pageNumbers: true, + sortDesc: true, + sortAsc: false + }; + + } else if (config.addressApi == "electrumx") { + return { + pageNumbers: true, + sortDesc: true, + sortAsc: true + }; + } +} + +function getAddressDetails(address, scriptPubkey, sort, limit, offset) { + return new Promise(function(resolve, reject) { + var promises = []; + + if (config.addressApi == "blockchain.com") { + promises.push(blockchainAddressApi.getAddressDetails(address, scriptPubkey, sort, limit, offset)); + + } else if (config.addressApi == "blockcypher.com") { + promises.push(blockcypherAddressApi.getAddressDetails(address, scriptPubkey, sort, limit, offset)); + + } else if (config.addressApi == "electrumx") { + promises.push(electrumAddressApi.getAddressDetails(address, scriptPubkey, sort, limit, offset)); + + } else { + promises.push(new Promise(function(resolve, reject) { + resolve({addressDetails:null, errors:["No address API configured"]}); + })); + } + + Promise.all(promises).then(function(results) { + if (results && results.length > 0) { + resolve(results[0]); + + } else { + resolve(null); + } + }).catch(function(err) { + utils.logError("239x7rhsd0gs", err); + + reject(err); + }); + }); +} + + + +module.exports = { + getSupportedAddressApis: getSupportedAddressApis, + getCurrentAddressApiFeatureSupport: getCurrentAddressApiFeatureSupport, + getAddressDetails: getAddressDetails +}; \ No newline at end of file diff --git a/app/api/blockchainAddressApi.js b/app/api/blockchainAddressApi.js new file mode 100644 index 0000000..9e4b98d --- /dev/null +++ b/app/api/blockchainAddressApi.js @@ -0,0 +1,116 @@ +var request = require("request"); +var utils = require("./../utils.js"); + + +function getAddressDetails(address, scriptPubkey, sort, limit, offset) { + return new Promise(function(resolve, reject) { + if (address.startsWith("bc1")) { + reject({userText:"blockchain.com API does not support bc1 (native Segwit) addresses"}); + + return; + } + + if (sort == "asc") { + // need to query the total number of tx first, then build paging info from that value + var options = { + url: `https://blockchain.info/rawaddr/${address}?limit=1`, + headers: { + 'User-Agent': 'request' + } + }; + + request(options, function(error, response, body) { + if (error == null && response && response.statusCode && response.statusCode == 200) { + var blockchainJson = JSON.parse(body); + + var txCount = blockchainJson.n_tx; + var pageCount = parseInt(txCount / limit); + var lastPageSize = limit; + if (pageCount * limit < txCount) { + lastPageSize = txCount - pageCount * limit; + } + + var dynamicOffset = txCount - limit - offset; + if (dynamicOffset < 0) { + limit += dynamicOffset; + dynamicOffset += limit; + } + + getAddressDetailsSortDesc(address, limit, dynamicOffset).then(function(result) { + result.txids.reverse(); + + resolve({addressDetails:result}); + + }).catch(function(err) { + utils.logError("2308hsghse", err); + + reject(err); + }); + + } else { + var fullError = {error:error, response:response, body:body}; + + utils.logError("we0f8hasd0fhas", fullError); + + reject(fullError); + } + }); + } else { + getAddressDetailsSortDesc(address, limit, offset).then(function(result) { + resolve({addressDetails:result}); + + }).catch(function(err) { + utils.logError("3208hwssse", err); + + reject(err); + }); + } + }); +} + +function getAddressDetailsSortDesc(address, limit, offset) { + return new Promise(function(resolve, reject) { + var options = { + url: `https://blockchain.info/rawaddr/${address}?limit=${limit}&offset=${offset}`, + headers: { + 'User-Agent': 'request' + } + }; + + request(options, function(error, response, body) { + if (error == null && response && response.statusCode && response.statusCode == 200) { + var blockchainJson = JSON.parse(body); + + var response = {}; + + response.txids = []; + response.blockHeightsByTxid = {}; + blockchainJson.txs.forEach(function(tx) { + response.txids.push(tx.hash); + response.blockHeightsByTxid[tx.hash] = tx.block_height; + }); + + response.txCount = blockchainJson.n_tx; + response.hash160 = blockchainJson.hash160; + response.totalReceivedSat = blockchainJson.total_received; + response.totalSentSat = blockchainJson.total_sent; + response.balanceSat = blockchainJson.final_balance; + response.source = "blockchain.com"; + + resolve(response); + + } else { + var fullError = {error:error, response:response, body:body}; + + utils.logError("32907shsghs", fullError); + + reject(fullError); + } + }); + }); +} + + +module.exports = { + getAddressDetails: getAddressDetails +}; diff --git a/app/api/blockcypherAddressApi.js b/app/api/blockcypherAddressApi.js new file mode 100644 index 0000000..bb5d304 --- /dev/null +++ b/app/api/blockcypherAddressApi.js @@ -0,0 +1,60 @@ +var request = require("request"); + + +function getAddressDetails(address, scriptPubkey, sort, limit, offset) { + return new Promise(function(resolve, reject) { + if (address.startsWith("bc1")) { + reject({userText:"blockcypher.com API does not support bc1 (native Segwit) addresses"}); + + return; + } + + var limitOffset = limit + offset; + + var options = { + url: `https://api.blockcypher.com/v1/btc/main/addrs/${address}?limit=${limitOffset}`, + headers: { + 'User-Agent': 'request' + } + }; + + request(options, function(error, response, body) { + if (error == null && response && response.statusCode && response.statusCode == 200) { + var blockcypherJson = JSON.parse(body); + + var response = {}; + + response.txids = []; + response.blockHeightsByTxid = {}; + + // blockcypher doesn't support offset for paging, so simulate up to the hard cap of 2,000 + for (var i = offset; i < Math.min(blockcypherJson.txrefs.length, limitOffset); i++) { + var tx = blockcypherJson.txrefs[i]; + + response.txids.push(tx.tx_hash); + response.blockHeightsByTxid[tx.tx_hash] = tx.block_height; + } + + response.txCount = blockcypherJson.n_tx; + response.totalReceivedSat = blockcypherJson.total_received; + response.totalSentSat = blockcypherJson.total_sent; + response.balanceSat = blockcypherJson.final_balance; + response.source = "blockcypher.com"; + + resolve({addressDetails:response}); + + } else { + var fullError = {error:error, response:response, body:body}; + + utils.logError("097wef0adsgadgs", fullError); + + reject(fullError); + } + }); + }); +} + + +module.exports = { + getAddressDetails: getAddressDetails +}; \ No newline at end of file diff --git a/app/api/electrumApi.js b/app/api/electrumAddressApi.js similarity index 60% rename from app/api/electrumApi.js rename to app/api/electrumAddressApi.js index f032f4d..69fe331 100644 --- a/app/api/electrumApi.js +++ b/app/api/electrumAddressApi.js @@ -1,7 +1,12 @@ +var debug = require("debug"); +var debugLog = debug("btcexp:electrumx"); + var config = require("./../config.js"); var coins = require("../coins.js"); var utils = require("../utils.js"); - +var sha256 = require("crypto-js/sha256"); +var hexEnc = require("crypto-js/enc-hex"); + var coinConfig = coins[config.coin]; const ElectrumClient = require('electrum-client'); @@ -21,52 +26,30 @@ function connectToServers() { resolve(); }).catch(function(err) { - console.log("Error 120387rygxx231gwe40: " + err); + utils.logError("120387rygxx231gwe40", err); reject(err); }); }); } -function reconnectToServers() { - return new Promise(function(resolve, reject) { - for (var i = 0; i < electrumClients.length; i++) { - electrumClients[i].close(); - } - - electrumClients = []; - - console.log("Reconnecting ElectrumX sockets..."); - - connectToServers().then(function() { - console.log("Done reconnecting ElectrumX sockets."); - - resolve(); - - }).catch(function(err) { - console.log("Error 317fh29y7fg3333: " + err); - - resolve(); - }); - }); -} - function connectToServer(host, port, protocol) { return new Promise(function(resolve, reject) { - console.log("Connecting to ElectrumX Server: " + host + ":" + port); + debugLog("Connecting to ElectrumX Server: " + host + ":" + port); // default protocol is 'tcp' if port is 50001, which is the default unencrypted port for electrumx var defaultProtocol = port === 50001 ? 'tcp' : 'tls'; + var electrumClient = new ElectrumClient(port, host, protocol || defaultProtocol); - electrumClient.initElectrum({client:"btc-rpc-explorer-v1.1", version:"1.2"}).then(function(res) { - console.log("Connected to ElectrumX Server: " + host + ":" + port + ", versions: " + JSON.stringify(res)); + electrumClient.initElectrum({client:"btc-rpc-explorer-v1.1", version:"1.4"}).then(function(res) { + debugLog("Connected to ElectrumX Server: " + host + ":" + port + ", versions: " + JSON.stringify(res)); electrumClients.push(electrumClient); resolve(); }).catch(function(err) { - console.log("Error 137rg023xx7gerfwdd: " + err + ", when trying to connect to ElectrumX server at " + host + ":" + port); + utils.logError("137rg023xx7gerfwdd", err, {host:host, port:port, protocol:protocol}); reject(err); }); @@ -84,7 +67,7 @@ function runOnServer(electrumClient, f) { reject({error:result.error, server:electrumClient.host}); } }).catch(function(err) { - console.log("Error dif0e21qdh: " + err + ", host=" + electrumClient.host + ", port=" + electrumClient.port); + utils.logError("dif0e21qdh", err, {host:electrumClient.host, port:electrumClient.port}); reject(err); }); @@ -108,12 +91,85 @@ function runOnAllServers(f) { }); } +function getAddressDetails(address, scriptPubkey, sort, limit, offset) { + return new Promise(function(resolve, reject) { + var addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubkey))); + addrScripthash = addrScripthash.match(/.{2}/g).reverse().join(""); + + var promises = []; + + var txidData = null; + var balanceData = null; + + promises.push(new Promise(function(resolve, reject) { + getAddressTxids(addrScripthash).then(function(result) { + txidData = result.result; + + resolve(); + + }).catch(function(err) { + utils.logError("2397wgs0sgse", err); + + reject(err); + }); + })); + + promises.push(new Promise(function(resolve, reject) { + getAddressBalance(addrScripthash).then(function(result) { + balanceData = result.result; + + resolve(); + + }).catch(function(err) { + utils.logError("21307ws70sg", err); + + reject(err); + }); + })); + + Promise.all(promises.map(utils.reflectPromise)).then(function(results) { + var addressDetails = {}; + + if (txidData) { + addressDetails.txCount = txidData.length; + + addressDetails.txids = []; + addressDetails.blockHeightsByTxid = {}; + + if (sort == "desc") { + txidData.reverse(); + } + + for (var i = offset; i < Math.min(txidData.length, limit + offset); i++) { + addressDetails.txids.push(txidData[i].tx_hash); + addressDetails.blockHeightsByTxid[txidData[i].tx_hash] = txidData[i].height; + } + } + + if (balanceData) { + addressDetails.balanceSat = balanceData.confirmed; + } + + var errors = []; + results.forEach(function(x) { + if (x.status == "rejected") { + errors.push(x); + } + }); + + resolve({addressDetails:addressDetails, errors:errors}); + }); + }); +} + function getAddressTxids(addrScripthash) { return new Promise(function(resolve, reject) { runOnAllServers(function(electrumClient) { return electrumClient.blockchainScripthash_getHistory(addrScripthash); }).then(function(results) { + debugLog(`getAddressTxids=${utils.ellipsize(JSON.stringify(results), 200)}`); + if (addrScripthash == coinConfig.genesisCoinbaseOutputAddressScripthash) { for (var i = 0; i < results.length; i++) { results[i].result.unshift({tx_hash:coinConfig.genesisCoinbaseTransactionId, height:0}); @@ -146,6 +202,8 @@ function getAddressBalance(addrScripthash) { return electrumClient.blockchainScripthash_getBalance(addrScripthash); }).then(function(results) { + debugLog(`getAddressBalance=${JSON.stringify(results)}`); + if (addrScripthash == coinConfig.genesisCoinbaseOutputAddressScripthash) { for (var i = 0; i < results.length; i++) { var coinbaseBlockReward = coinConfig.blockRewardFunction(0); @@ -176,7 +234,5 @@ function getAddressBalance(addrScripthash) { module.exports = { connectToServers: connectToServers, - reconnectToServers: reconnectToServers, - getAddressTxids: getAddressTxids, - getAddressBalance: getAddressBalance + getAddressDetails: getAddressDetails }; \ No newline at end of file diff --git a/app/config.js b/app/config.js index 6ebb7e4..3d1456a 100644 --- a/app/config.js +++ b/app/config.js @@ -124,13 +124,14 @@ module.exports = { "walletpassphrasechange", ], + addressApi:process.env.BTCEXP_ADDRESS_API, electrumXServers:electrumXServers, redisUrl:process.env.BTCEXP_REDIS_URL, site: { blockTxPageSize:20, - addressTxPageSize:20, + addressTxPageSize:10, txMaxInput:15, browseBlocksPageSize:20, addressPage:{ diff --git a/app/utils.js b/app/utils.js index 6b9598b..0c47d5d 100644 --- a/app/utils.js +++ b/app/utils.js @@ -241,6 +241,15 @@ function logAppStats() { } } +function ellipsize(str, length) { + if (str.length <= length) { + return str; + + } else { + return str.substring(0, length - 3) + "..."; + } +} + function logMemoryUsage() { var mbUsed = process.memoryUsage().heapUsed / 1024 / 1024; mbUsed = Math.round(mbUsed * 100) / 100; @@ -511,6 +520,11 @@ function colorHexToHsl(hex) { return rgbToHsl(rgb.r, rgb.g, rgb.b); } + +// https://stackoverflow.com/a/31424853/673828 +const reflectPromise = p => p.then(v => ({v, status: "resolved" }), + e => ({e, status: "rejected" })); + function logError(errorId, err, optionalUserData = null) { if (!global.errorLog) { global.errorLog = []; @@ -580,6 +594,7 @@ function buildQrCodeUrl(str, results) { module.exports = { + reflectPromise: reflectPromise, redirectToConnectPageIfNeeded: redirectToConnectPageIfNeeded, hex2ascii: hex2ascii, splitArrayIntoChunks: splitArrayIntoChunks, @@ -605,5 +620,6 @@ module.exports = { colorHexToRgb: colorHexToRgb, colorHexToHsl: colorHexToHsl, logError: logError, - buildQrCodeUrls: buildQrCodeUrls + buildQrCodeUrls: buildQrCodeUrls, + ellipsize: ellipsize }; diff --git a/bin/cli.js b/bin/cli.js index 02e0e31..f1010fe 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -17,7 +17,8 @@ const args = require('meow')(` -u, --bitcoind-user username for bitcoind rpc [default: none] -w, --bitcoind-pass password for bitcoind rpc [default: none] - -E, --electrumx-servers <..> comma separated list of electrum servers to use for address queries [default: none] + --address-api