diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6790a..52610f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ * **IMPORTANT**: Use of `/block-analysis` can put heavy memory pressure on this app, depending on the details of the block being analyzed. If your app is crashing, consider setting a higher memory ceiling: `node --max_old_space_size=XXX bin/www` (where `XXX` is measured in MB). * Change `/mempool-summary` to load data via ajax (UX improvement to give feedback while loading large data sets) * Zero-indexing for tx index-in-block values +* Reduced memory usage +* Versioning for cache keys if using persistent cache (redis) * Configurable UI "sub-header" links * Start of RPC API versioning support * Tweaked styling across site @@ -46,6 +48,7 @@ * Remove "Bitcoin Explorer" H1 (it's redundant) * Hide the "Date" (timestamp) column for recent blocks (the Age+TTM is more valuable) * Updated miner configs +* Lots of minor bug fixes #### v1.1.9 ##### 2020-02-23 diff --git a/README.md b/README.md index 3186a93..a7e9548 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,13 @@ This tool is intended to be a simple, self-hosted explorer for the Bitcoin block Whatever reasons one might have for running a full node (trustlessness, technical curiosity, supporting the network, etc) it's helpful to appreciate the "fullness" of your node. With this explorer, you can not only explore the blockchain (in the traditional sense of the term "explorer"), but also explore the functional capabilities of your own node. -Live demo available at: [https://btc-explorer.com](https://btc-explorer.com) +Live demo available at: [https://explorer.btc21.org](https://explorer.btc21.org) # Features -* Browse blocks -* View block details -* View transaction details, with navigation "backward" via spent transaction outputs +* Network Summary "dashboard" +* View details of blocks, transactions, and addresses +* Analysis tools for viewing stats on blocks, transactions, and miner activity * View JSON content used to generate most pages * Search by transaction ID, block hash/height, and address * Optional transaction history for addresses by querying from ElectrumX, blockchain.com, blockchair.com, or blockcypher.com @@ -71,7 +71,7 @@ See `btc-rpc-explorer --help` for the full list of CLI options. # Support -* [https://donation.btc21.org](https://donate.btc21.org/apps/2TBP2GuQnYXGBiHQkmf4jNuMh6eN/pos) +* [https://donate.btc21.org](https://donate.btc21.org/apps/2TBP2GuQnYXGBiHQkmf4jNuMh6eN/pos) [npm-ver-img]: https://img.shields.io/npm/v/btc-rpc-explorer.svg?style=flat diff --git a/app/api/coreApi.js b/app/api/coreApi.js index ca3437c..67e97a0 100644 --- a/app/api/coreApi.js +++ b/app/api/coreApi.js @@ -16,6 +16,10 @@ var rpcApi = require("./rpcApi.js"); //var rpcApi = require("./mockApi.js"); +// this value should be incremented whenever data format changes, to avoid +// pulling old-format data from a persistent cache +var cacheKeyVersion = "v0"; + function onCacheEvent(cacheType, hitOrMiss, cacheKey) { //debugLog(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`); } @@ -84,7 +88,10 @@ function getGenesisCoinbaseTransactionId() { function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheConditionFunction) { - //debugLog("tryCache: " + cacheKey + ", " + cacheMaxAge); + var versionedCacheKey = `${cacheKeyVersion}-${cacheKey}`; + + //debugLog("tryCache: " + versionedCacheKey + ", " + cacheMaxAge); + if (cacheConditionFunction == null) { cacheConditionFunction = function(obj) { return true; @@ -101,7 +108,7 @@ function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheC } else { rpcApiFunction().then(function(rpcResult) { if (rpcResult != null && cacheConditionFunction(rpcResult)) { - cache.set(cacheKey, rpcResult, cacheMaxAge); + cache.set(versionedCacheKey, rpcResult, cacheMaxAge); } resolve(rpcResult); @@ -112,13 +119,13 @@ function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheC } }; - cache.get(cacheKey).then(function(result) { + cache.get(versionedCacheKey).then(function(result) { cacheResult = result; finallyFunc(); }).catch(function(err) { - utils.logError("nds9fc2eg621tf3", err, {cacheKey:cacheKey}); + utils.logError("nds9fc2eg621tf3", err, {cacheKey:versionedCacheKey}); finallyFunc(); }); @@ -387,49 +394,8 @@ function getMempoolDetails(start, count) { txids.push(resultTxids[i]); } - getRawTransactions(txids).then(function(transactions) { - var maxInputsTracked = config.site.txMaxInput; - var vinTxids = []; - for (var i = 0; i < transactions.length; i++) { - var transaction = transactions[i]; - - if (transaction && transaction.vin) { - for (var j = 0; j < Math.min(maxInputsTracked, transaction.vin.length); j++) { - if (transaction.vin[j].txid) { - vinTxids.push(transaction.vin[j].txid); - } - } - } - } - - var txInputsByTransaction = {}; - getRawTransactions(vinTxids).then(function(vinTransactions) { - var vinTxById = {}; - - vinTransactions.forEach(function(tx) { - vinTxById[tx.txid] = tx; - }); - - transactions.forEach(function(tx) { - txInputsByTransaction[tx.txid] = {}; - - if (tx && tx.vin) { - for (var i = 0; i < Math.min(maxInputsTracked, tx.vin.length); i++) { - if (vinTxById[tx.vin[i].txid]) { - txInputsByTransaction[tx.txid][i] = vinTxById[tx.vin[i].txid]; - } - } - } - }); - - resolve({ txCount:resultTxids.length, transactions:transactions, txInputsByTransaction:txInputsByTransaction }); - - }).catch(function(err) { - reject(err); - }); - - }).catch(function(err) { - reject(err); + getRawTransactionsWithInputs(txids, config.site.txMaxInput).then(function(result) { + resolve({ txCount:resultTxids.length, transactions:result.transactions, txInputsByTransaction:result.txInputsByTransaction }); }); }).catch(function(err) { @@ -696,6 +662,37 @@ function getRawTransaction(txid) { return tryCacheThenRpcApi(txCache, "getRawTransaction-" + txid, 3600000, rpcApiFunction, shouldCacheTransaction); } +/* + This function pulls raw tx data and then summarizes the outputs. It's used in memory-constrained situations. +*/ +function getSummarizedTransactionOutput(txid, voutIndex) { + var rpcApiFunction = function() { + return new Promise(function(resolve, reject) { + rpcApi.getRawTransaction(txid).then(function(rawTx) { + var vout = rawTx.vout[voutIndex]; + if (vout.scriptPubKey) { + if (vout.scriptPubKey.asm) { + delete vout.scriptPubKey.asm; + } + + if (vout.scriptPubKey.hex) { + delete vout.scriptPubKey.hex; + } + } + + vout.txid = txid; + + resolve(vout); + + }).catch(function(err) { + reject(err); + }); + }); + }; + + return tryCacheThenRpcApi(txCache, `txoSummary-${txid}-${voutIndex}`, 3600000, rpcApiFunction, shouldCacheTransaction); +} + function getTxUtxos(tx) { return new Promise(function(resolve, reject) { var promises = []; @@ -813,7 +810,7 @@ function summarizeBlockAnalysisData(blockHeight, tx, inputs) { } else { for (var i = 0; i < tx.vin.length; i++) { var vin = tx.vin[i]; - var inputVout = inputs[i].vout[vin.vout]; + var inputVout = inputs[i]; txSummary.totalInput = txSummary.totalInput.plus(new Decimal(inputVout.value)); @@ -866,34 +863,45 @@ function getRawTransactionsWithInputs(txids, maxInputs=-1) { maxInputsTracked = maxInputs; } - var vinTxids = []; + var vinIds = []; for (var i = 0; i < transactions.length; i++) { var transaction = transactions[i]; if (transaction && transaction.vin) { for (var j = 0; j < Math.min(maxInputsTracked, transaction.vin.length); j++) { if (transaction.vin[j].txid) { - vinTxids.push(transaction.vin[j].txid); + vinIds.push({txid:transaction.vin[j].txid, voutIndex:transaction.vin[j].vout}); } } } } - var txInputsByTransaction = {}; - getRawTransactions(vinTxids).then(function(vinTransactions) { - var vinTxById = {}; + var promises = []; - vinTransactions.forEach(function(tx) { - vinTxById[tx.txid] = tx; - }); + for (var i = 0; i < vinIds.length; i++) { + var vinId = vinIds[i]; + + promises.push(getSummarizedTransactionOutput(vinId.txid, vinId.voutIndex)); + } + + Promise.all(promises).then(function(promiseResults) { + var summarizedTxOutputs = {}; + for (var i = 0; i < promiseResults.length; i++) { + var summarizedTxOutput = promiseResults[i]; + + summarizedTxOutputs[`${summarizedTxOutput.txid}:${summarizedTxOutput.n}`] = summarizedTxOutput; + } + + var txInputsByTransaction = {}; transactions.forEach(function(tx) { txInputsByTransaction[tx.txid] = {}; if (tx && tx.vin) { for (var i = 0; i < Math.min(maxInputsTracked, tx.vin.length); i++) { - if (vinTxById[tx.vin[i].txid]) { - txInputsByTransaction[tx.txid][i] = vinTxById[tx.vin[i].txid]; + var summarizedTxOutput = summarizedTxOutputs[`${tx.vin[i].txid}:${tx.vin[i].vout}`]; + if (summarizedTxOutput) { + txInputsByTransaction[tx.txid][i] = summarizedTxOutput; } } } @@ -918,54 +926,19 @@ function getBlockByHashWithTransactions(blockHash, txLimit, txOffset) { txids.push(block.tx[i]); } - getRawTransactions(txids).then(function(transactions) { - if (transactions.length == txids.length) { - block.coinbaseTx = transactions[0]; + getRawTransactionsWithInputs(txids, config.site.txMaxInput).then(function(txsResult) { + if (txsResult.transactions && txsResult.transactions.length > 0) { + block.coinbaseTx = txsResult.transactions[0]; block.totalFees = utils.getBlockTotalFeesFromCoinbaseTxAndBlockHeight(block.coinbaseTx, block.height); block.miner = utils.getMinerFromCoinbaseTx(block.coinbaseTx); } - // if we're on page 2, we don't really want it anymore... + // if we're on page 2, we don't really want the coinbase tx in the tx list anymore if (txOffset > 0) { transactions.shift(); } - var maxInputsTracked = config.site.txMaxInput; - var vinTxids = []; - for (var i = 0; i < transactions.length; i++) { - var transaction = transactions[i]; - - if (transaction && transaction.vin) { - for (var j = 0; j < Math.min(maxInputsTracked, transaction.vin.length); j++) { - if (transaction.vin[j].txid) { - vinTxids.push(transaction.vin[j].txid); - } - } - } - } - - var txInputsByTransaction = {}; - getRawTransactions(vinTxids).then(function(vinTransactions) { - var vinTxById = {}; - - vinTransactions.forEach(function(tx) { - vinTxById[tx.txid] = tx; - }); - - transactions.forEach(function(tx) { - txInputsByTransaction[tx.txid] = {}; - - if (tx && tx.vin) { - for (var i = 0; i < Math.min(maxInputsTracked, tx.vin.length); i++) { - if (vinTxById[tx.vin[i].txid]) { - txInputsByTransaction[tx.txid][i] = vinTxById[tx.vin[i].txid]; - } - } - } - - resolve({ getblock:block, transactions:transactions, txInputsByTransaction:txInputsByTransaction }); - }); - }); + resolve({ getblock:block, transactions:txsResult.transactions, txInputsByTransaction:txsResult.txInputsByTransaction }); }); }); }); diff --git a/app/utils.js b/app/utils.js index 2c9c114..e6c2b96 100644 --- a/app/utils.js +++ b/app/utils.js @@ -411,7 +411,7 @@ function getTxTotalInputOutputValues(tx, txInputs, blockHeight) { if (txInput) { try { - var vout = txInput.vout[tx.vin[i].vout]; + var vout = txInput; if (vout.value) { totalInputValue = totalInputValue.plus(new Decimal(vout.value)); } diff --git a/docs/Server-Setup.md b/docs/Server-Setup.md index efa88b4..68f79f3 100644 --- a/docs/Server-Setup.md +++ b/docs/Server-Setup.md @@ -1,4 +1,4 @@ -### Setup of https://btc-explorer.com on Ubuntu 16.04 +### Setup of https://explorer.btc21.org on Ubuntu 16.04 apt update apt upgrade @@ -11,9 +11,9 @@ apt upgrade apt install python-certbot-nginx -Copy content from [./btc-explorer.com.conf](./btc-explorer.com.conf) into `/etc/nginx/sites-available/btc-explorer.com.conf` +Copy content from [./explorer.btc21.org.conf](./explorer.btc21.org.conf) into `/etc/nginx/sites-available/explorer.btc21.org.conf` - certbot --nginx -d btc-explorer.com + certbot --nginx -d explorer.btc21.org cd /etc/ssl/certs openssl dhparam -out dhparam.pem 4096 cd /home/bitcoin diff --git a/docs/btc-explorer.com.conf b/docs/btc-explorer.com.conf index 66f1277..b7a2972 100644 --- a/docs/btc-explorer.com.conf +++ b/docs/btc-explorer.com.conf @@ -1,17 +1,17 @@ ## http://domain.com redirects to https://domain.com server { - server_name btc-explorer.com; + server_name explorer.btc21.org; listen 80; #listen [::]:80 ipv6only=on; location / { - return 301 https://btc-explorer.com$request_uri; + return 301 https://explorer.btc21.org$request_uri; } } ## Serves httpS://domain.com server { - server_name btc-explorer.com; + server_name explorer.btc21.org; listen 443 ssl http2; #listen [::]:443 ssl http2 ipv6only=on; diff --git a/routes/baseActionsRouter.js b/routes/baseActionsRouter.js index c28dd72..02d5d66 100644 --- a/routes/baseActionsRouter.js +++ b/routes/baseActionsRouter.js @@ -22,6 +22,8 @@ var coreApi = require("./../app/api/coreApi.js"); var addressApi = require("./../app/api/addressApi.js"); var rpcApi = require("./../app/api/rpcApi.js"); +const v8 = require('v8'); + const forceCsrf = csurf({ ignoreMethods: [] }); router.get("/", function(req, res, next) { @@ -792,13 +794,16 @@ router.get("/tx/:transactionId", function(req, res, next) { res.locals.result = {}; - coreApi.getRawTransaction(txid).then(function(rawTxResult) { - res.locals.result.getrawtransaction = rawTxResult; + coreApi.getRawTransactionsWithInputs([txid]).then(function(rawTxResult) { + var tx = rawTxResult.transactions[0]; + + res.locals.result.getrawtransaction = tx; + res.locals.result.txInputs = rawTxResult.txInputsByTransaction[txid]; var promises = []; promises.push(new Promise(function(resolve, reject) { - coreApi.getTxUtxos(rawTxResult).then(function(utxos) { + coreApi.getTxUtxos(tx).then(function(utxos) { res.locals.utxos = utxos; resolve(); @@ -810,7 +815,7 @@ router.get("/tx/:transactionId", function(req, res, next) { }); })); - if (rawTxResult.confirmations == null) { + if (tx.confirmations == null) { promises.push(new Promise(function(resolve, reject) { coreApi.getMempoolTxDetails(txid, true).then(function(mempoolDetails) { res.locals.mempoolDetails = mempoolDetails; @@ -826,40 +831,23 @@ router.get("/tx/:transactionId", function(req, res, next) { } promises.push(new Promise(function(resolve, reject) { - global.rpcClient.command('getblock', rawTxResult.blockhash, function(err3, result3, resHeaders3) { + global.rpcClient.command('getblock', tx.blockhash, function(err3, result3, resHeaders3) { res.locals.result.getblock = result3; - var txids = []; - for (var i = 0; i < rawTxResult.vin.length; i++) { - if (!rawTxResult.vin[i].coinbase) { - txids.push(rawTxResult.vin[i].txid); - } - } - - coreApi.getRawTransactions(txids).then(function(txInputs) { - res.locals.result.txInputs = txInputs; - - resolve(); - }); + resolve(); }); })); Promise.all(promises).then(function() { res.render("transaction"); - next(); - - }).catch(function(err) { - res.locals.pageErrors.push(utils.logError("1237y4ewssgt", err)); - - res.render("transaction"); - next(); }); - }).catch(function(err) { res.locals.userMessage = "Failed to load transaction with txid=" + txid + ": " + err; + + res.locals.pageErrors.push(utils.logError("1237y4ewssgt", err)); res.render("transaction"); @@ -1053,12 +1041,12 @@ router.get("/address/:address", function(req, res, next) { var vinJ = tx.vin[j]; if (txInput != null) { - if (txInput.vout[vinJ.vout] && txInput.vout[vinJ.vout].scriptPubKey && txInput.vout[vinJ.vout].scriptPubKey.addresses && txInput.vout[vinJ.vout].scriptPubKey.addresses.includes(address)) { + if (txInput && txInput.scriptPubKey && txInput.scriptPubKey.addresses && txInput.scriptPubKey.addresses.includes(address)) { if (addrLossesByTx[tx.txid] == null) { addrLossesByTx[tx.txid] = new Decimal(0); } - addrLossesByTx[tx.txid] = addrLossesByTx[tx.txid].plus(new Decimal(txInput.vout[vinJ.vout].value)); + addrLossesByTx[tx.txid] = addrLossesByTx[tx.txid].plus(new Decimal(txInput.value)); } } } @@ -1448,6 +1436,14 @@ router.get("/tools", function(req, res, next) { next(); }); +router.get("/admin", function(req, res, next) { + res.locals.memstats = v8.getHeapStatistics(); + + res.render("admin"); + + next(); +}); + router.get("/changelog", function(req, res, next) { res.locals.changelogHtml = marked(global.changelogMarkdown); diff --git a/views/admin.pug b/views/admin.pug new file mode 100644 index 0000000..f252d8a --- /dev/null +++ b/views/admin.pug @@ -0,0 +1,11 @@ +extends layout + +block headContent + title Admin + +block content + h1.h3 Admin + hr + + pre + code.json #{JSON.stringify(memstats, null, 4)} \ No newline at end of file diff --git a/views/includes/index-network-summary.pug b/views/includes/index-network-summary.pug index 2dc0b99..53a7451 100644 --- a/views/includes/index-network-summary.pug +++ b/views/includes/index-network-summary.pug @@ -89,8 +89,7 @@ div.row.index-summary span Total Txs td.text-right.text-monospace if (txStats && txStats.totalTxCount) - - var totalTxData = utils.formatLargeNumber(txStats.totalTxCount, 2); - span.border-dotted(title=`${txStats.totalTxCount.toLocaleString()}`, data-toggle="tooltip") #{totalTxData[0]} #{totalTxData[1].abbreviation} + span #{txStats.totalTxCount.toLocaleString()} else span ??? diff --git a/views/includes/transaction-io-details.pug b/views/includes/transaction-io-details.pug index b7347aa..9de1964 100644 --- a/views/includes/transaction-io-details.pug +++ b/views/includes/transaction-io-details.pug @@ -22,8 +22,7 @@ div.row.text-monospace - var vout = null; if (txInputs && txInputs[txVinIndex]) - var txInput = txInputs[txVinIndex]; - if (txInput.vout && txInput.vout[txVin.vout]) - - var vout = txInput.vout[txVin.vout]; + - var vout = txInput; if (txVin.coinbase || vout) div.clearfix diff --git a/views/transaction.pug b/views/transaction.pug index 027dd84..fb4ef35 100644 --- a/views/transaction.pug +++ b/views/transaction.pug @@ -30,7 +30,7 @@ block content - totalInputValue = totalInputValue.plus(new Decimal(coinConfig.blockRewardFunction(result.getblock.height, global.activeBlockchain))); each txInput, txInputIndex in result.txInputs if (txInput) - - var vout = txInput.vout[result.getrawtransaction.vin[txInputIndex].vout]; + - var vout = txInput; if (vout.value) - totalInputValue = totalInputValue.plus(new Decimal(vout.value));