diff --git a/.env-sample b/.env-sample index 3acd3de..c1cd452 100644 --- a/.env-sample +++ b/.env-sample @@ -26,6 +26,12 @@ #BTCEXP_INFLUXDB_USER=dbuser #BTCEXP_INFLUXDB_PASS=dbpassword +# Disable app's in-memory RPC caching to reduce memory usage +#BTCEXP_NO_INMEMORY_RPC_CACHE=true + +# Optional redis server for RPC caching +#BTCEXP_REDIS_URL=redis://localhost:6379 + #BTCEXP_COOKIE_SECRET=0000aaaafffffgggggg # Whether public-demo aspects of the site are active diff --git a/app/api/coreApi.js b/app/api/coreApi.js index 30f9b3e..2b3348f 100644 --- a/app/api/coreApi.js +++ b/app/api/coreApi.js @@ -1,4 +1,6 @@ var debug = require("debug")("coreApi"); +var redis = require("redis"); +var bluebird = require("bluebird"); var LRU = require("lru-cache"); var fs = require('fs'); @@ -11,9 +13,95 @@ var coins = require("../coins.js"); var rpcApi = require("./rpcApi.js"); //var rpcApi = require("./mockApi.js"); -var miscCache = LRU(50); -var blockCache = LRU(50); -var txCache = LRU(200); +var redisClient = null; +if (config.redisUrl) { + bluebird.promisifyAll(redis.RedisClient.prototype); + + redisClient = redis.createClient({url:config.redisUrl}); +} + +function onCacheEvent(cacheType, hitOrMiss, cacheKey) { + //console.log(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`); +} + +function createMemoryLruCache(cacheObj) { + return { + get:function(key) { + return new Promise(function(resolve, reject) { + var val = cacheObj.get(key); + + if (val != null) { + onCacheEvent("memory", "hit", key); + + } else { + onCacheEvent("memory", "miss", key); + } + + resolve(cacheObj.get(key)); + }); + }, + set:function(key, obj, maxAge) { cacheObj.set(key, obj, maxAge); } + } +} + +var noopCache = { + get:function(key) { + return new Promise(function(resolve, reject) { + resolve(null); + }); + }, + set:function(key, obj, maxAge) {} +}; + +var miscCache = null; +var blockCache = null; +var txCache = null; + +if (config.noInmemoryRpcCache) { + miscCache = noopCache; + blockCache = noopCache; + txCache = noopCache; + +} else { + miscCache = createMemoryLruCache(LRU(50)); + blockCache = createMemoryLruCache(LRU(50)); + txCache = createMemoryLruCache(LRU(200)); +} + +if (redisClient) { + var redisCache = { + get:function(key) { + return new Promise(function(resolve, reject) { + redisClient.getAsync(key).then(function(result) { + if (result == null) { + onCacheEvent("redis", "miss", key); + + resolve(result); + + return; + } + + onCacheEvent("redis", "hit", key); + + resolve(JSON.parse(result)); + + }).catch(function(err) { + console.log(`Error 328rhwefghsdgsdss: ${err}`); + + reject(err); + }); + }); + }, + set:function(key, obj, maxAge) { + redisClient.set(key, JSON.stringify(obj), "PX", maxAge); + } + }; + + miscCache = redisCache; + blockCache = redisCache; + txCache = redisCache; +} + @@ -36,27 +124,44 @@ function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheC } return new Promise(function(resolve, reject) { - var result = cache.get(cacheKey); - if (result) { - resolve(result); + var cacheResult = null; - } else { - rpcApiFunction().then(function(result) { - if (result != null && cacheConditionFunction(result)) { - cache.set(cacheKey, result, cacheMaxAge); - } + cache.get(cacheKey).then(function(result) { + cacheResult = result; + + }).catch(function(err) { + console.log(`Error nds9fc2eg621tf3: key=${cacheKey}, err=${err}`); - resolve(result); + }).finally(function() { + if (cacheResult != null) { + resolve(cacheResult); - }).catch(function(err) { - reject(err); - }); - } + } else { + rpcApiFunction().then(function(rpcResult) { + if (rpcResult != null && cacheConditionFunction(rpcResult)) { + cache.set(cacheKey, rpcResult, cacheMaxAge); + } + + resolve(rpcResult); + + }).catch(function(err) { + reject(err); + }); + } + }); }); } function shouldCacheTransaction(tx) { - return (tx.confirmations > 0); + if (tx.confirmations < 1) { + return false; + } + + if (tx.vin.length > 9) { + return false; + } + + return true; } @@ -78,7 +183,7 @@ function getMempoolInfo() { } function getMiningInfo() { - return tryCacheThenRpcApi(miscCache, "getMiningInfo", 1000, rpcApi.getMiningInfo); + return tryCacheThenRpcApi(miscCache, "getMiningInfo", 30000, rpcApi.getMiningInfo); } function getUptimeSeconds() { @@ -468,7 +573,7 @@ function getMempoolStats() { debug(JSON.stringify(sizeBucketLabels));*/ resolve(summary); - + }).catch(function(err) { reject(err); }); @@ -482,51 +587,18 @@ function getBlockByHeight(blockHeight) { } function getBlocksByHeight(blockHeights) { - var blockHeightsNotInCache = []; - var blocksByIndex = {}; - - for (var i = 0; i < blockHeights.length; i++) { - var blockI = blockCache.get("getBlockByHeight-" + blockHeights[i]); - if (blockI == null) { - blockHeightsNotInCache.push(blockHeights[i]); - - } else { - blocksByIndex[i] = blockI; - } - } - return new Promise(function(resolve, reject) { - var combinedBlocks = []; - if (blockHeightsNotInCache.length > 0) { - rpcApi.getBlocksByHeight(blockHeightsNotInCache).then(function(queriedBlocks) { - var queriedBlocksCurrentIndex = 0; - for (var i = 0; i < blockHeights.length; i++) { - if (blocksByIndex.hasOwnProperty(i)) { - combinedBlocks.push(blocksByIndex[i]); - - } else { - var queriedBlock = queriedBlocks[queriedBlocksCurrentIndex]; - - combinedBlocks.push(queriedBlock); - - blockCache.set("getBlockByHeight-" + queriedBlock.height, queriedBlock, 3600000); - - queriedBlocksCurrentIndex++; - } - } - - resolve(combinedBlocks); + var promises = []; + for (var i = 0; i < blockHeights.length; i++) { + promises.push(getBlockByHeight(blockHeights[i])); + } - }).catch(function(err) { - console.log("Error 39g2rfyewgf: " + err); - }); - } else { - for (var i = 0; i < blockHeights.length; i++) { - combinedBlocks.push(blocksByIndex[i]); - } + Promise.all(promises).then(function(results) { + resolve(results); - resolve(combinedBlocks); - } + }).catch(function(err) { + reject(err); + }); }); } @@ -537,48 +609,18 @@ function getBlockByHash(blockHash) { } function getBlocksByHash(blockHashes) { - var blockHashesNotInCache = []; - var blocksByIndex = {}; - - for (var i = 0; i < blockHashes.length; i++) { - var blockI = blockCache.get("getBlockByHash-" + blockHashes[i]); - if (blockI == null) { - blockHashesNotInCache.push(blockHashes[i]); - - } else { - blocksByIndex[i] = blockI; - } - } - return new Promise(function(resolve, reject) { - var combinedBlocks = []; - if (blockHashesNotInCache.length > 0) { - rpcApi.getBlocksByHash(blockHashesNotInCache).then(function(queriedBlocks) { - var queriedBlocksCurrentIndex = 0; - for (var i = 0; i < blockHashes.length; i++) { - if (blocksByIndex.hasOwnProperty(i)) { - combinedBlocks.push(blocksByIndex[i]); - - } else { - var queriedBlock = queriedBlocks[queriedBlocksCurrentIndex]; - - combinedBlocks.push(queriedBlock); - - blockCache.set("getBlockByHash-" + queriedBlock.hash, queriedBlock, 3600000); - - queriedBlocksCurrentIndex++; - } - } + var promises = []; + for (var i = 0; i < blockHashes.length; i++) { + promises.push(getBlockByHash(blockHashes[i])); + } - resolve(combinedBlocks); - }); - } else { - for (var i = 0; i < blockHeights.length; i++) { - combinedBlocks.push(blocksByIndex[i]); - } + Promise.all(promises).then(function(results) { + resolve(results); - resolve(combinedBlocks); - } + }).catch(function(err) { + reject(err); + }); }); } @@ -597,51 +639,18 @@ function getAddress(address) { } function getRawTransactions(txids) { - var txidsNotInCache = []; - var txsByIndex = {}; - - for (var i = 0; i < txids.length; i++) { - var txI = txCache.get("getRawTransaction-" + txids[i]); - if (txI == null) { - txidsNotInCache.push(txids[i]); - - } else { - txsByIndex[i] = txI; - } - } - return new Promise(function(resolve, reject) { - var combinedTxs = []; - if (txidsNotInCache.length > 0) { - rpcApi.getRawTransactions(txidsNotInCache).then(function(queriedTxs) { - var queriedTxsCurrentIndex = 0; - for (var i = 0; i < txids.length; i++) { - if (txsByIndex.hasOwnProperty(i)) { - combinedTxs.push(txsByIndex[i]); - - } else { - var queriedTx = queriedTxs[queriedTxsCurrentIndex]; - if (queriedTx != null) { - combinedTxs.push(queriedTx); - - if (shouldCacheTransaction(queriedTx)) { - txCache.set("getRawTransaction-" + queriedTx.txid, queriedTx, 3600000); - } - } + var promises = []; + for (var i = 0; i < txids.length; i++) { + promises.push(getRawTransaction(txids[i])); + } - queriedTxsCurrentIndex++; - } - } + Promise.all(promises).then(function(results) { + resolve(results); - resolve(combinedTxs); - }); - } else { - for (var i = 0; i < txids.length; i++) { - combinedTxs.push(txsByIndex[i]); - } - - resolve(combinedTxs); - } + }).catch(function(err) { + reject(err); + }); }); } diff --git a/app/config.js b/app/config.js index 4e51ee3..80c68a0 100644 --- a/app/config.js +++ b/app/config.js @@ -33,7 +33,7 @@ for (var i = 0; i < electrumXServerUriStrings.length; i++) { electrumXServers.push({protocol:uri.protocol.substring(0, uri.protocol.length - 1), host:uri.hostname, port:parseInt(uri.port)}); } -["BTCEXP_DEMO", "BTCEXP_PRIVACY_MODE"].forEach(function(item) { +["BTCEXP_DEMO", "BTCEXP_PRIVACY_MODE", "BTCEXP_NO_INMEMORY_RPC_CACHE"].forEach(function(item) { if (process.env[item] === undefined) { process.env[item] = "false"; } @@ -50,6 +50,7 @@ module.exports = { privacyMode: (process.env.BTCEXP_PRIVACY_MODE == "true"), demoSite: (process.env.BTCEXP_DEMO == "true"), queryExchangeRates: (process.env.BTCEXP_NO_RATES != "true"), + noInmemoryRpcCache: (process.env.BTCEXP_NO_INMEMORY_RPC_CACHE.toLowerCase() == "true"), coin: currentCoin, rpcBlacklist: @@ -121,6 +122,8 @@ module.exports = { electrumXServers:electrumXServers, + redisUrl:process.env.BTCEXP_REDIS_URL, + site: { blockTxPageSize:20, addressTxPageSize:20, diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index ed6fc41..5a83a58 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -716,6 +716,11 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.2.0.tgz", "integrity": "sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==" }, + "double-ended-queue": { + "version": "2.1.0-0", + "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", + "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=" + }, "dtrace-provider": { "version": "0.8.7", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.7.tgz", @@ -2204,6 +2209,26 @@ "strip-indent": "^2.0.0" } }, + "redis": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", + "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", + "requires": { + "double-ended-queue": "^2.1.0-0", + "redis-commands": "^1.2.0", + "redis-parser": "^2.6.0" + } + }, + "redis-commands": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.4.0.tgz", + "integrity": "sha512-cu8EF+MtkwI4DLIT0x9P8qNTLFhQD4jLfxLR0cCNkeGzs87FN6879JOJwNQR/1zD7aSYNbU0hgsV9zGY71Itvw==" + }, + "redis-parser": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", + "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" + }, "regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", diff --git a/package.json b/package.json index 2f9906d..71909ce 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "morgan": "^1.9.1", "pug": "2.0.1", "qrcode": "1.2.0", + "redis": "2.8.0", "request": "2.88.0", "serve-favicon": "^2.5.0", "simple-git": "1.92.0"