lukechilds
/
btc-rpc-explorer
mirror of https://github.com/lukechilds/btc-rpc-explorer.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

overhaul of RPC caching logic 

- support for redis RPC caching
- support for disabling in-memory RPC caching
- skip caching many-input transactions
- cleanup/simplify caching logic for bulk functions (getBlocksByHeight, getBlocksByHash, getRawTransactions)
fix-133-memory-crash
Dan Janosik 6 years ago
parent
3d31bc01ba
commit
f586e81585
No known key found for this signature in database GPG Key ID: C6F8CE9FFDB2CED2
5 changed files with 187 additions and 143 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      .env-sample
  2. 293
      app/api/coreApi.js
  3. 5
      app/config.js
  4. 25
      npm-shrinkwrap.json
  5. 1
      package.json

6
.env-sample
View File

@ -26,6 +26,12 @@
#BTCEXP_INFLUXDB_USER=dbuser
#BTCEXP_INFLUXDB_PASS=dbpassword
# Disable app's in-memory RPC caching to reduce memory usage
#BTCEXP_NO_INMEMORY_RPC_CACHE=true
# Optional redis server for RPC caching
#BTCEXP_REDIS_URL=redis://localhost:6379
#BTCEXP_COOKIE_SECRET=0000aaaafffffgggggg
# Whether public-demo aspects of the site are active

293
app/api/coreApi.js
View File

@ -1,4 +1,6 @@
var debug = require("debug")("coreApi");
var redis = require("redis");
var bluebird = require("bluebird");
var LRU = require("lru-cache");
var fs = require('fs');
@ -11,9 +13,95 @@ var coins = require("../coins.js");
var rpcApi = require("./rpcApi.js");
//var rpcApi = require("./mockApi.js");
var miscCache = LRU(50);
var blockCache = LRU(50);
var txCache = LRU(200);
var redisClient = null;
if (config.redisUrl) {
bluebird.promisifyAll(redis.RedisClient.prototype);
redisClient = redis.createClient({url:config.redisUrl});
}
function onCacheEvent(cacheType, hitOrMiss, cacheKey) {
//console.log(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`);
}
function createMemoryLruCache(cacheObj) {
return {
get:function(key) {
return new Promise(function(resolve, reject) {
var val = cacheObj.get(key);
if (val != null) {
onCacheEvent("memory", "hit", key);
} else {
onCacheEvent("memory", "miss", key);
}
resolve(cacheObj.get(key));
});
},
set:function(key, obj, maxAge) { cacheObj.set(key, obj, maxAge); }
}
}
var noopCache = {
get:function(key) {
return new Promise(function(resolve, reject) {
resolve(null);
});
},
set:function(key, obj, maxAge) {}
};
var miscCache = null;
var blockCache = null;
var txCache = null;
if (config.noInmemoryRpcCache) {
miscCache = noopCache;
blockCache = noopCache;
txCache = noopCache;
} else {
miscCache = createMemoryLruCache(LRU(50));
blockCache = createMemoryLruCache(LRU(50));
txCache = createMemoryLruCache(LRU(200));
}
if (redisClient) {
var redisCache = {
get:function(key) {
return new Promise(function(resolve, reject) {
redisClient.getAsync(key).then(function(result) {
if (result == null) {
onCacheEvent("redis", "miss", key);
resolve(result);
return;
}
onCacheEvent("redis", "hit", key);
resolve(JSON.parse(result));
}).catch(function(err) {
console.log(`Error 328rhwefghsdgsdss: ${err}`);
reject(err);
});
});
},
set:function(key, obj, maxAge) {
redisClient.set(key, JSON.stringify(obj), "PX", maxAge);
}
};
miscCache = redisCache;
blockCache = redisCache;
txCache = redisCache;
}
@ -36,27 +124,44 @@ function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheC
}
return new Promise(function(resolve, reject) {
var result = cache.get(cacheKey);
if (result) {
resolve(result);
var cacheResult = null;
} else {
rpcApiFunction().then(function(result) {
if (result != null && cacheConditionFunction(result)) {
cache.set(cacheKey, result, cacheMaxAge);
}
cache.get(cacheKey).then(function(result) {
cacheResult = result;
}).catch(function(err) {
console.log(`Error nds9fc2eg621tf3: key=${cacheKey}, err=${err}`);
resolve(result);
}).finally(function() {
if (cacheResult != null) {
resolve(cacheResult);
}).catch(function(err) {
reject(err);
});
}
} else {
rpcApiFunction().then(function(rpcResult) {
if (rpcResult != null && cacheConditionFunction(rpcResult)) {
cache.set(cacheKey, rpcResult, cacheMaxAge);
}
resolve(rpcResult);
}).catch(function(err) {
reject(err);
});
}
});
});
}
function shouldCacheTransaction(tx) {
return (tx.confirmations > 0);
if (tx.confirmations < 1) {
return false;
}
if (tx.vin.length > 9) {
return false;
}
return true;
}
@ -78,7 +183,7 @@ function getMempoolInfo() {
}
function getMiningInfo() {
return tryCacheThenRpcApi(miscCache, "getMiningInfo", 1000, rpcApi.getMiningInfo);
return tryCacheThenRpcApi(miscCache, "getMiningInfo", 30000, rpcApi.getMiningInfo);
}
function getUptimeSeconds() {
@ -468,7 +573,7 @@ function getMempoolStats() {
debug(JSON.stringify(sizeBucketLabels));*/
resolve(summary);
}).catch(function(err) {
reject(err);
});
@ -482,51 +587,18 @@ function getBlockByHeight(blockHeight) {
}
function getBlocksByHeight(blockHeights) {
var blockHeightsNotInCache = [];
var blocksByIndex = {};
for (var i = 0; i < blockHeights.length; i++) {
var blockI = blockCache.get("getBlockByHeight-" + blockHeights[i]);
if (blockI == null) {
blockHeightsNotInCache.push(blockHeights[i]);
} else {
blocksByIndex[i] = blockI;
}
}
return new Promise(function(resolve, reject) {
var combinedBlocks = [];
if (blockHeightsNotInCache.length > 0) {
rpcApi.getBlocksByHeight(blockHeightsNotInCache).then(function(queriedBlocks) {
var queriedBlocksCurrentIndex = 0;
for (var i = 0; i < blockHeights.length; i++) {
if (blocksByIndex.hasOwnProperty(i)) {
combinedBlocks.push(blocksByIndex[i]);
} else {
var queriedBlock = queriedBlocks[queriedBlocksCurrentIndex];
combinedBlocks.push(queriedBlock);
blockCache.set("getBlockByHeight-" + queriedBlock.height, queriedBlock, 3600000);
queriedBlocksCurrentIndex++;
}
}
resolve(combinedBlocks);
var promises = [];
for (var i = 0; i < blockHeights.length; i++) {
promises.push(getBlockByHeight(blockHeights[i]));
}
}).catch(function(err) {
console.log("Error 39g2rfyewgf: " + err);
});
} else {
for (var i = 0; i < blockHeights.length; i++) {
combinedBlocks.push(blocksByIndex[i]);
}
Promise.all(promises).then(function(results) {
resolve(results);
resolve(combinedBlocks);
}
}).catch(function(err) {
reject(err);
});
});
}
@ -537,48 +609,18 @@ function getBlockByHash(blockHash) {
}
function getBlocksByHash(blockHashes) {
var blockHashesNotInCache = [];
var blocksByIndex = {};
for (var i = 0; i < blockHashes.length; i++) {
var blockI = blockCache.get("getBlockByHash-" + blockHashes[i]);
if (blockI == null) {
blockHashesNotInCache.push(blockHashes[i]);
} else {
blocksByIndex[i] = blockI;
}
}
return new Promise(function(resolve, reject) {
var combinedBlocks = [];
if (blockHashesNotInCache.length > 0) {
rpcApi.getBlocksByHash(blockHashesNotInCache).then(function(queriedBlocks) {
var queriedBlocksCurrentIndex = 0;
for (var i = 0; i < blockHashes.length; i++) {
if (blocksByIndex.hasOwnProperty(i)) {
combinedBlocks.push(blocksByIndex[i]);
} else {
var queriedBlock = queriedBlocks[queriedBlocksCurrentIndex];
combinedBlocks.push(queriedBlock);
blockCache.set("getBlockByHash-" + queriedBlock.hash, queriedBlock, 3600000);
queriedBlocksCurrentIndex++;
}
}
var promises = [];
for (var i = 0; i < blockHashes.length; i++) {
promises.push(getBlockByHash(blockHashes[i]));
}
resolve(combinedBlocks);
});
} else {
for (var i = 0; i < blockHeights.length; i++) {
combinedBlocks.push(blocksByIndex[i]);
}
Promise.all(promises).then(function(results) {
resolve(results);
resolve(combinedBlocks);
}
}).catch(function(err) {
reject(err);
});
});
}
@ -597,51 +639,18 @@ function getAddress(address) {
}
function getRawTransactions(txids) {
var txidsNotInCache = [];
var txsByIndex = {};
for (var i = 0; i < txids.length; i++) {
var txI = txCache.get("getRawTransaction-" + txids[i]);
if (txI == null) {
txidsNotInCache.push(txids[i]);
} else {
txsByIndex[i] = txI;
}
}
return new Promise(function(resolve, reject) {
var combinedTxs = [];
if (txidsNotInCache.length > 0) {
rpcApi.getRawTransactions(txidsNotInCache).then(function(queriedTxs) {
var queriedTxsCurrentIndex = 0;
for (var i = 0; i < txids.length; i++) {
if (txsByIndex.hasOwnProperty(i)) {
combinedTxs.push(txsByIndex[i]);
} else {
var queriedTx = queriedTxs[queriedTxsCurrentIndex];
if (queriedTx != null) {
combinedTxs.push(queriedTx);
if (shouldCacheTransaction(queriedTx)) {
txCache.set("getRawTransaction-" + queriedTx.txid, queriedTx, 3600000);
}
}
var promises = [];
for (var i = 0; i < txids.length; i++) {
promises.push(getRawTransaction(txids[i]));
}
queriedTxsCurrentIndex++;
}
}
Promise.all(promises).then(function(results) {
resolve(results);
resolve(combinedTxs);
});
} else {
for (var i = 0; i < txids.length; i++) {
combinedTxs.push(txsByIndex[i]);
}
resolve(combinedTxs);
}
}).catch(function(err) {
reject(err);
});
});
}

5
app/config.js
View File

@ -33,7 +33,7 @@ for (var i = 0; i < electrumXServerUriStrings.length; i++) {
electrumXServers.push({protocol:uri.protocol.substring(0, uri.protocol.length - 1), host:uri.hostname, port:parseInt(uri.port)});
}
["BTCEXP_DEMO", "BTCEXP_PRIVACY_MODE"].forEach(function(item) {
["BTCEXP_DEMO", "BTCEXP_PRIVACY_MODE", "BTCEXP_NO_INMEMORY_RPC_CACHE"].forEach(function(item) {
if (process.env[item] === undefined) {
process.env[item] = "false";
}
@ -50,6 +50,7 @@ module.exports = {
privacyMode: (process.env.BTCEXP_PRIVACY_MODE == "true"),
demoSite: (process.env.BTCEXP_DEMO == "true"),
queryExchangeRates: (process.env.BTCEXP_NO_RATES != "true"),
noInmemoryRpcCache: (process.env.BTCEXP_NO_INMEMORY_RPC_CACHE.toLowerCase() == "true"),
coin: currentCoin,
rpcBlacklist:
@ -121,6 +122,8 @@ module.exports = {
electrumXServers:electrumXServers,
redisUrl:process.env.BTCEXP_REDIS_URL,
site: {
blockTxPageSize:20,
addressTxPageSize:20,

25
npm-shrinkwrap.json
View File

@ -716,6 +716,11 @@
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.2.0.tgz",
"integrity": "sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w=="
},
"double-ended-queue": {
"version": "2.1.0-0",
"resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz",
"integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw="
},
"dtrace-provider": {
"version": "0.8.7",
"resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.7.tgz",
@ -2204,6 +2209,26 @@
"strip-indent": "^2.0.0"
}
},
"redis": {
"version": "2.8.0",
"resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz",
"integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==",
"requires": {
"double-ended-queue": "^2.1.0-0",
"redis-commands": "^1.2.0",
"redis-parser": "^2.6.0"
}
},
"redis-commands": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.4.0.tgz",
"integrity": "sha512-cu8EF+MtkwI4DLIT0x9P8qNTLFhQD4jLfxLR0cCNkeGzs87FN6879JOJwNQR/1zD7aSYNbU0hgsV9zGY71Itvw=="
},
"redis-parser": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz",
"integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs="
},
"regenerator-runtime": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",

1
package.json
View File

@ -45,6 +45,7 @@
"morgan": "^1.9.1",
"pug": "2.0.1",
"qrcode": "1.2.0",
"redis": "2.8.0",
"request": "2.88.0",
"serve-favicon": "^2.5.0",
"simple-git": "1.92.0"

Write Preview
Loading…
Cancel
Save