lukechilds
/
btc-rpc-explorer
mirror of https://github.com/lukechilds/btc-rpc-explorer.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Caching improvements: 

- Cache key prefixing only for redis keys - this data is persistent across app executions, thus the need for versioning, and limiting prefixes to here avoids wasting app memory for in-memory cache-key prefixes
- Tiered caching when both memory and redis caching is active (memory cache tried first, fall back to redis cache, fall back to RPC)
- Cleaner setup/configuration of caches
master
Dan Janosik 5 years ago
parent
ce185d63da
commit
03a29dda77
No known key found for this signature in database GPG Key ID: C6F8CE9FFDB2CED2
3 changed files with 122 additions and 72 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 116
      app/api/coreApi.js
  2. 59
      app/redisCache.js
  3. 19
      app/utils.js

116
app/api/coreApi.js
View File

@ -20,6 +20,7 @@ var rpcApi = require("./rpcApi.js");
// pulling old-format data from a persistent cache
var cacheKeyVersion = "v0";
const ONE_SEC = 1000;
const ONE_MIN = 60 * ONE_SEC;
const ONE_HR = 60 * ONE_MIN;
@ -27,20 +28,13 @@ const ONE_DAY = 24 * ONE_HR;
const ONE_YR = 265 * ONE_DAY;
global.cacheStats.memory = {
hit: 0,
miss: 0
};
function onCacheEvent(cacheType, hitOrMiss, cacheKey) {
global.cacheStats.memory[hitOrMiss]++;
//debugLog(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`);
}
function createMemoryLruCache(cacheObj) {
function createMemoryLruCache(cacheObj, onCacheEvent) {
return {
get:function(key) {
return new Promise(function(resolve, reject) {
onCacheEvent("memory", "try", key);
var val = cacheObj.get(key);
if (val != null) {
@ -57,36 +51,86 @@ function createMemoryLruCache(cacheObj) {
}
}
var noopCache = {
get:function(key) {
return new Promise(function(resolve, reject) {
resolve(null);
});
},
set:function(key, obj, maxAge) {}
};
function tryCache(cacheKey, cacheObjs, index, resolve, reject) {
if (index == cacheObjs.length) {
resolve(null);
return;
}
cacheObjs[index].get(cacheKey).then(function(result) {
if (result != null) {
resolve(result);
} else {
tryCache(cacheKey, cacheObjs, index + 1, resolve, reject);
}
});
}
function createTieredCache(cacheObjs) {
return {
get:function(key) {
return new Promise(function(resolve, reject) {
tryCache(key, cacheObjs, 0, resolve, reject);
});
},
set:function(key, obj, maxAge) {
for (var i = 0; i < cacheObjs.length; i++) {
cacheObjs[i].set(key, obj, maxAge);
}
}
}
}
var miscCache = null;
var blockCache = null;
var txCache = null;
var miscCaches = [];
var blockCaches = [];
var txCaches = [];
if (config.noInmemoryRpcCache) {
miscCache = noopCache;
blockCache = noopCache;
txCache = noopCache;
if (!config.noInmemoryRpcCache) {
global.cacheStats.memory = {
try: 0,
hit: 0,
miss: 0
};
var onMemoryCacheEvent = function(cacheType, eventType, cacheKey) {
global.cacheStats.memory[eventType]++;
//debugLog(`cache.${cacheType}.${eventType}: ${cacheKey}`);
}
} else {
miscCache = createMemoryLruCache(new LRU(2000));
blockCache = createMemoryLruCache(new LRU(2000));
txCache = createMemoryLruCache(new LRU(10000));
miscCaches.push(createMemoryLruCache(new LRU(2000), onMemoryCacheEvent));
blockCaches.push(createMemoryLruCache(new LRU(2000), onMemoryCacheEvent));
txCaches.push(createMemoryLruCache(new LRU(10000), onMemoryCacheEvent));
}
if (redisCache.active) {
miscCache = redisCache;
blockCache = redisCache;
txCache = redisCache;
global.cacheStats.redis = {
try: 0,
hit: 0,
miss: 0,
error: 0
};
var onRedisCacheEvent = function(cacheType, eventType, cacheKey) {
global.cacheStats.redis[eventType]++;
//debugLog(`cache.${cacheType}.${eventType}: ${cacheKey}`);
}
var redisCacheObj = redisCache.createCache(cacheKeyVersion, onRedisCacheEvent);
miscCaches.push(redisCacheObj);
blockCaches.push(redisCacheObj);
txCaches.push(redisCacheObj);
}
var miscCache = createTieredCache(miscCaches);
var blockCache = createTieredCache(blockCaches);
var txCache = createTieredCache(txCaches);
@ -101,8 +145,6 @@ function getGenesisCoinbaseTransactionId() {
function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheConditionFunction) {
var versionedCacheKey = `${cacheKeyVersion}-${cacheKey}`;
//debugLog("tryCache: " + versionedCacheKey + ", " + cacheMaxAge);
if (cacheConditionFunction == null) {
@ -121,7 +163,7 @@ function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheC
} else {
rpcApiFunction().then(function(rpcResult) {
if (rpcResult != null && cacheConditionFunction(rpcResult)) {
cache.set(versionedCacheKey, rpcResult, cacheMaxAge);
cache.set(cacheKey, rpcResult, cacheMaxAge);
}
resolve(rpcResult);
@ -132,13 +174,13 @@ function tryCacheThenRpcApi(cache, cacheKey, cacheMaxAge, rpcApiFunction, cacheC
}
};
cache.get(versionedCacheKey).then(function(result) {
cache.get(cacheKey).then(function(result) {
cacheResult = result;
finallyFunc();
}).catch(function(err) {
utils.logError("nds9fc2eg621tf3", err, {cacheKey:versionedCacheKey});
utils.logError("nds9fc2eg621tf3", err, {cacheKey:cacheKey});
finallyFunc();
});

59
app/redisCache.js
View File

@ -11,46 +11,43 @@ if (config.redisUrl) {
redisClient = redis.createClient({url:config.redisUrl});
}
global.cacheStats.redis = {
hit: 0,
miss: 0
};
function onCacheEvent(cacheType, hitOrMiss, cacheKey) {
global.cacheStats.redis[hitOrMiss]++;
//console.log(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`);
}
function createCache(keyPrefix, onCacheEvent) {
return {
get: function(key) {
var prefixedKey = `${keyPrefix}-${key}`;
var redisCache = {
get:function(key) {
return new Promise(function(resolve, reject) {
redisClient.getAsync(key).then(function(result) {
if (result == null) {
onCacheEvent("redis", "miss", key);
return new Promise(function(resolve, reject) {
onCacheEvent("redis", "try", key);
resolve(null);
redisClient.getAsync(prefixedKey).then(function(result) {
if (result == null) {
onCacheEvent("redis", "miss", key);
return;
}
resolve(null);
onCacheEvent("redis", "hit", key);
} else {
onCacheEvent("redis", "hit", key);
resolve(JSON.parse(result));
resolve(JSON.parse(result));
}
}).catch(function(err) {
onCacheEvent("redis", "error", key);
}).catch(function(err) {
utils.logError("328rhwefghsdgsdss", err);
utils.logError("328rhwefghsdgsdss", err);
reject(err);
reject(err);
});
});
});
},
set:function(key, obj, maxAgeMillis) {
redisClient.set(key, JSON.stringify(obj), "PX", maxAgeMillis);
}
};
},
set: function(key, obj, maxAgeMillis) {
var prefixedKey = `${keyPrefix}-${key}`;
redisClient.set(prefixedKey, JSON.stringify(obj), "PX", maxAgeMillis);
}
};
}
module.exports = {
active: (redisClient != null),
get: redisCache.get,
set: redisCache.set
createCache: createCache
}

19
app/utils.js
View File

@ -29,6 +29,17 @@ var exponentScales = [
];
var ipMemoryCache = {};
var ipRedisCache = null;
if (redisCache.active) {
var onRedisCacheEvent = function(cacheType, eventType, cacheKey) {
global.cacheStats.redis[eventType]++;
//debugLog(`cache.${cacheType}.${eventType}: ${cacheKey}`);
}
ipRedisCache = redisCache.createCache("v0", onRedisCacheEvent);
}
var ipCache = {
get:function(key) {
return new Promise(function(resolve, reject) {
@ -38,8 +49,8 @@ var ipCache = {
return;
}
if (redisCache.active) {
redisCache.get("ip-" + key).then(function(redisResult) {
if (ipRedisCache != null) {
ipRedisCache.get("ip-" + key).then(function(redisResult) {
if (redisResult != null) {
resolve({key:key, value:redisResult});
@ -57,8 +68,8 @@ var ipCache = {
set:function(key, value, expirationMillis) {
ipMemoryCache[key] = value;
if (redisCache.active) {
redisCache.set("ip-" + key, value, expirationMillis);
if (ipRedisCache != null) {
ipRedisCache.set("ip-" + key, value, expirationMillis);
}
}
};

Write Preview
Loading…
Cancel
Save