From 6db2516446e280436376cd67587f7497b0bf1b5f Mon Sep 17 00:00:00 2001 From: Dan Janosik Date: Mon, 11 Mar 2019 14:18:19 -0400 Subject: [PATCH] back to ipstack for geo-loc ips (ip-api blocked too much); redis caching for ips when active; redis-caching code reorg; --- .env-sample | 1 + app/api/coreApi.js | 39 +--------------- app/credentials.js | 2 +- app/redisCache.js | 49 +++++++++++++++++++ app/utils.js | 114 +++++++++++++++++++++++++++++---------------- views/peers.pug | 25 +++++----- 6 files changed, 140 insertions(+), 90 deletions(-) create mode 100644 app/redisCache.js diff --git a/.env-sample b/.env-sample index 98f70bc..0c8eb8e 100644 --- a/.env-sample +++ b/.env-sample @@ -59,6 +59,7 @@ #BTCEXP_GANALYTICS_TRACKING=UA-XXXX-X #BTCEXP_SENTRY_URL=https://00000fffffff@sentry.io/XXXX +#BTCEXP_IPSTACK_APIKEY=000000fffffaaaaa # Optional value for "max_old_space_size", default: 1024 #BTCEXP_OLD_SPACE_MAX_SIZE=2048 \ No newline at end of file diff --git a/app/api/coreApi.js b/app/api/coreApi.js index ff0789a..d24b45d 100644 --- a/app/api/coreApi.js +++ b/app/api/coreApi.js @@ -1,6 +1,4 @@ var debug = require("debug")("btcexp:coreApi"); -var redis = require("redis"); -var bluebird = require("bluebird"); var LRU = require("lru-cache"); var fs = require('fs'); @@ -8,17 +6,12 @@ var fs = require('fs'); var utils = require("../utils.js"); var config = require("../config.js"); var coins = require("../coins.js"); +var redisCache = require("../redisCache.js"); // choose one of the below: RPC to a node, or mock data while testing var rpcApi = require("./rpcApi.js"); //var rpcApi = require("./mockApi.js"); -var redisClient = null; -if (config.redisUrl) { - bluebird.promisifyAll(redis.RedisClient.prototype); - - redisClient = redis.createClient({url:config.redisUrl}); -} function onCacheEvent(cacheType, hitOrMiss, cacheKey) { //console.log(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`); @@ -68,35 +61,7 @@ if (config.noInmemoryRpcCache) { txCache = createMemoryLruCache(LRU(200)); } -if (redisClient) { - var redisCache = { - get:function(key) { - return new Promise(function(resolve, reject) { - redisClient.getAsync(key).then(function(result) { - if (result == null) { - onCacheEvent("redis", "miss", key); - - resolve(result); - - return; - } - - onCacheEvent("redis", "hit", key); - - resolve(JSON.parse(result)); - - }).catch(function(err) { - console.log(`Error 328rhwefghsdgsdss: ${err}`); - - reject(err); - }); - }); - }, - set:function(key, obj, maxAge) { - redisClient.set(key, JSON.stringify(obj), "PX", maxAge); - } - }; - +if (redisCache.active) { miscCache = redisCache; blockCache = redisCache; txCache = redisCache; diff --git a/app/credentials.js b/app/credentials.js index 308c97d..bdc6e03 100644 --- a/app/credentials.js +++ b/app/credentials.js @@ -38,7 +38,7 @@ module.exports = { // to include a map of the estimated locations of your node's // peers // format: "ID_FROM_IPSTACK" - ipStackComApiAccessKey: process.env.BTCEXP_IPSTACK_KEY, + ipStackComApiAccessKey: process.env.BTCEXP_IPSTACK_APIKEY, // optional: GA tracking code // format: "UA-..." diff --git a/app/redisCache.js b/app/redisCache.js new file mode 100644 index 0000000..a5676ae --- /dev/null +++ b/app/redisCache.js @@ -0,0 +1,49 @@ +var redis = require("redis"); +var bluebird = require("bluebird"); + +var config = require("./config.js"); + +var redisClient = null; +if (config.redisUrl) { + bluebird.promisifyAll(redis.RedisClient.prototype); + + redisClient = redis.createClient({url:config.redisUrl}); +} + +function onCacheEvent(cacheType, hitOrMiss, cacheKey) { + //console.log(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`); +} + +var redisCache = { + get:function(key) { + return new Promise(function(resolve, reject) { + redisClient.getAsync(key).then(function(result) { + if (result == null) { + onCacheEvent("redis", "miss", key); + + resolve(null); + + return; + } + + onCacheEvent("redis", "hit", key); + + resolve(JSON.parse(result)); + + }).catch(function(err) { + console.log(`Error 328rhwefghsdgsdss: ${err}`); + + reject(err); + }); + }); + }, + set:function(key, obj, maxAgeMillis) { + redisClient.set(key, JSON.stringify(obj), "PX", maxAgeMillis); + } +}; + +module.exports = { + active: (redisClient != null), + get: redisCache.get, + set: redisCache.set +} \ No newline at end of file diff --git a/app/utils.js b/app/utils.js index 47c57e7..5eee2a1 100644 --- a/app/utils.js +++ b/app/utils.js @@ -5,8 +5,8 @@ var qrcode = require("qrcode"); var config = require("./config.js"); var coins = require("./coins.js"); var coinConfig = coins[config.coin]; +var redisCache = require("./redisCache.js"); -var ipCache = {}; var exponentScales = [ {val:1000000000000000000000000000000000, name:"?", abbreviation:"V", exponent:"33"}, @@ -22,6 +22,43 @@ var exponentScales = [ {val:1000, name:"kilo", abbreviation:"K", exponent:"3"} ]; +var ipMemoryCache = {}; +var ipCache = { + get:function(key) { + return new Promise(function(resolve, reject) { + if (ipMemoryCache[key] != null) { + resolve({key:key, value:ipMemoryCache[key]}); + + return; + } + + if (redisCache.active) { + redisCache.get("ip-" + key).then(function(redisResult) { + if (redisResult != null) { + resolve({key:key, value:redisResult}); + + return; + } + + resolve({key:key, value:null}); + }); + + } else { + resolve({key:key, value:null}); + } + }); + }, + set:function(key, value, expirationMillis) { + ipMemoryCache[key] = value; + + if (redisCache.active) { + redisCache.set("ip-" + key, value, expirationMillis); + } + } +}; + + + function redirectToConnectPageIfNeeded(req, res) { if (!req.session.host) { req.session.redirectUrl = req.originalUrl; @@ -340,8 +377,8 @@ function refreshExchangeRates() { } } -// Uses ip-api.com API -function geoLocateIpAddresses(ipAddresses) { +// Uses ipstack.com API +function geoLocateIpAddresses(ipAddresses, provider) { return new Promise(function(resolve, reject) { if (config.privacyMode) { resolve({}); @@ -349,53 +386,48 @@ function geoLocateIpAddresses(ipAddresses) { return; } - var chunks = splitArrayIntoChunks(ipAddresses, 1); + var ipDetails = {ips:ipAddresses, detailsByIp:{}}; var promises = []; - for (var i = 0; i < chunks.length; i++) { - var ipStr = ""; - for (var j = 0; j < chunks[i].length; j++) { - if (j > 0) { - ipStr = ipStr + ","; - } - - ipStr = ipStr + chunks[i][j]; - } + for (var i = 0; i < ipAddresses.length; i++) { + var ipStr = ipAddresses[i]; + + promises.push(new Promise(function(resolve2, reject2) { + ipCache.get(ipStr).then(function(result) { + if (result.value == null) { + var apiUrl = "http://api.ipstack.com/" + result.key + "?access_key=" + config.credentials.ipStackComApiAccessKey; + + console.log("Requesting IP-geo: " + apiUrl); + + request(apiUrl, function(error, response, body) { + if (error) { + reject2(error); + + } else { + resolve2({needToProcess:true, response:response}); + } + }); - if (ipCache[ipStr] != null) { - promises.push(new Promise(function(resolve2, reject2) { - resolve2(ipCache[ipStr]); - })); + } else { + ipDetails.detailsByIp[result.key] = result.value; - } else { - var apiUrl = "http://ip-api.com/json/" + ipStr; - promises.push(new Promise(function(resolve2, reject2) { - request(apiUrl, function(error, response, body) { - if (error) { - reject2(error); - - } else { - resolve2(response); - } - }); - })); - } + resolve2({needToProcess:false}); + } + }); + })); } Promise.all(promises).then(function(results) { - var ipDetails = {ips:[], detailsByIp:{}}; - for (var i = 0; i < results.length; i++) { - var res = results[i]; - if (res != null && res["statusCode"] == 200) { - var resBody = JSON.parse(res["body"]); - var ip = resBody["query"]; + if (results[i].needToProcess) { + var res = results[i].response; + if (res != null && res["statusCode"] == 200) { + var resBody = JSON.parse(res["body"]); + var ip = resBody["ip"]; - ipDetails.ips.push(ip); - ipDetails.detailsByIp[ip] = resBody; + ipDetails.detailsByIp[ip] = resBody; - if (ipCache[ip] == null) { - ipCache[ip] = res; + ipCache.set(ip, resBody, 1000 * 60 * 60 * 24 * 365); } } } @@ -403,6 +435,8 @@ function geoLocateIpAddresses(ipAddresses) { resolve(ipDetails); }).catch(function(err) { + console.log("Error 80342hrf78wgehdf07gds: " + err); + reject(err); }); }); diff --git a/views/peers.pug b/views/peers.pug index a18adbf..3909d5f 100644 --- a/views/peers.pug +++ b/views/peers.pug @@ -115,12 +115,13 @@ block content - var ipAddr = item.addr.substring(0, item.addr.lastIndexOf(":")); if (peerIpSummary.ips && peerIpSummary.ips.includes(ipAddr)) - var ipDetails = peerIpSummary.detailsByIp[ipAddr]; - if (ipDetails.city) - span #{ipDetails.city}, - if (ipDetails.region) - span #{ipDetails.region}, - if (ipDetails.country) - span #{ipDetails.country} + if (ipDetails) + if (ipDetails.city) + span #{ipDetails.city}, + if (ipDetails.region_name) + span #{ipDetails.region_name}, + if (ipDetails.country_name) + span #{ipDetails.country_name} else span ? @@ -182,18 +183,18 @@ block endOfBody each ipAddress, index in peerIpSummary.ips - var ipDetails = peerIpSummary.detailsByIp[ipAddress]; - if (ipDetails && ipDetails.lat && ipDetails.lon) + if (ipDetails && ipDetails.latitude && ipDetails.longitude) - var ipDetailsPopupHtml = "" + ipAddress + "
"; if (ipDetails.city) - var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.city + ", "; - if (ipDetails.region) - - var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.region + ", "; + if (ipDetails.region_name) + - var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.region_name + ", "; - if (ipDetails.country) - - var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.countryCode + " "; + if (ipDetails.country_name) + - var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.country_name + " "; - script L.marker([#{ipDetails.lat}, #{ipDetails.lon}]).addTo(mymap).bindPopup("!{ipDetailsPopupHtml}"); + script L.marker([#{ipDetails.latitude}, #{ipDetails.longitude}]).addTo(mymap).bindPopup("!{ipDetailsPopupHtml}");