back to ipstack for geo-loc ips (ip-api blocked too much); redis caching for ips when active; redis-caching code reorg;

.env-sample

@@ -59,6 +59,7 @@
 
 #BTCEXP_GANALYTICS_TRACKING=UA-XXXX-X
 #BTCEXP_SENTRY_URL=https://00000fffffff@sentry.io/XXXX
+#BTCEXP_IPSTACK_APIKEY=000000fffffaaaaa
 
 # Optional value for "max_old_space_size", default: 1024
 #BTCEXP_OLD_SPACE_MAX_SIZE=2048

app/api/coreApi.js

@@ -1,6 +1,4 @@
 var debug = require("debug")("btcexp:coreApi");
-var redis = require("redis");
-var bluebird = require("bluebird");
 
 var LRU = require("lru-cache");
 var fs = require('fs');
@@ -8,17 +6,12 @@ var fs = require('fs');
 var utils = require("../utils.js");
 var config = require("../config.js");
 var coins = require("../coins.js");
+var redisCache = require("../redisCache.js");
 
 // choose one of the below: RPC to a node, or mock data while testing
 var rpcApi = require("./rpcApi.js");
 //var rpcApi = require("./mockApi.js");
 
-var redisClient = null;
-if (config.redisUrl) {
-	bluebird.promisifyAll(redis.RedisClient.prototype);
-
-	redisClient = redis.createClient({url:config.redisUrl});
-}
 
 function onCacheEvent(cacheType, hitOrMiss, cacheKey) {
 	//console.log(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`);
@@ -68,35 +61,7 @@ if (config.noInmemoryRpcCache) {
 	txCache = createMemoryLruCache(LRU(200));
 }
 
-if (redisClient) {
-	var redisCache = {
-		get:function(key) {
-			return new Promise(function(resolve, reject) {
-				redisClient.getAsync(key).then(function(result) {
-					if (result == null) {
-						onCacheEvent("redis", "miss", key);
-
-						resolve(result);
-
-						return;
-					}
-
-					onCacheEvent("redis", "hit", key);
-
-					resolve(JSON.parse(result));
-
-				}).catch(function(err) {
-					console.log(`Error 328rhwefghsdgsdss: ${err}`);
-
-					reject(err);
-				});
-			});
-		},
-		set:function(key, obj, maxAge) {
-			redisClient.set(key, JSON.stringify(obj), "PX", maxAge);
-		}
-	};
-
+if (redisCache.active) {
 	miscCache = redisCache;
 	blockCache = redisCache;
 	txCache = redisCache;

app/credentials.js

@@ -38,7 +38,7 @@ module.exports = {
 	// to include a map of the estimated locations of your node's
 	// peers
 	// format: "ID_FROM_IPSTACK"
-	ipStackComApiAccessKey: process.env.BTCEXP_IPSTACK_KEY,
+	ipStackComApiAccessKey: process.env.BTCEXP_IPSTACK_APIKEY,
 
 	// optional: GA tracking code
 	// format: "UA-..."

app/redisCache.js

@@ -0,0 +1,49 @@
+var redis = require("redis");
+var bluebird = require("bluebird");
+
+var config = require("./config.js");
+
+var redisClient = null;
+if (config.redisUrl) {
+	bluebird.promisifyAll(redis.RedisClient.prototype);
+
+	redisClient = redis.createClient({url:config.redisUrl});
+}
+
+function onCacheEvent(cacheType, hitOrMiss, cacheKey) {
+	//console.log(`cache.${cacheType}.${hitOrMiss}: ${cacheKey}`);
+}
+
+var redisCache = {
+	get:function(key) {
+		return new Promise(function(resolve, reject) {
+			redisClient.getAsync(key).then(function(result) {
+				if (result == null) {
+					onCacheEvent("redis", "miss", key);
+
+					resolve(null);
+
+					return;
+				}
+
+				onCacheEvent("redis", "hit", key);
+
+				resolve(JSON.parse(result));
+
+			}).catch(function(err) {
+				console.log(`Error 328rhwefghsdgsdss: ${err}`);
+
+				reject(err);
+			});
+		});
+	},
+	set:function(key, obj, maxAgeMillis) {
+		redisClient.set(key, JSON.stringify(obj), "PX", maxAgeMillis);
+	}
+};
+
+module.exports = {
+	active: (redisClient != null),
+	get: redisCache.get,
+	set: redisCache.set
+}

app/utils.js

@@ -5,8 +5,8 @@ var qrcode = require("qrcode");
 var config = require("./config.js");
 var coins = require("./coins.js");
 var coinConfig = coins[config.coin];
+var redisCache = require("./redisCache.js");
 
-var ipCache = {};
 
 var exponentScales = [
 	{val:1000000000000000000000000000000000, name:"?", abbreviation:"V", exponent:"33"},
@@ -22,6 +22,43 @@ var exponentScales = [
 	{val:1000, name:"kilo", abbreviation:"K", exponent:"3"}
 ];
 
+var ipMemoryCache = {};
+var ipCache = {
+	get:function(key) {
+		return new Promise(function(resolve, reject) {
+			if (ipMemoryCache[key] != null) {
+				resolve({key:key, value:ipMemoryCache[key]});
+
+				return;
+			}
+
+			if (redisCache.active) {
+				redisCache.get("ip-" + key).then(function(redisResult) {
+					if (redisResult != null) {
+						resolve({key:key, value:redisResult});
+
+						return;
+					}
+
+					resolve({key:key, value:null});
+				});
+
+			} else {
+				resolve({key:key, value:null});
+			}
+		});
+	},
+	set:function(key, value, expirationMillis) {
+		ipMemoryCache[key] = value;
+
+		if (redisCache.active) {
+			redisCache.set("ip-" + key, value, expirationMillis);
+		}
+	}
+};
+
+
+
 function redirectToConnectPageIfNeeded(req, res) {
 	if (!req.session.host) {
 		req.session.redirectUrl = req.originalUrl;
@@ -340,8 +377,8 @@ function refreshExchangeRates() {
 	}
 }
 
-// Uses ip-api.com API
-function geoLocateIpAddresses(ipAddresses) {
+// Uses ipstack.com API
+function geoLocateIpAddresses(ipAddresses, provider) {
 	return new Promise(function(resolve, reject) {
 		if (config.privacyMode) {
 			resolve({});
@@ -349,53 +386,48 @@ function geoLocateIpAddresses(ipAddresses) {
 			return;
 		}
 
-		var chunks = splitArrayIntoChunks(ipAddresses, 1);
+		var ipDetails = {ips:ipAddresses, detailsByIp:{}};
 
 		var promises = [];
-		for (var i = 0; i < chunks.length; i++) {
-			var ipStr = "";
-			for (var j = 0; j < chunks[i].length; j++) {
-				if (j > 0) {
-					ipStr = ipStr + ",";
-				}
-
-				ipStr = ipStr + chunks[i][j];
-			}
+		for (var i = 0; i < ipAddresses.length; i++) {
+			var ipStr = ipAddresses[i];
+			
+			promises.push(new Promise(function(resolve2, reject2) {
+				ipCache.get(ipStr).then(function(result) {
+					if (result.value == null) {
+						var apiUrl = "http://api.ipstack.com/" + result.key + "?access_key=" + config.credentials.ipStackComApiAccessKey;
+						
+						console.log("Requesting IP-geo: " + apiUrl);
+
+						request(apiUrl, function(error, response, body) {
+							if (error) {
+								reject2(error);
+
+							} else {
+								resolve2({needToProcess:true, response:response});
+							}
+						});
 
-			if (ipCache[ipStr] != null) {
-				promises.push(new Promise(function(resolve2, reject2) {
-					resolve2(ipCache[ipStr]);
-				}));
+					} else {
+						ipDetails.detailsByIp[result.key] = result.value;
 
-			} else {
-				var apiUrl = "http://ip-api.com/json/" + ipStr;
-				promises.push(new Promise(function(resolve2, reject2) {
-					request(apiUrl, function(error, response, body) {
-						if (error) {
-							reject2(error);
-
-						} else {
-							resolve2(response);
-						}
-					});
-				}));
-			}
+						resolve2({needToProcess:false});
+					}
+				});
+			}));
 		}
 
 		Promise.all(promises).then(function(results) {
-			var ipDetails = {ips:[], detailsByIp:{}};
-
 			for (var i = 0; i < results.length; i++) {
-				var res = results[i];
-				if (res != null && res["statusCode"] == 200) {
-					var resBody = JSON.parse(res["body"]);
-					var ip = resBody["query"];
+				if (results[i].needToProcess) {
+					var res = results[i].response;
+					if (res != null && res["statusCode"] == 200) {
+						var resBody = JSON.parse(res["body"]);
+						var ip = resBody["ip"];
 
-					ipDetails.ips.push(ip);
-					ipDetails.detailsByIp[ip] = resBody;
+						ipDetails.detailsByIp[ip] = resBody;
 
-					if (ipCache[ip] == null) {
-						ipCache[ip] = res;
+						ipCache.set(ip, resBody, 1000 * 60 * 60 * 24 * 365);
 					}
 				}
 			}
@@ -403,6 +435,8 @@ function geoLocateIpAddresses(ipAddresses) {
 			resolve(ipDetails);
 
 		}).catch(function(err) {
+			console.log("Error 80342hrf78wgehdf07gds: " + err);
+
 			reject(err);
 		});
 	});

views/peers.pug

@@ -115,12 +115,13 @@ block content
 											- var ipAddr = item.addr.substring(0, item.addr.lastIndexOf(":"));
 											if (peerIpSummary.ips && peerIpSummary.ips.includes(ipAddr))
 												- var ipDetails = peerIpSummary.detailsByIp[ipAddr];
-												if (ipDetails.city)
-													span #{ipDetails.city}, 
-												if (ipDetails.region)
-													span #{ipDetails.region}, 
-												if (ipDetails.country)
-													span #{ipDetails.country}
+												if (ipDetails)
+													if (ipDetails.city)
+														span #{ipDetails.city}, 
+													if (ipDetails.region_name)
+														span #{ipDetails.region_name}, 
+													if (ipDetails.country_name)
+														span #{ipDetails.country_name}
 											else
 												span ?
 
@@ -182,18 +183,18 @@ block endOfBody
 
 		each ipAddress, index in peerIpSummary.ips
 			- var ipDetails = peerIpSummary.detailsByIp[ipAddress];
-			if (ipDetails && ipDetails.lat && ipDetails.lon)
+			if (ipDetails && ipDetails.latitude && ipDetails.longitude)
 				- var ipDetailsPopupHtml = "<b>" + ipAddress + "</b><br>";
 				if (ipDetails.city)
 					- var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.city + ", ";
 
-				if (ipDetails.region)
-					- var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.region + ", ";
+				if (ipDetails.region_name)
+					- var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.region_name + ", ";
 
-				if (ipDetails.country)
-					- var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.countryCode + " ";
+				if (ipDetails.country_name)
+					- var ipDetailsPopupHtml = ipDetailsPopupHtml + ipDetails.country_name + " ";
 
-				script L.marker([#{ipDetails.lat}, #{ipDetails.lon}]).addTo(mymap).bindPopup("!{ipDetailsPopupHtml}");
+				script L.marker([#{ipDetails.latitude}, #{ipDetails.longitude}]).addTo(mymap).bindPopup("!{ipDetailsPopupHtml}");