From b026a3075ea15b8741597df3047f1782d21e0f6c Mon Sep 17 00:00:00 2001 From: pbca26 Date: Sat, 8 Jul 2017 19:44:04 +0300 Subject: [PATCH 1/4] dl bins (wip) --- routes/shepherd.js | 116 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/routes/shepherd.js b/routes/shepherd.js index dd72c6d..eb3bf41 100644 --- a/routes/shepherd.js +++ b/routes/shepherd.js @@ -18,6 +18,7 @@ const electron = require('electron'), rimraf = require('rimraf'), portscanner = require('portscanner'), AdmZip = require('adm-zip'), + remoteFileSize = require('remote-file-size'), Promise = require('bluebird'); const fixPath = require('fix-path'); @@ -188,6 +189,121 @@ function downloadFile(configuration) { }); } +const remoteBinLocation = { + 'win32': 'https://artifacts.supernet.org/latest/windows/', + 'darwin': 'https://artifacts.supernet.org/latest/osx/', + 'linux': 'https://artifacts.supernet.org/latest/linux/' +}; +const localBinLocation = { + 'win32': 'assets/bin/win64/', + 'darwin': 'assets/bin/osx/', + 'linux': 'assets/bin/linux64/' +}; +const latestBins = { + 'win32': [ + 'iguana.exe', + 'komodo-cli.exe', + 'komodo-tx.exe', + 'komodod.exe', + 'libcrypto-1_1.dll', + 'libcurl-4.dll', + 'libcurl.dll', + 'libgcc_s_sjlj-1.dll', + 'libnanomsg.dll', + 'libssl-1_1.dll', + 'libwinpthread-1.dll', + 'nanomsg.dll', + 'pthreadvc2.dll' + ], + 'darwin': [ + 'iguana', + 'komodo-cli', + 'komodod', + 'libgcc_s.1.dylib', + 'libgomp.1.dylib', + 'libnanomsg.5.0.0.dylib', + 'libstdc++.6.dylib' // encode %2B + ], + 'linux': [ + 'iguana', + 'komodo-cli', + 'komodod' + ] +}; + +let binsToUpdate = []; + +/* + * Check bins file size + * type: + * params: + */ +shepherd.get('/check-bins', function(req, res, next) { + const rootLocation = path.join(__dirname, '../'); + + const successObj = { + 'msg': 'success', + 'result': 'bins' + }; + + res.end(JSON.stringify(successObj)); + + const _os = os.platform(); + console.log('checking bins: ' + _os); + + // get list of bins/dlls that can be updated to the latest + for (let i = 0; i < latestBins[_os].length; i++) { + remoteFileSize(remoteBinLocation[_os] + latestBins[_os][i], function(err, remoteBinSize) { + const localBinSize = fs.statSync(rootLocation + localBinLocation[_os] + latestBins[_os][i]).size; + + console.log('remote url: ' + (remoteBinLocation[_os] + latestBins[_os][i]) + ' (' + remoteBinSize + ')'); + console.log('local file: ' + (rootLocation + localBinLocation[_os] + latestBins[_os][i]) + ' (' + localBinSize + ')'); + + if (remoteBinSize !== localBinSize) { + console.log(latestBins[_os][i] + ' can be updated'); + binsToUpdate.push({ + 'name': latestBins[_os][i], + 'rSize': remoteBinSize, + 'lSize': localBinSize + }); + } + }); + } +}); + +/* + * Update bins + * type: + * params: + */ +shepherd.get('/update-bins', function(req, res, next) { + const rootLocation = path.join(__dirname, '../'); + const _os = os.platform(); + const successObj = { + 'msg': 'success', + 'result': { + 'filesCount': binsToUpdate.length, + 'list': binsToUpdate + } + }; + + res.end(JSON.stringify(successObj)); + + for (let i = 0; i < binsToUpdate.length; i++) { + downloadFile({ + remoteFile: remoteBinLocation[_os] + binsToUpdate[i].name, + localFile: rootLocation + localBinLocation[_os] + 'new/' + binsToUpdate[i].name, + onProgress: function(received, total) { + const percentage = (received * 100) / total; + console.log(binsToUpdate[i].name + ' ' + percentage + '% | ' + received + ' bytes out of ' + total + ' bytes.'); + } + }) + .then(function() { + console.log('File ' + binsToUpdate[i].name + ' succesfully downloaded'); + }); + } +}); + /* * DL app patch * type: From 660c3099f69639854b1725550a4fd7c4ec9a3b9b Mon Sep 17 00:00:00 2001 From: pbca26 Date: Sun, 9 Jul 2017 03:10:19 -0700 Subject: [PATCH 2/4] dl bins (wip #2) --- routes/shepherd.js | 55 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/routes/shepherd.js b/routes/shepherd.js index eb3bf41..40dc451 100644 --- a/routes/shepherd.js +++ b/routes/shepherd.js @@ -158,7 +158,11 @@ function downloadFile(configuration) { let req = request({ method: 'GET', - uri: configuration.remoteFile + uri: configuration.remoteFile, + agentOptions: { + keepAlive: true, + keepAliveMsecs: 15000 + } }); let out = fs.createWriteStream(configuration.localFile); @@ -203,7 +207,6 @@ const latestBins = { 'win32': [ 'iguana.exe', 'komodo-cli.exe', - 'komodo-tx.exe', 'komodod.exe', 'libcrypto-1_1.dll', 'libcurl-4.dll', @@ -292,14 +295,22 @@ shepherd.get('/update-bins', function(req, res, next) { for (let i = 0; i < binsToUpdate.length; i++) { downloadFile({ remoteFile: remoteBinLocation[_os] + binsToUpdate[i].name, - localFile: rootLocation + localBinLocation[_os] + 'new/' + binsToUpdate[i].name, + localFile: rootLocation + localBinLocation[_os] + binsToUpdate[i].name, onProgress: function(received, total) { const percentage = (received * 100) / total; console.log(binsToUpdate[i].name + ' ' + percentage + '% | ' + received + ' bytes out of ' + total + ' bytes.'); } }) .then(function() { - console.log('File ' + binsToUpdate[i].name + ' succesfully downloaded'); + // verify that remote file is matching to DL'ed file + const localBinSize = fs.statSync(rootLocation + localBinLocation[_os] + binsToUpdate[i].name).size; + console.log('compare dl file size'); + + if (localBinSize === binsToUpdate[i].rSize) { + console.log('file ' + binsToUpdate[i].name + ' succesfully downloaded'); + } else { + console.log('error: ' + binsToUpdate[i].name + ' file size doesnt match remote!'); + } }); } }); @@ -310,8 +321,6 @@ shepherd.get('/update-bins', function(req, res, next) { * params: patchList */ shepherd.get('/patch', function(req, res, next) { - const dlLocation = path.join(__dirname, '../'); - const successObj = { 'msg': 'success', 'result': 'dl started' @@ -323,12 +332,14 @@ shepherd.get('/patch', function(req, res, next) { }); shepherd.updateAgama = function() { + const rootLocation = path.join(__dirname, '../'); + downloadFile({ remoteFile: 'https://github.com/pbca26/dl-test/raw/master/patch.zip', - localFile: dlLocation + 'patch.zip', + localFile: rootLocation + 'patch.zip', onProgress: function(received, total) { const percentage = (received * 100) / total; - // console.log(percentage + '% | ' + received + ' bytes out of ' + total + ' bytes.'); + console.log('patch ' + percentage + '% | ' + received + ' bytes out of ' + total + ' bytes.'); cache.io.emit('service', { 'patch': { 'status': 'dl', @@ -340,15 +351,25 @@ shepherd.updateAgama = function() { } }) .then(function() { - console.log('File succesfully downloaded'); - console.log('extracting contents'); - - var zip = new AdmZip(dlLocation + 'patch.zip'); - zip.extractAllTo(/*target path*/dlLocation + '/patch/unpack', /*overwrite*/true); - // TODO: extract files in chunks - cache.io.emit('service', { - 'patch': { - 'status': 'done' + remoteFileSize('https://github.com/pbca26/dl-test/raw/master/patch.zip', function(err, remotePatchSize) { + // verify that remote file is matching to DL'ed file + const localPatchSize = fs.statSync(rootLocation + 'patch.zip').size; + console.log('compare dl file size'); + + if (localPatchSize === remotePatchSize) { + console.log('patch succesfully downloaded'); + console.log('extracting contents'); + + var zip = new AdmZip(rootLocation + 'patch.zip'); + zip.extractAllTo(/*target path*/rootLocation + '/patch', /*overwrite*/true); + // TODO: extract files in chunks + cache.io.emit('service', { + 'patch': { + 'status': 'done' + } + }); + } else { + console.log('patch file size doesnt match remote!'); } }); }); From 2ef5bd2b47dad824aa48e2e3a4485d33429b3161 Mon Sep 17 00:00:00 2001 From: pbca26 Date: Sun, 9 Jul 2017 21:05:13 +0300 Subject: [PATCH 3/4] updater (ui) --- package.json | 1 + routes/shepherd.js | 86 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 2dfd09d..91a4482 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "graceful-fs": "^4.1.11", "md5": "^2.2.1", "mkdirp": "^0.5.1", + "nodejs-aes256": "^1.0.1", "pm2": "^2.4.3", "portscanner": "^2.1.1", "ps-node": "^0.1.5", diff --git a/routes/shepherd.js b/routes/shepherd.js index 40dc451..e26802d 100644 --- a/routes/shepherd.js +++ b/routes/shepherd.js @@ -241,7 +241,7 @@ let binsToUpdate = []; * type: * params: */ -shepherd.get('/check-bins', function(req, res, next) { +shepherd.get('/update/bins/check', function(req, res, next) { const rootLocation = path.join(__dirname, '../'); const successObj = { @@ -254,6 +254,13 @@ shepherd.get('/check-bins', function(req, res, next) { const _os = os.platform(); console.log('checking bins: ' + _os); + cache.io.emit('patch', { + 'patch': { + 'type': 'bins-check', + 'status': 'progress', + 'message': 'checking bins: ' + _os + } + }); // get list of bins/dlls that can be updated to the latest for (let i = 0; i < latestBins[_os].length; i++) { remoteFileSize(remoteBinLocation[_os] + latestBins[_os][i], function(err, remoteBinSize) { @@ -270,6 +277,16 @@ shepherd.get('/check-bins', function(req, res, next) { 'lSize': localBinSize }); } + + if (i === latestBins[_os].length - 1) { + cache.io.emit('patch', { + 'patch': { + 'type': 'bins-check', + 'status': 'done', + 'fileList': binsToUpdate + } + }); + } }); } }); @@ -279,7 +296,7 @@ shepherd.get('/check-bins', function(req, res, next) { * type: * params: */ -shepherd.get('/update-bins', function(req, res, next) { +shepherd.get('/update/bins', function(req, res, next) { const rootLocation = path.join(__dirname, '../'); const _os = os.platform(); const successObj = { @@ -295,20 +312,43 @@ shepherd.get('/update-bins', function(req, res, next) { for (let i = 0; i < binsToUpdate.length; i++) { downloadFile({ remoteFile: remoteBinLocation[_os] + binsToUpdate[i].name, - localFile: rootLocation + localBinLocation[_os] + binsToUpdate[i].name, + localFile: rootLocation + localBinLocation[_os] + 'patch/' + binsToUpdate[i].name, onProgress: function(received, total) { const percentage = (received * 100) / total; + cache.io.emit('patch', { + 'msg': { + 'type': 'bins-update', + 'status': 'progress', + 'file': binsToUpdate[i].name, + 'bytesTotal': total, + 'bytesReceived': received + } + }); console.log(binsToUpdate[i].name + ' ' + percentage + '% | ' + received + ' bytes out of ' + total + ' bytes.'); } }) .then(function() { // verify that remote file is matching to DL'ed file - const localBinSize = fs.statSync(rootLocation + localBinLocation[_os] + binsToUpdate[i].name).size; + const localBinSize = fs.statSync(rootLocation + localBinLocation[_os] + 'patch/' + binsToUpdate[i].name).size; console.log('compare dl file size'); - + if (localBinSize === binsToUpdate[i].rSize) { + cache.io.emit('patch', { + 'msg': { + 'type': 'bins-update', + 'file': binsToUpdate[i].name, + 'status': 'done' + } + }); console.log('file ' + binsToUpdate[i].name + ' succesfully downloaded'); } else { + cache.io.emit('patch', { + 'msg': { + 'type': 'bins-update', + 'file': binsToUpdate[i].name, + 'message': 'size mismatch' + } + }); console.log('error: ' + binsToUpdate[i].name + ' file size doesnt match remote!'); } }); @@ -320,7 +360,7 @@ shepherd.get('/update-bins', function(req, res, next) { * type: * params: patchList */ -shepherd.get('/patch', function(req, res, next) { +shepherd.get('/update/patch', function(req, res, next) { const successObj = { 'msg': 'success', 'result': 'dl started' @@ -340,9 +380,10 @@ shepherd.updateAgama = function() { onProgress: function(received, total) { const percentage = (received * 100) / total; console.log('patch ' + percentage + '% | ' + received + ' bytes out of ' + total + ' bytes.'); - cache.io.emit('service', { - 'patch': { - 'status': 'dl', + cache.io.emit('patch', { + 'msg': { + 'status': 'progress', + 'type': 'ui', 'progress': percentage, 'bytesTotal': total, 'bytesReceived': received @@ -355,20 +396,29 @@ shepherd.updateAgama = function() { // verify that remote file is matching to DL'ed file const localPatchSize = fs.statSync(rootLocation + 'patch.zip').size; console.log('compare dl file size'); - + if (localPatchSize === remotePatchSize) { console.log('patch succesfully downloaded'); console.log('extracting contents'); - var zip = new AdmZip(rootLocation + 'patch.zip'); - zip.extractAllTo(/*target path*/rootLocation + '/patch', /*overwrite*/true); + const zip = new AdmZip(rootLocation + 'patch.zip'); + zip.extractAllTo(/*target path*/rootLocation, /*overwrite*/true); // TODO: extract files in chunks - cache.io.emit('service', { - 'patch': { + cache.io.emit('patch', { + 'msg': { + 'type': 'ui', 'status': 'done' } }); + fs.unlink(rootLocation + 'patch.zip'); } else { + cache.io.emit('patch', { + 'msg': { + 'type': 'ui', + 'status': 'error', + 'message': 'size mismatch' + } + }); console.log('patch file size doesnt match remote!'); } }); @@ -380,7 +430,7 @@ shepherd.updateAgama = function() { * type: * params: */ -shepherd.get('/update-check', function(req, res, next) { +shepherd.get('/update/patch/check', function(req, res, next) { const rootLocation = path.join(__dirname, '../'); const options = { url: 'https://github.com/pbca26/dl-test/raw/master/version', @@ -404,7 +454,11 @@ shepherd.get('/update-check', function(req, res, next) { } else { const successObj = { 'msg': 'success', - 'result': 'update' + 'result': 'update', + 'version': { + 'local': localVersion[0], + 'remote': remoteVersion[0], + } }; res.end(JSON.stringify(successObj)); From 04f1742c5f23a0e578374e88280e447daa0a2694 Mon Sep 17 00:00:00 2001 From: pbca26 Date: Sun, 9 Jul 2017 21:26:39 +0300 Subject: [PATCH 4/4] updater dev mode; reduce sockets msg count --- routes/shepherd.js | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/routes/shepherd.js b/routes/shepherd.js index e26802d..f9fd409 100644 --- a/routes/shepherd.js +++ b/routes/shepherd.js @@ -379,16 +379,19 @@ shepherd.updateAgama = function() { localFile: rootLocation + 'patch.zip', onProgress: function(received, total) { const percentage = (received * 100) / total; - console.log('patch ' + percentage + '% | ' + received + ' bytes out of ' + total + ' bytes.'); - cache.io.emit('patch', { - 'msg': { - 'status': 'progress', - 'type': 'ui', - 'progress': percentage, - 'bytesTotal': total, - 'bytesReceived': received - } - }); + if (Math.floor(percentage) % 5 === 0 || + Math.floor(percentage) % 10 === 0) { + console.log('patch ' + percentage + '% | ' + received + ' bytes out of ' + total + ' bytes.'); + cache.io.emit('patch', { + 'msg': { + 'status': 'progress', + 'type': 'ui', + 'progress': percentage, + 'bytesTotal': total, + 'bytesReceived': received + } + }); + } } }) .then(function() { @@ -402,7 +405,14 @@ shepherd.updateAgama = function() { console.log('extracting contents'); const zip = new AdmZip(rootLocation + 'patch.zip'); - zip.extractAllTo(/*target path*/rootLocation, /*overwrite*/true); + + if (shepherd.appConfig.dev) { + if (!fs.existsSync(`${rootLocation}/patch`)) { + fs.mkdirSync(`${rootLocation}/patch`); + } + } + + zip.extractAllTo(/*target path*/rootLocation + (shepherd.appConfig.dev ? '/patch' : ''), /*overwrite*/true); // TODO: extract files in chunks cache.io.emit('patch', { 'msg': {