lukechilds
/
insight-ui-komodo
mirror of https://github.com/lukechilds/insight-ui-komodo.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

better smart syncs

generic-ui
Matias Alejo Garcia 11 years ago
parent
c8adefddd9
commit
4d23191116
6 changed files with 132 additions and 77 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      app/models/Block.js
  2. 4
      app/models/Transaction.js
  3. 110
      lib/HistoricSync.js
  4. 76
      lib/Sync.js
  5. 7
      server.js
  6. 11
      util/sync.js

1
app/models/Block.js
View File

@ -27,6 +27,7 @@ var BlockSchema = new Schema({
},
time: Number,
nextBlockHash: String,
isOrphan: Boolean,
});
/**

4
app/models/Transaction.js
View File

@ -142,7 +142,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) {
}
else {
if ( !i.coinbase ) {
console.log ('TX: %s,%d could not parse INPUT', txid, i.n);
console.log ('WARN in TX: %s: could not parse INPUT %d', txid, i.n);
}
return next_in();
}
@ -165,7 +165,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) {
}, next_out);
}
else {
console.log ('TX: %s,%d could not parse OUTPUT', txid, o.n);
console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n);
return next_out();
}
},

110
lib/HistoricSync.js
View File

@ -13,14 +13,16 @@ function spec() {
var Sync = require('./Sync').class();
function HistoricSync(opts) {
this.block_count= 0;
this.block_total= 0;
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
var genesisHashReversed = new Buffer(32);
this.network.genesisBlock.hash.copy(genesisHashReversed);
this.genesis = genesisHashReversed.reverse().toString('hex');
this.sync = new Sync(opts);
//available status: new / syncing / finished / aborted
this.status = 'new';
this.syncInfo = {};
}
function p() {
@ -32,8 +34,9 @@ function spec() {
console.log.apply(this, args);
}
var progress_bar = function(string, current, total) {
p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total)));
var printProgress = function(i) {
var per = parseInt(100 * i.syncedBlocks / i.blocksToSync);
p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per));
};
HistoricSync.prototype.init = function(opts,cb) {
@ -47,7 +50,6 @@ function spec() {
};
HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
var self = this;
// recursion end.
@ -71,8 +73,11 @@ function spec() {
},
//show some (inacurate) status
function(c) {
if (self.block_count % 1000 === 1) {
progress_bar('sync status:', self.block_count, self.block_total);
var step = parseInt(self.syncInfo.blocksToSync / 100);
if (step < 10) step = 10;
if (self.syncInfo.syncedBlocks % step === 1) {
printProgress(self.syncInfo);
}
return c();
},
@ -113,15 +118,30 @@ function spec() {
],
function (err){
if (err)
p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count);
if (err) {
self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks);
self.status = 'aborted';
p(self.err);
}
if (opts.uptoexisting && existed) {
p('DONE. Found existing block: ', blockHash);
return cb(err);
else {
self.err = null;
self.status = 'syncing';
}
if (opts.uptoexisting && existed ) {
if (self.syncInfo.blocksToSync <= self.syncInfo.syncedBlocks) {
self.status = 'finished';
p('DONE. Found existing block: ', blockHash);
return cb(err);
}
else {
p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks);
}
}
if (blockEnd && blockEnd === blockHash) {
self.status = 'finished';
p('DONE. Found END block: ', blockHash);
return cb(err);
}
@ -129,7 +149,7 @@ function spec() {
// Continue
if (blockInfo && blockInfo.result) {
self.block_count++;
self.syncInfo.syncedBlocks++;
if (opts.prev && blockInfo.result.previousblockhash) {
return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
}
@ -144,11 +164,10 @@ function spec() {
HistoricSync.prototype.import_history = function(opts, next) {
var self = this;
var retry_attemps = 100;
var retry_secs = 2;
var block_best;
var block_height;
var bestBlock;
var blockChainHeight;
async.series([
function(cb) {
@ -175,41 +194,56 @@ function spec() {
return cb();
}
},
function(cb) {
self.rpc.getInfo(function(err, res) {
if (err) return cb(err);
self.block_total = res.result.blocks;
return cb();
});
},
// We are not using getBestBlockHash, because is not available in all clients
function(cb) {
if (!opts.reverse) return cb();
self.rpc.getBlockCount(function(err, res) {
if (err) return cb(err);
block_height = res.result;
blockChainHeight = res.result;
return cb();
});
},
function(cb) {
if (!opts.reverse) return cb();
self.rpc.getBlockHash(block_height, function(err, res) {
self.rpc.getBlockHash(blockChainHeight, function(err, res) {
if (err) return cb(err);
block_best = res.result;
bestBlock = res.result;
return cb();
});
},
function(cb) {
// This is only to inform progress.
if (!opts.uptoexisting) {
self.rpc.getInfo(function(err, res) {
if (err) return cb(err);
self.syncInfo.blocksToSync = res.result.blocks;
return cb();
});
}
else {
// should be isOrphan = true or null to be more accurate.
Block.count({ isOrphan: null}, function(err, count) {
if (err) return cb(err);
self.syncInfo.blocksToSync = blockChainHeight - count;
if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1;
return cb();
});
}
},
],
function(err) {
var start, end;
function sync() {
if (opts.reverse) {
start = block_best;
start = bestBlock;
end = self.genesis;
opts.prev = true;
}
@ -219,25 +253,38 @@ function spec() {
opts.next = true;
}
self.syncInfo = util._extend(self.syncInfo, {
start: start,
isStartGenesis: start === self.genesis,
end: end,
isEndGenesis: end === self.genesis,
scanningForward: opts.next,
scanningBackward: opts.prev,
uptoexisting: opts.uptoexisting,
syncedBlocks: 0,
});
p('Starting from: ', start);
p(' to : ', end);
p(' opts: ', JSON.stringify(opts));
self.getPrevNextBlock( start, end, opts , function(err) {
if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
if (err && err.message.match(/ECONNREFUSED/)){
setTimeout(function() {
p('Retrying in %d secs', retry_secs);
sync();
}, retry_secs * 1000);
}
else
return next(err, self.block_count);
return next(err);
});
}
if (!err)
sync();
else
else {
return next(err, 0);
}
});
};
@ -246,6 +293,7 @@ function spec() {
var self = this;
Block.findOne({hash:self.genesis}, function(err, b){
if (err) return next(err);
@ -253,7 +301,7 @@ function spec() {
p('Could not find Genesis block. Running FULL SYNC');
}
else {
p('Genesis block found. Syncing upto know blocks.');
p('Genesis block found. Syncing upto known blocks.');
}
var opts = {

76
lib/Sync.js
View File

@ -15,6 +15,48 @@ function spec() {
this.tx_count = 0;
}
Sync.prototype.init = function(opts, cb) {
var that = this;
that.opts = opts;
if (!(opts && opts.skip_db_connection)) {
if (!mongoose.connection.readyState == 1) {
mongoose.connect(config.db, function(err) {
if (err) {
console.log('CRITICAL ERROR: connecting to mongoDB:',err);
return (err);
}
});
}
that.db = mongoose.connection;
that.db.on('error', function(err) {
console.log('MongoDB ERROR:' + err);
return cb(err);
});
that.db.on('disconnect', function(err) {
console.log('MongoDB disconnect:' + err);
return cb(err);
});
return that.db.once('open', function(err) {
return cb(err);
});
}
else return cb();
};
Sync.prototype.close = function() {
if ( this.db && this.db.readyState ) {
this.db.close();
}
};
Sync.prototype.storeBlock = function(block, cb) {
var that = this;
@ -55,40 +97,6 @@ function spec() {
return cb(err);
});
};
Sync.prototype.init = function(opts, cb) {
var that = this;
that.opts = opts;
if (!(opts && opts.skip_db_connection)) {
if (!mongoose.connection) {
mongoose.connect(config.db, {server: {auto_reconnect: true}} );
}
this.db = mongoose.connection;
this.db.on('error', function(err) {
console.log('connection error:' + err);
mongoose.disconnect();
});
this.db.on('disconnect', function(err) {
console.log('disconnect:' + err);
mongoose.connect(config.db, {server: {auto_reconnect: true}} );
});
return that.db.once('open', cb);
}
else return cb();
};
Sync.prototype.close = function() {
if (!(this.opts && this.opts.skip_db_connection)) {
this.db.close();
}
};
return Sync;
}
module.defineClass(spec);

7
server.js
View File

@ -50,8 +50,11 @@ if (!config.disableHistoricSync) {
skip_db_connection: true,
networkName: config.network
}, function() {
hs.smart_import(function(){
console.log('[historic_sync] finished!');
hs.smart_import(function(err){
var txt= 'ended.';
if (err) txt = 'ABORTED with error: ' + err.message;
console.log('[historic_sync] ' + txt, hs.syncInfo);
});
});
}

11
util/sync.js
View File

@ -29,7 +29,6 @@ if (program.remove) {
}
*/
async.series([
function(cb) {
historicSync.init(program, cb);
@ -46,18 +45,14 @@ async.series([
}, cb);
}
},
function(cb) {
historicSync.close();
return cb();
},
],
function(err, count) {
function(err) {
historicSync.close();
if (err) {
console.log('CRITICAL ERROR: ', err);
}
else {
console.log('Finished. [%d blocks synced]', count[1]);
console.log('Finished.\n Status:\n', historicSync.syncInfo);
}
return;
});

Write Preview
Loading…
Cancel
Save