diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 44831ae..13dd2e8 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -16,8 +16,7 @@ function spec() { util = require('bitcore/util/util'), levelup = require('levelup'), BitcoreBlock= require('bitcore/Block').class(), - config = require('../config/config'), - fs = require('fs'); + config = require('../config/config'); var BlockDb = function() { this.db = levelup(config.leveldb + '/blocks'); @@ -26,11 +25,11 @@ function spec() { BlockDb.prototype.drop = function(cb) { var self = this; var path = config.leveldb + '/blocks'; - require('leveldown').destroy(path, function () { - fs.mkdirSync(config.leveldb); - fs.mkdirSync(path); - self.db = levelup(path); - return cb(); + self.db.close(function() { + require('leveldown').destroy(path, function () { + self.db = levelup(path); + return cb(); + }); }); }; @@ -52,21 +51,36 @@ function spec() { BlockDb.prototype.setOrphan = function(hash, cb) { var self = this; - var k = PREV_ROOT + hash; + self.db.get(k, function (err,oldPrevHash) { if (err || !oldPrevHash) return cb(err); - self.db.put(PREV_ROOT + hash, 0, function() { return cb(err, oldPrevHash); }); }); - // We keep the block in TIMESTAMP_ROOT }; - BlockDb.prototype.countNotOrphan = function(hash, cb) { + //mainly for testing + BlockDb.prototype.setPrev = function(hash, prevHash, cb) { + this.db.put(PREV_ROOT + hash, prevHash, function(err) { + return cb(err); + }); + }; + + //mainly for testing + BlockDb.prototype.getPrev = function(hash, cb) { + this.db.get(PREV_ROOT + hash, function(err,val) { + return cb(err,val); + }); + }; + + + + BlockDb.prototype.countNotOrphan = function(cb) { + var self = this; var c = 0; this.db.createReadStream({start: PREV_ROOT}) .on('data', function (data) { @@ -75,9 +89,6 @@ function spec() { .on('error', function (err) { return cb(err); }) - .on('close', function () { - return cb(null); - }) .on('end', function () { return cb(null); }); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 69ac2ea..1c2c3ee 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -313,11 +313,12 @@ var kk=0; }, //store it function(c) { - if (self.prevHash && blockInfo.prev_block !== self.prevHash) { + if (self.prevHash && blockinfo.prev_block !== self.prevHash) { - console.log('Orphans found: %s vs %s @%s', - self.prevHash + ' vs. ' + blockInfo.prev_block, blockInfo.hash); + console.log('Orphans found: @%s', blockInfo.hash); + console.log('From: %s To: %s', self.prevHash, blockInfo.prev_block); +process.exit(1); self.sync.setOrphan(self.prevHash, blockInfo.prev_block, c); } else return c(); @@ -349,6 +350,17 @@ var kk=0; }; + HistoricSync.prototype.getBlockCount = function(cb) { + var self = this; + + if (self.blockChainHeight) return cb(); + + self.rpc.getBlockCount(function(err, res) { + if (err) return cb(err); + self.blockChainHeight = res.result; + return cb(); + }); + }; HistoricSync.prototype.importHistory = function(scanOpts, next) { var self = this; @@ -364,13 +376,7 @@ var kk=0; return cb(); }, // We are not using getBestBlockHash, because is not available in all clients - function(cb) { - self.rpc.getBlockCount(function(err, res) { - if (err) return cb(err); - self.blockChainHeight = res.result; - return cb(); - }); - }, + function (cb) { return self.getBlockCount(cb); }, function(cb) { if (!scanOpts.reverse) return cb(); @@ -461,27 +467,37 @@ var kk=0; var self = this; self.sync.hasBlock(self.genesis, function(err, b) { - if (err) return next(err); - - if (!b || scanOpts.destroy) { - p('Could not find Genesis block. Running FULL SYNC'); - if (config.bitcoind.dataDir) { - p('bitcoind dataDir configured...importing blocks from .dat files'); - scanOpts.fromFiles = true; - self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); - } - else { - scanOpts.reverse = true; - } - } - else { - p('Genesis block found. Syncing upto known blocks.'); - scanOpts.reverse = true; - scanOpts.upToExisting = true; - } - - return self.importHistory(scanOpts, next); + self.sync.countNotOrphan(function(err, count) { + if (err) return next(err); + self.getBlockCount(function(err) { + if (err) return next(err); + + if (!b || scanOpts.destroy || self.blockChainHeight > count * 0.7 ) { + + if (!b) + p('Could not find Genesis block. Running FULL SYNC'); + else + p('Less that 70% of current blockchain is stored. Running FULL SYNC', + parseInt(count/self.blockChainHeight*100)); + + if (config.bitcoind.dataDir) { + p('bitcoind dataDir configured...importing blocks from .dat files'); + scanOpts.fromFiles = true; + self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network); + } + else { + scanOpts.reverse = true; + } + } + else { + p('Genesis block found. Syncing upto known blocks.'); + scanOpts.reverse = true; + scanOpts.upToExisting = true; + } + return self.importHistory(scanOpts, next); + }); + }); }); }; diff --git a/lib/Sync.js b/lib/Sync.js index bfa2d55..27362de 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -63,8 +63,7 @@ function spec() { var self = this; async.series([ function(b) { try {self.blockDb.drop(b);} catch (e) { return b(); } }, - function(b) { try {self.db.collections.transactionitems.drop(b);} catch (e) { return b(); } }, - function(b) { try {self.db.collections.transactionouts.drop(b);} catch (e) { return b(); } }, + function(b) { try {self.TransactionDb.drop(b);} catch (e) { return b(); } }, ], next); }; @@ -81,7 +80,6 @@ function spec() { }; - Sync.prototype.storeBlock = function(block, cb) { var self = this; @@ -99,26 +97,24 @@ function spec() { Sync.prototype.setOrphan = function(fromBlock, toBlock, c) { var self = this; - var c = fromBlock; + var hash = fromBlock; async.whilst( function () { - return c !== toBlock; + return hash && hash !== toBlock; }, - function () { - console.log('[Sync.js.113]: setOrphan', c); //TODO + function (w_c) { self.txDb.setOrphan(c, function(err, insertedTxs, updateAddrs) { - if (err) return cb(err); - - self.blockDb.setOrphan(c, function(err, prevHash){ + if (err) return w_c(err); + self.blockDb.setOrphan(hash, function(err, prevHash){ - c = prevHash; - return cb(err); + hash = prevHash; + return w_c(err); }); }); }, - function (err) { - return c(err); + function (err) { + return c(err); } ); }; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index b4c3ecb..f3ab69b 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -26,8 +26,7 @@ function spec() { util = require('bitcore/util/util'), levelup = require('levelup'), async = require('async'), - config = require('../config/config'), - fs = require('fs'); + config = require('../config/config'); var TransactionDb = function() { this.db = levelup(config.leveldb + '/txs'); @@ -35,12 +34,12 @@ function spec() { TransactionDb.prototype.drop = function(cb) { var self = this; - var path = config.leveldb + '/blocks'; - require('leveldown').destroy(path, function () { - fs.mkdirSync(config.leveldb); - fs.mkdirSync(path); - self.db = levelup(path); - return cb(); + var path = config.leveldb + '/txs'; + self.db.close(function() { + require('leveldown').destroy(path, function () { + self.db = levelup(path); + return cb(); + }); }); }; @@ -129,9 +128,6 @@ function spec() { var k = ADDR_ROOT + addr; var ret=[]; - //var ADDR_ROOT = 'txouts-addr-'; //txouts-addr---- => + btc_sat - //var SPEND_ROOT = 'txouts-spend-';//txouts-spend-- => [txid(in),n(in),ts] - // // self.db.createReadStream({start: k, end: k + '~'}) .on('data', function (data) { @@ -149,7 +145,7 @@ function spec() { }) .on('end', function () { async.each(ret, function(o, e_c) { - var k = SPEND_ROOT + '-' + o.txid + '-' + o.index; //TODO --- + var k = SPEND_ROOT + o.txid + '-' + o.index; self.db.get(k, function(err, val) { if (err && err.notFound) err=null; if (err || !val) return e_c(err); diff --git a/package.json b/package.json index 6860d54..4a6e18d 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,6 @@ "bignum": "*", "express": "~3.4.7", "jade": "~1.0.2", - "mongoose": "~3.8.3", "lodash": "~2.4.1", "bower": "~1.2.8", "buffertools": "*", diff --git a/test/integration/block.js b/test/integration/block.js index 1b6ea50..3650313 100644 --- a/test/integration/block.js +++ b/test/integration/block.js @@ -27,18 +27,25 @@ describe('BlockDb fromHashWithInfo', function(){ it('return true in has', function(done) { bdb.has(TESTING_BLOCK, function(err, has) { assert.equal(has, true); -console.log('[block.js.29:has:]',has); //TODO done(); }); }); - it('return false in has', function(done) { - bdb.has('111', function(err, has) { - assert.equal(has, false); - done(); + it('setOrphan', function(done) { + var b16 = '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b'; + var b17 = '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743'; + + bdb.has(b17, function(err, has) { + assert(has); + bdb.setOrphan(b17, function(err, oldPrev) { + assert.equal(oldPrev, b16); + bdb.setPrev(b17, b16, function(err, oldPrev) { + bdb.getPrev(b17, function(err, p) { + assert.equal(p, b16); + done(); + }); + }); + }); }); }); - - - }); diff --git a/test/integration/sync.js b/test/integration/sync.js new file mode 100644 index 0000000..f0e2589 --- /dev/null +++ b/test/integration/sync.js @@ -0,0 +1,71 @@ +#!/usr/bin/env node +'use strict'; + +process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + + +var + assert = require('assert'), + async = require('async'), + Sync = require('../../lib/Sync').class(); + + +var b = [ + '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b', //B#16 + '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743', + '000000008d55c3e978639f70af1d2bf1fe6f09cb3143e104405a599215c89a48', + '000000009b3bca4909f38313f2746120129cce4a699a1f552390955da470c5a9', + '00000000ede57f31cc598dc241d129ccb4d8168ef112afbdc870dc60a85f5dd3', //B#20 +]; + +var fix = function(s,cb) { + async.each([1,2,3,4], function(i,c) { + s.blockDb.setPrev(b[i],b[i-1], function() { + return c(); + }); + }, cb); +}; + +var test = function(s,cb) { + async.each([2,3,4], function(i,c) { + s.blockDb.getPrev(b[i], function(err, p) { + assert.equal(p,0); + return c(); + }); + }, function() { + s.blockDb.getPrev(b[1], function(err, p) { + assert.equal(p,b[0]); + return cb(); + }); + }); +}; + + + +var s; +describe('Sync setOrphan', function(){ + + before(function(done) { + s = new Sync(); + fix(s,done); + }); + + after(function(done) { + fix(s,done); + }); + + it('setOrphan', function(done) { + this.timeout(100000); + + s.blockDb.has(b[0], function(err, has) { + assert(has); + s.blockDb.has(b[1], function(err, has) { + assert(has); + s.setOrphan(b[4],b[1], function() { + test(s,done); + }); + }); + }); + }); +}); +