diff --git a/Gruntfile.js b/Gruntfile.js index 743d984..ab534a1 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -39,6 +39,13 @@ module.exports = function(grunt) { } } }, + mochaTest: { + options: { + reporter: 'spec', + }, + src: ['test/*.js'] + }, + nodemon: { dev: { options: { @@ -61,13 +68,6 @@ module.exports = function(grunt) { logConcurrentOutput: true } }, - mochaTest: { - options: { - reporter: 'spec', - require: 'server.js' - }, - src: ['test/*.js'] - }, env: { test: { NODE_ENV: 'test' @@ -87,7 +87,7 @@ module.exports = function(grunt) { grunt.option('force', true); //Default task(s). - grunt.registerTask('default', ['jshint', 'concurrent']); + grunt.registerTask('default', ['jshint','concurrent']); //Test task. grunt.registerTask('test', ['env:test', 'mochaTest']); diff --git a/README.md b/README.md index eb30b46..c986815 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,21 @@ $ npm install -g bower Run sync from mystery repository: $ utils/sync.js + +check utils/sync.js --help for options. + ### Blocks ``` /block/[:hash] /block/00000000a967199a2fad0877433c93df785a8d8ce062e5f9b451cd1397bdbf62 ``` +### Transactions +``` + /tx/[:txid] + /tx/525de308971eabd941b139f46c7198b5af9479325c2395db7f2fb5ae8562556c +``` + diff --git a/Sync.js b/Sync.js new file mode 100644 index 0000000..ae7d23e --- /dev/null +++ b/Sync.js @@ -0,0 +1,219 @@ +require('classtool'); + +function spec(b) { + var mongoose = require('mongoose'); + var util = require('util'); + + var RpcClient = require('bitcore/RpcClient').class(); + var networks = require('bitcore/networks'); + var async = require('async'); + + var config = require('./config/config'); + var Block = require('./app/models/Block'); + var Transaction=require('./app/models/Transaction'); + + function Sync(config) { + this.network = config.networkName == 'testnet' ? networks.testnet : networks.livenet; + } + + var progress_bar = function(string, current, total) { + console.log( util.format("\t%s %d/%d [%d%%]", + string, current, total, parseInt(100 * current/total)) + ); + } + + Sync.prototype.getNextBlock = function (blockHash,cb) { + var that = this; + + if ( !blockHash ) { + return cb(); + } + + this.rpc.getBlock(blockHash, function(err, blockInfo) { + if (err) return cb(err); + + if ( ! ( blockInfo.result.height % 1000) ) { + var h = blockInfo.result.height, + d = blockInfo.result.confirmations; + progress_bar('height', h, h + d); + } + + Block.create( blockInfo.result, function(err, inBlock) { + + // E11000 => already exists + if (err && ! err.toString().match(/E11000/)) { + return cb(err); + } + + if (inBlock) { + inBlock.explodeTransactions(function (err) { + return that.getNextBlock(blockInfo.result.nextblockhash, cb); + }); + } + else + return that.getNextBlock(blockInfo.result.nextblockhash, cb); + }); + }); + } + + Sync.prototype.syncBlocks = function (reindex, cb) { + + var that = this; + var genesisHash = this.network.genesisBlock.hash.reverse().toString('hex'); + + console.log("Syncing Blocks..."); + if (reindex) + return this.getNextBlock(genesisHash, cb); + + + Block.findOne({}, {}, { sort: { 'confirmations' : 1 } }, function(err, block) { + if (err) return cb(err); + + var nextHash = + block && block.hash + ? block.hash + : genesisHash + ; + + + console.log('\tStarting at hash: ' + nextHash); + return that.getNextBlock(nextHash, cb); + }); + } + + + Sync.prototype.syncTXs = function (reindex, cb) { + + var that = this; + + console.log("Syncing TXs..."); + if (reindex) { + // TODO? + } + + + Transaction.find({blockhash: null}, function(err, txs) { + if (err) return cb(err); + + var read = 0; + var pull = 0; + var write = 0; + var total = txs.length; + console.log("\tneed to pull %d txs", total); + + if (!total) return cb(); + + async.each(txs, + function(tx, next){ + if (! tx.txid) { + console.log("NO TXID skipping...", tx); + return next(); + } + + if ( ! ( read++ % 1000) ) + progress_bar('read', read, total); + + + that.rpc.getRawTransaction(tx.txid, 1, function(err, txInfo) { + + if ( ! ( pull++ % 1000) ) + progress_bar('\tpull', pull, total); + + if (!err && txInfo) { + Transaction.update({txid: tx.txid}, txInfo.result, function(err) { + if (err) return next(err); + + if ( ! ( write++ % 1000) ) + progress_bar('\t\twrite', write, total); + + return next(); + }); + } + else return next(); + }); + }, + function(err){ + if (err) return cb(err); + return cb(err); + } + ); + }); + } + + Sync.prototype.start = function (opts, next) { + + + mongoose.connect(config.db); + var db = mongoose.connection; + this.rpc = new RpcClient(config.bitcoind); + var that = this; + + + db.on('error', console.error.bind(console, 'connection error:')); + + db.once('open', function (){ + + async.series([ + function(cb){ + if (opts.destroy) { + console.log("Deleting Blocks..."); + return Block.remove().exec(cb); + } + return cb(); + }, + function(cb){ + if (opts.destroy) { + console.log("Deleting TXs..."); + return Transaction.remove().exec(cb); + } + return cb(); + }, + function(cb) { + + if (! opts.skip_blocks) { + that.syncBlocks(opts.reindex, function(err) { + if (err) { + return cb(err); + + } + console.log("\tBlocks done."); + + return cb(); + }); + } + else { + return cb(); + } + }, + function(cb) { + if (! opts.skip_txs) { + that.syncTXs(opts.reindex, function(err) { + if (err) { + return cb(err); + + } + return cb(); + }); + } + else { + return cb(); + } + }, + function(cb) { + db.close(); + return cb(); + }, + ], + function(err) { + if (err) { + db.close(); + return next(err); + } + return next(); + }); + }); + } + return Sync; +}; +module.defineClass(spec); + diff --git a/app/controllers/transactions.js b/app/controllers/transactions.js new file mode 100644 index 0000000..8bf4635 --- /dev/null +++ b/app/controllers/transactions.js @@ -0,0 +1,33 @@ +'use strict'; + + +var Transaction = require('../models/Transaction'); +//, _ = require('lodash'); + + + +/** + * Module dependencies. + */ + + +/** + * Find block by hash ... + */ +exports.transaction = function(req, res, next, txid) { + Transaction.fromID(txid, function(err, tx) { + if (err) return next(err); + if (!tx) return next(new Error('Failed to load TX ' + txid)); + req.transaction = tx; + next(); + }); +}; + + +/** + * Show block + */ +exports.show = function(req, res) { + res.jsonp(req.transaction); +}; + diff --git a/app/models/Block.js b/app/models/Block.js index 53272b5..3ce94ef 100644 --- a/app/models/Block.js +++ b/app/models/Block.js @@ -3,9 +3,11 @@ /** * Module dependencies. */ -var mongoose = require('mongoose'), - Schema = mongoose.Schema; +var mongoose = require('mongoose'), + Schema = mongoose.Schema; +var async = require('async'); +var Transaction = require('./Transaction'); /** * Block Schema @@ -38,6 +40,32 @@ var BlockSchema = new Schema({ }, }); +BlockSchema.methods.explodeTransactions = function(next) { + + // console.log('exploding %s', this.hash, typeof this.tx); + + async.forEach( this.tx, + function(tx, callback) { + // console.log('procesing TX %s', tx); + Transaction.create({ txid: tx }, function(err) { + if (err && ! err.toString().match(/E11000/)) { + return callback(); + } + if (err) { + + return callback(err); + } + return callback(); + + }); + }, + function(err) { + if (err) return next(err); + return next(); + } + ); +}; + /** * Validations */ @@ -65,4 +93,5 @@ BlockSchema.statics.fromHash = function(hash, cb) { }).exec(cb); }; + module.exports = mongoose.model('Block', BlockSchema); diff --git a/app/models/Transaction.js b/app/models/Transaction.js new file mode 100644 index 0000000..566a480 --- /dev/null +++ b/app/models/Transaction.js @@ -0,0 +1,64 @@ +'use strict'; + +/** + * Module dependencies. + */ +var mongoose = require('mongoose'), + Schema = mongoose.Schema; + + +/** + */ +var TransactionSchema = new Schema({ + txid: { + type: String, + index: true, + unique: true, + }, + version: Number, + locktime: Number, + vin: { + type: Array, + default: [], + }, + vout: { + type: Array, + default: [], + }, + blockhash: { + type: String, + index: true, + default: null, + }, + confirmations: Number, + time: Number, + blocktime: Number, +}); + +/** + * Statics + */ + +TransactionSchema.statics.load = function(id, cb) { + this.findOne({ + _id: id + }).exec(cb); +}; + + +TransactionSchema.statics.fromID = function(txid, cb) { + this.findOne({ + txid: txid, + }).exec(cb); +}; + +/* + * virtual + */ + +// ugly? new object every call? +TransactionSchema.virtual('date').get(function () { + return new Date(this.time); +}); + +module.exports = mongoose.model('Transaction', TransactionSchema); diff --git a/config/routes.js b/config/routes.js index 7150af0..507378f 100644 --- a/config/routes.js +++ b/config/routes.js @@ -7,11 +7,13 @@ module.exports = function(app) { app.get('/', index.render); //Block routes - var blocks = require('../app/controllers/blocks'); app.get('/block/:blockHash', blocks.show); - - app.param('blockHash', blocks.block); + + var transactions = require('../app/controllers/transactions'); + app.get('/tx/:txid', transactions.show); + + app.param('txid', transactions.transaction); }; diff --git a/package.json b/package.json index 0fc2f26..1dd908e 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,9 @@ "postinstall": "node node_modules/bower/bin/bower install" }, "dependencies": { + "async": "*", "classtool": "*", + "commander": "*", "express": "~3.4.7", "jade": "~1.0.2", "mongoose": "~3.8.3", diff --git a/util/sync.js b/util/sync.js index 9aff445..22d1a53 100755 --- a/util/sync.js +++ b/util/sync.js @@ -1,89 +1,34 @@ #!/usr/bin/env node process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -require('buffertools').extend(); - -var util = require('util'); -var RpcClient = require('../node_modules/bitcore/RpcClient').class(); -var networks = require('../node_modules/bitcore/networks'); - -var Block = require('../app/models/Block'); -var config = require('../config/config'); -var mongoose = require('mongoose'); - -var networkName = process.argv[2] || 'testnet'; -var network = networkName == 'testnet' ? networks.testnet : networks.livenet; - - -function getNextBlock(blockHash,cb) { - - if ( !blockHash ) { - console.log("done"); - return cb(); - } - - rpc.getBlock(blockHash, function(err, blockInfo) { - if (err) { - return cb(err); - } - - if ( ! ( blockInfo.result.height % 1000) ) { - var h = blockInfo.result.height, - d = blockInfo.result.confirmations; - console.log( util.format("Height: %d/%d [%d%%]", h, d, 100*h/(h+d))); - } - - Block.create( blockInfo.result, function(err, inBlock) { - - // E11000 => already exists - if (err && ! err.toString().match(/E11000/)) { - return cb(err); - } - - return getNextBlock(blockInfo.result.nextblockhash, cb); - }); - }); - -} - -function syncBlocks(network, cb) { +require('buffertools').extend(); - Block.findOne({}, {}, { sort: { 'confirmations' : 1 } }, function(err, block) { - if (err) { - return cb(err); - } +var SYNC_VERSION = '0.1'; +var program = require('commander'); +var Sync = require('../Sync').class(); +program + .version(SYNC_VERSION) + .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet') + .option('-R --reindex', 'Force reindexing', '0') + .option('-D --destroy', 'Remove current DB', '0') + .option('--skip_blocks', 'Sync blocks') + .option('--skip_txs', 'Sync transactions') + .parse(process.argv); +var sync = new Sync({ networkName: program.network }); - var nextHash = - block && block.hash - ? block.hash - : network.genesisBlock.hash.reverse().toString('hex') - ; +if (program.remove) { - - console.log('Starting at hash: ' + nextHash); - getNextBlock(nextHash, cb); - }); } - -mongoose.connect(config.db); - -var db = mongoose.connection; -var rpc = new RpcClient(config.bitcoind); - - -db.on('error', console.error.bind(console, 'connection error:')); -db.once('open', function callback () { - syncBlocks(network, function(err) { - if (err) { - console.log(err); - } - mongoose.connection.close(); - }); +sync.start( program, function(err){ + if (err) { + console.log("CRITICAL ERROR: ", err); + } + else { + console.log('Done!'); + } }); - -