Merge pull request #80 from Samourai-Wallet/feat_blocks_rescan

add support of blocks rescans in the maintenance tool

docker/my-dojo/.env

@@ -13,8 +13,8 @@ COMPOSE_CONVERT_WINDOWS_PATHS=1
 DOJO_VERSION_TAG=1.3.0
 DOJO_DB_VERSION_TAG=1.1.0
 DOJO_BITCOIND_VERSION_TAG=1.2.0
-DOJO_NODEJS_VERSION_TAG=1.2.0
-DOJO_NGINX_VERSION_TAG=1.2.0
+DOJO_NODEJS_VERSION_TAG=1.3.0
+DOJO_NGINX_VERSION_TAG=1.3.0
 DOJO_TOR_VERSION_TAG=1.2.0
 
 

docker/my-dojo/docker-compose.yaml

@@ -36,6 +36,7 @@ services:
     expose:
       - "8080"
       - "8081"
+      - "8082"
     volumes:
       - data-nodejs:/data
     depends_on:

docker/my-dojo/nginx/mainnet.conf

@@ -34,6 +34,11 @@ server {
         proxy_pass http://node:8081/;
     }
 
+    # Tracker server is separate, so proxy first
+    location /v2/tracker/ {
+        proxy_pass http://node:8082/;
+    }
+
     # Proxy requests to maintenance tool
     location /admin/ {
         proxy_pass http://node:8080/static/admin/;

docker/my-dojo/nginx/testnet.conf

@@ -34,6 +34,11 @@ server {
         proxy_pass http://node:8081/;
     }
 
+    # Tracker server is separate, so proxy first
+    location /test/v2/tracker/ {
+        proxy_pass http://node:8082/;
+    }
+
     # Proxy requests to maintenance tool
     location /admin/ {
         proxy_pass http://node:8080/static/admin/;

docker/my-dojo/node/keys.index.js

@@ -70,8 +70,10 @@ module.exports = {
     ports: {
       // Port used by the API
       account: 8080,
-      // Port used by pushtx
+      // Port used by the pushtx API
       pushtx: 8081,
+      // Port used by the tracker API
+      trackerApi: 8082,
       // Port used by the tracker for its notifications
       tracker: 5555,
       // Port used by pushtx for its notifications

keys/index-example.js

@@ -67,8 +67,10 @@ module.exports = {
     ports: {
       // Port used by the API
       account: 8080,
-      // Port used by pushtx
+      // Port used by pushtx API
       pushtx: 8081,
+      // Port used by the tracker API
+      trackerApi: 8082,
       // Port used by the tracker for its notifications
       tracker: 5555,
       // Port used by pushtx for its notifications
@@ -270,6 +272,7 @@ module.exports = {
     ports: {
       account: 18080,
       pushtx: 18081,
+      trackerApi: 18082,
       tracker: 15555,
       notifpushtx: 15556,
       orchestrator: 15557

static/admin/lib/api-wrapper.js

@@ -133,7 +133,23 @@ var lib_api = {
     return this.sendGetUriEncoded(uri, {});
   },
 
+  /**
+   * Rescans a range of blocks
+   */
+  getBlocksRescan: function(fromHeight, toHeight) {
+    let prefix = conf['prefixes']['support'];
+    let uri = this.baseUri + '/tracker/' + prefix + '/rescan';
+    //let uri = 'http://127.0.0.1:8082/' + prefix + '/rescan';
+    return this.sendGetUriEncoded(
+      uri,
+      {
+        'fromHeight': fromHeight,
+        'toHeight': toHeight
+      }
+    );
+  },
 
+  
   /**
    * HTTP requests methods
    */

static/admin/tool/index.html

@@ -71,7 +71,10 @@
             <a href="#">UNSPENT</a>
           </li>
           <li id="link-tx">
-            <a href="#">TRANSACTION</a>
+            <a href="#">TX</a>
+          </li>
+          <li id="link-rescan-blocks">
+            <a href="#">BLOCKS RESCAN</a>
           </li>
         </ul>
       </div>

static/admin/tool/index.js

@@ -48,6 +48,7 @@ function initTabs() {
     '#link-xpub',
     '#link-info-address',
     '#link-rescan-address',
+    '#link-rescan-blocks',
     '#link-multiaddr',
     '#link-unspent',
     '#link-tx'
@@ -110,6 +111,12 @@ function preparePage() {
       placeholder = 'ENTER A BITCOIN ADDRESS';
     } else if (activeTab == '#link-rescan-address') {
       placeholder = 'ENTER A BITCOIN ADDRESS';
+    } else if (activeTab == '#link-rescan-blocks') {
+      $("#cell-args").removeClass('fullwidth');
+      $("#cell-args").addClass('halfwidth');
+      $("#cell-args2").show();
+      placeholder = 'RESCAN BLOCKS FROM HEIGHT...';
+      placeholder2 = '...TO HEIGHT (OPTIONAL)';
     } else if (activeTab == '#link-multiaddr') {
       placeholder = 'ENTER /MULTIADDR URL ARGUMENTS (e.g.: active=xpub0123456789&new=address2|address3&pubkey=pubkey4)';
     } else if (activeTab == '#link-unspent') {
@@ -164,6 +171,10 @@ function processAction(activeTab, args, args2, args3) {
     return lib_api.getAddressInfo(args);
   } else if (activeTab == '#link-rescan-address') {
     return lib_api.getAddressRescan(args);
+  } else if (activeTab == '#link-rescan-blocks') {
+    const fromHeight = parseInt(args);
+    const toHeight = (args2) ? parseInt(args2) : fromHeight;
+    return lib_api.getBlocksRescan(fromHeight, toHeight);
   } else if (activeTab == '#link-tx') {
     return lib_api.getTransaction(args);
   }

tracker/blockchain-processor.js

@@ -320,6 +320,39 @@ class BlockchainProcessor extends AbstractProcessor {
     await db.deleteBlocksAfterHeight(height)
   }
 
+  /**
+   * Rescan a range of blocks
+   * @param {integer} fromHeight - height of first block
+   * @param {integer} toHeight - height of last block
+   * @returns {Promise}
+   */
+  async rescanBlocks(fromHeight, toHeight) {
+    // Get highest block processed by the tracker
+    const highest = await db.getHighestBlock()
+    const dbMaxHeight = highest.blockHeight
+
+    if (toHeight == null)
+      toHeight = fromHeight
+
+    toHeight = Math.min(toHeight, dbMaxHeight)
+    const blockRange = _.range(fromHeight, toHeight + 1)
+
+    Logger.info(`Blocks Rescan : starting a rescan for ${blockRange.length} blocks`)
+
+    // Process the blocks
+    return util.seriesCall(blockRange, async height => {
+      try {
+        Logger.info(`Rescanning block ${height}`)
+        const hash = await this.client.getblockhash(height)
+        const header = await this.client.getblockheader(hash)
+        return this.processBlock(header) 
+      } catch(e) {
+        Logger.error(e, 'BlockchainProcessor.rescan()')
+        throw e
+      }
+    }, 'Tracker rescan', true)
+  }
+
   /**
    * Process a block
    * @param {object} header - block header

tracker/index.js

@@ -11,7 +11,9 @@
   const keys = require('../keys')[network.key]
   const db = require('../lib/db/mysql-db-wrapper')
   const Logger = require('../lib/logger')
+  const HttpServer = require('../lib/http-server/http-server')
   const Tracker = require('./tracker')
+  const TrackerRestApi = require('./tracker-rest-api')
 
 
   Logger.info('Process ID: ' + process.pid)
@@ -33,8 +35,20 @@
 
   db.connect(dbConfig)
 
-  // Start the tracker
+  // Initialize the tracker
   const tracker = new Tracker()
+
+  // Initialize the http server
+  const port = keys.ports.trackerApi
+  const httpServer = new HttpServer(port)
+
+  // Initialize the rest api endpoints
+  const trackerRestApi = new TrackerRestApi(httpServer, tracker)
+
+  // Start the http server
+  httpServer.start()
+
+  // Start the tracker
   tracker.start()
 
 })().catch(err => {

tracker/tracker-rest-api.js

@@ -0,0 +1,79 @@
+/*!
+ * tracker/tracker-rest-api.js
+ * Copyright (c) 2016-2019, Samourai Wallet (CC BY-NC-ND 4.0 License).
+ */
+'use strict'
+
+const qs = require('querystring')
+const validator = require('validator')
+const bodyParser = require('body-parser')
+const Logger = require('../lib/logger')
+const errors = require('../lib/errors')
+const authMgr = require('../lib/auth/authorizations-manager')
+const HttpServer = require('../lib/http-server/http-server')
+const network = require('../lib/bitcoin/network')
+const keys = require('../keys')[network.key]
+
+
+/**
+ * Tracker API endpoints
+ */
+class TrackerRestApi {
+
+  /**
+   * Constructor
+   * @param {pushtx.HttpServer} httpServer - HTTP server
+   * @param {tracker.Tracker} tracker - tracker
+   */
+  constructor(httpServer, tracker) {
+    this.httpServer = httpServer
+    this.tracker = tracker
+
+    const urlencodedParser = bodyParser.urlencoded({ extended: true })
+
+    // Establish routes. Proxy server strips /pushtx
+    this.httpServer.app.get(
+      `/${keys.prefixes.support}/rescan`,
+      authMgr.checkHasAdminProfile.bind(authMgr),
+      this.getBlocksRescan.bind(this),
+      HttpServer.sendAuthError
+    )
+  }
+
+  /**
+   * Rescan a range of blocks
+   */
+  async getBlocksRescan(req, res) {
+    // Check request arguments
+    if (!req.query)
+      return HttpServer.sendError(res, errors.body.INVDATA)
+
+    if (!req.query.fromHeight || !validator.isInt(req.query.fromHeight))
+      return HttpServer.sendError(res, errors.body.INVDATA)
+
+    if (req.query.toHeight && !validator.isInt(req.query.toHeight))
+      return HttpServer.sendError(res, errors.body.INVDATA)
+
+    // Retrieve the request arguments
+    const fromHeight = parseInt(req.query.fromHeight)
+    const toHeight = req.query.toHeight ? parseInt(req.query.toHeight) : fromHeight
+
+    if (req.query.toHeight && (toHeight < fromHeight))
+      return HttpServer.sendError(res, errors.body.INVDATA)
+
+    try {
+      await this.tracker.blockchainProcessor.rescanBlocks(fromHeight, toHeight)
+      const ret = {
+        status: 'Rescan complete',
+        fromHeight: fromHeight,
+        toHeight: toHeight
+      }
+      HttpServer.sendRawData(res, JSON.stringify(ret, null, 2))
+    } catch(e) {
+      return HttpServer.sendError(res, e)
+    }
+  }
+
+}
+
+module.exports = TrackerRestApi