From 0b8c1965e29610815cefb10cfb0fbbac52463be8 Mon Sep 17 00:00:00 2001 From: kenshin-samourai Date: Mon, 29 Mar 2021 16:31:02 +0000 Subject: [PATCH] add support of rest api provided by addrindexrs --- docker/my-dojo/indexer/Dockerfile | 3 +- docker/my-dojo/indexer/restart.sh | 1 + docker/my-dojo/overrides/indexer.install.yaml | 1 + .../local-rest-indexer-wrapper.js | 157 ++++++++++++++++++ lib/remote-importer/sources-mainnet.js | 7 + lib/remote-importer/sources-testnet.js | 7 + static/admin/dmt/xpubs-tools/xpubs-tools.js | 4 +- 7 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 lib/remote-importer/local-rest-indexer-wrapper.js diff --git a/docker/my-dojo/indexer/Dockerfile b/docker/my-dojo/indexer/Dockerfile index d868bee..5e6f81a 100644 --- a/docker/my-dojo/indexer/Dockerfile +++ b/docker/my-dojo/indexer/Dockerfile @@ -1,7 +1,7 @@ FROM rust:1.42.0-slim-buster ENV INDEXER_HOME /home/indexer -ENV INDEXER_VERSION 0.4.0 +ENV INDEXER_VERSION 0.5.0 ENV INDEXER_URL https://code.samourai.io/dojo/addrindexrs.git ARG INDEXER_LINUX_GID @@ -42,5 +42,6 @@ RUN cd "$INDEXER_HOME/addrindexrs" && \ cargo install --locked --path . EXPOSE 50001 +EXPOSE 8080 STOPSIGNAL SIGINT diff --git a/docker/my-dojo/indexer/restart.sh b/docker/my-dojo/indexer/restart.sh index cb1373e..b42770e 100644 --- a/docker/my-dojo/indexer/restart.sh +++ b/docker/my-dojo/indexer/restart.sh @@ -7,6 +7,7 @@ indexer_options=( --jsonrpc-import --db-dir="/home/indexer/db" --indexer-rpc-addr="$NET_DOJO_INDEXER_IPV4:50001" + --indexer-http-addr="$NET_DOJO_INDEXER_IPV4:8080" --daemon-rpc-addr="$BITCOIND_IP:$BITCOIND_RPC_PORT" --cookie="$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD" --txid-limit="$INDEXER_TXID_LIMIT" diff --git a/docker/my-dojo/overrides/indexer.install.yaml b/docker/my-dojo/overrides/indexer.install.yaml index 488dbc1..e2f9a16 100644 --- a/docker/my-dojo/overrides/indexer.install.yaml +++ b/docker/my-dojo/overrides/indexer.install.yaml @@ -18,6 +18,7 @@ services: command: "/wait-for-it.sh tor:9050 --timeout=360 --strict -- /restart.sh" expose: - "50001" + - "8080" volumes: - data-indexer:/home/indexer logging: diff --git a/lib/remote-importer/local-rest-indexer-wrapper.js b/lib/remote-importer/local-rest-indexer-wrapper.js new file mode 100644 index 0000000..0ec7677 --- /dev/null +++ b/lib/remote-importer/local-rest-indexer-wrapper.js @@ -0,0 +1,157 @@ +/*! + * lib/remote-importer/local-rest-indexer-wrapper.js + * Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. + */ +'use strict' + +const axios = require('axios') +const bitcoin = require('bitcoinjs-lib') +const Logger = require('../logger') +const util = require('../util') +const network = require('../bitcoin/network') +const activeNet = network.network +const keys = require('../../keys')[network.key] +const Wrapper = require('./wrapper') + + +/** + * Wrapper for a local indexer + * providing a REST API + */ +class LocalRestIndexerWrapper extends Wrapper { + + /** + * Constructor + */ + constructor(url) { + super(url, null) + } + + /** + * Send a GET request to the API + * @param {string} route + * @returns {Promise} + */ + async _get(route) { + const params = { + url: `${this.base}${route}`, + method: 'GET', + responseType: 'json', + timeout: 15000, + headers: { + 'User-Agent': 'Dojo' + } + } + + const result = await axios(params) + return result.data + } + + /** + * Translate a bitcoin address into a script hash + * (@see https://electrumx.readthedocs.io/en/latest/protocol-basics.html#script-hashes) + * @param {string} address - bitcoin address + * @returns {string} returns the script hash associated to the address + */ + _getScriptHash(address) { + const bScriptPubKey = bitcoin.address.toOutputScript(address, activeNet) + const bScriptHash = bitcoin.crypto.sha256(bScriptPubKey) + return bScriptHash.reverse().toString('hex') + } + + /** + * Retrieve information for a given address + * @param {string} address - bitcoin address + * @param {boolean} filterAddr - True if an upper bound should be used + * for #transactions associated to the address, False otherwise + * @returns {Promise} returns an object + * { address: , txids: , ntx: } + */ + async getAddress(address, filterAddr) { + const ret = { + address: address, + ntx: 0, + txids: [] + } + + const scriptHash = this._getScriptHash(address) + const uri = `/blockchain/scripthash/${scriptHash}/history` + const results = await this._get(uri) + + for (let r of results) { + ret.txids.push(r.tx_hash) + ret.ntx++ + } + + if (filterAddr && ret.ntx > keys.addrFilterThreshold) { + Logger.info(`Importer : Import of ${address} rejected (too many transactions - ${ret.ntx})`) + return { + address: address, + ntx: 0, + txids: [] + } + } + + return ret + } + + /** + * Retrieve information for a given list of addresses + * @param {string} addresses - array of bitcoin addresses + * @param {boolean} filterAddr - True if an upper bound should be used + * for #transactions associated to the address, False otherwise + * @returns {Promise} returns an array of objects + * { address: , txids: , ntx: } + */ + async getAddresses(addresses, filterAddr) { + const ret = {} + const scriptHash2Address = {} + const scriptHashes = [] + + for (let a of addresses) { + const scriptHash = this._getScriptHash(a) + scriptHashes.push(scriptHash) + scriptHash2Address[scriptHash] = a + } + + const sScriptHashes = scriptHashes.join(',') + const uri = `/blockchain/scripthashes/history?scripthashes=${sScriptHashes}` + const results = await this._get(uri) + + for (let r of results) { + const a = scriptHash2Address[r.script_hash] + ret[a] = { + address: a, + ntx: r.txids.length, + txids: r.txids + } + } + + const aRet = Object.values(ret) + + for (let i in aRet) { + if (filterAddr && aRet[i].ntx > keys.addrFilterThreshold) { + Logger.info(`Importer : Import of ${aRet[i].address} rejected (too many transactions - ${aRet[i].ntx})`) + aRet.splice(i, 1) + } + } + + return aRet + } + + /** + * Retrieve the height of the chaintip for the remote source + * @returns {Promise} returns an object + * {chainTipHeight: } + */ + async getChainTipHeight() { + let chainTipHeight = null + const result = await this._get(`/blocks/tip`) + if (result != null && result['height'] != null) + chainTipHeight = parseInt(result['height']) + return {'chainTipHeight': chainTipHeight} + } + +} + +module.exports = LocalRestIndexerWrapper diff --git a/lib/remote-importer/sources-mainnet.js b/lib/remote-importer/sources-mainnet.js index 6850b7f..48ce6a7 100644 --- a/lib/remote-importer/sources-mainnet.js +++ b/lib/remote-importer/sources-mainnet.js @@ -10,6 +10,7 @@ const keys = require('../../keys')[network.key] const Sources = require('./sources') const BitcoindWrapper = require('./bitcoind-wrapper') const LocalIndexerWrapper = require('./local-indexer-wrapper') +const LocalRestIndexerWrapper = require('./local-rest-indexer-wrapper') const OxtWrapper = require('./oxt-wrapper') @@ -40,6 +41,12 @@ class SourcesMainnet extends Sources { // we'll use the local indexer as our unique source this.source = new LocalIndexerWrapper() Logger.info('Importer : Activated local indexer as the data source for imports') + } else if (keys.indexer.active == 'local_rest_indexer') { + // If local rest indexer option is activated + // we'll use the local indexer as our unique source + const uri = `http://${keys.indexer.localIndexer.host}:${keys.indexer.localIndexer.port}` + this.source = new LocalRestIndexerWrapper(uri) + Logger.info('Importer : Activated local indexer (REST API) as the data source for imports') } else { // Otherwise, we'll use the rest api provided by OXT this.source = new OxtWrapper(keys.indexer.oxt) diff --git a/lib/remote-importer/sources-testnet.js b/lib/remote-importer/sources-testnet.js index 3efb3e6..3ee3907 100644 --- a/lib/remote-importer/sources-testnet.js +++ b/lib/remote-importer/sources-testnet.js @@ -11,6 +11,7 @@ const keys = require('../../keys')[network.key] const Sources = require('./sources') const BitcoindWrapper = require('./bitcoind-wrapper') const LocalIndexerWrapper = require('./local-indexer-wrapper') +const LocalRestIndexerWrapper = require('./local-rest-indexer-wrapper') const EsploraWrapper = require('./esplora-wrapper') @@ -41,6 +42,12 @@ class SourcesTestnet extends Sources { // we'll use the local indexer as our unique source this.source = new LocalIndexerWrapper() Logger.info('Importer : Activated local indexer as the data source for imports') + } else if (keys.indexer.active == 'local_rest_indexer') { + // If local rest indexer option is activated + // we'll use the local indexer as our unique source + const uri = `http://${keys.indexer.localIndexer.host}:${keys.indexer.localIndexer.port}` + this.source = new LocalRestIndexerWrapper(uri) + Logger.info('Importer : Activated local indexer (REST API) as the data source for imports') } else { // Otherwise, we'll use the rest api provided by Esplora this.source = new EsploraWrapper(keys.indexer.esplora) diff --git a/static/admin/dmt/xpubs-tools/xpubs-tools.js b/static/admin/dmt/xpubs-tools/xpubs-tools.js index 1858517..20d93cc 100644 --- a/static/admin/dmt/xpubs-tools/xpubs-tools.js +++ b/static/admin/dmt/xpubs-tools/xpubs-tools.js @@ -31,7 +31,9 @@ const screenXpubsToolsScript = { preparePage: function() { // Disable custom lookahead if data source is a third party explorer - const disableLookahead = sessionStorage.getItem('indexerType') == 'third_party_explorer' + const isTPE = sessionStorage.getItem('indexerType') == 'third_party_explorer' + const isLRI = sessionStorage.getItem('indexerType') == 'local_rest_indexer' + const disableLookahead = isTPE || isLRI $('#rescan-lookahead').prop('disabled', disableLookahead) this.hideRescanForm()