kenshin samourai
5 years ago
committed by
GitHub
GitHub
16 changed files with 447 additions and 306 deletions
@ -0,0 +1,224 @@ |
|||
/*! |
|||
* lib/indexer_rpc/rpc-client.js |
|||
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. |
|||
*/ |
|||
'use strict' |
|||
|
|||
const net = require('net') |
|||
const makeConcurrent = require('make-concurrent') |
|||
const network = require('../bitcoin/network') |
|||
const keys = require('../../keys')[network.key] |
|||
|
|||
|
|||
/** |
|||
* RPC client for an indexer |
|||
* following the Electrum protocol |
|||
*/ |
|||
class RpcClient { |
|||
|
|||
/** |
|||
* Constructor |
|||
*/ |
|||
constructor(opts) { |
|||
this._opts = { |
|||
host: keys.indexer.localIndexer.host, |
|||
port: keys.indexer.localIndexer.port, |
|||
concurrency: Infinity, |
|||
timeout: 10000 |
|||
} |
|||
|
|||
if (this._opts.concurrency !== Infinity) { |
|||
this._call = makeConcurrent( |
|||
this._call.bind(this), |
|||
{concurrency: this._opts.concurrency} |
|||
) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Set an option |
|||
* @param {string} key |
|||
* @param {*} value |
|||
* @return {RpcClient} |
|||
*/ |
|||
set(key, value) { |
|||
this._opts[key] = value |
|||
return this |
|||
} |
|||
|
|||
/** |
|||
* Get an option |
|||
* @param {string} key |
|||
* @return {*} |
|||
*/ |
|||
get(key) { |
|||
return this._opts[key] |
|||
} |
|||
|
|||
/** |
|||
* Check if an error returned by the wrapper |
|||
* is a connection error. |
|||
* @param {string} err - error message |
|||
* @returns {boolean} returns true if message related to a connection error |
|||
*/ |
|||
static isConnectionError(err) { |
|||
if (typeof err != 'string') |
|||
return false |
|||
|
|||
const isTimeoutError = (err.indexOf('connect ETIMEDOUT') != -1) |
|||
const isConnRejected = (err.indexOf('Connection Rejected') != -1) |
|||
|
|||
return (isTimeoutError || isConnRejected) |
|||
} |
|||
|
|||
/** |
|||
* Check if the rpc api is ready to process requests |
|||
* @returns {Promise} |
|||
*/ |
|||
static async waitForIndexerRpcApi(opts) { |
|||
let client = new RpcClient(opts) |
|||
|
|||
try { |
|||
await client.sendRequest('server.version', 'dojo', ['1.0', '1.4']) |
|||
} catch(e) { |
|||
client = null |
|||
Logger.info('Indexer RPC API is still unreachable. New attempt in 20s.') |
|||
return util.delay(20000).then(() => { |
|||
return RpcClient.waitForIndexerRpcApi() |
|||
}) |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* Send multiple requests (batch mode) |
|||
* @param {Object[]} batch - array of objects {method: ..., params: ...} |
|||
* @return {Promise} |
|||
*/ |
|||
async sendBatch(batch) { |
|||
return this._call(batch, true) |
|||
} |
|||
|
|||
/** |
|||
* Send multiple requests (flood mode) |
|||
* @param {Object[]} batch - array of objects {method: ..., params: ...} |
|||
* @return {Promise} |
|||
*/ |
|||
async sendRequests(batch) { |
|||
return this._call(batch, false) |
|||
} |
|||
|
|||
/** |
|||
* Send a request |
|||
* @param {string} method - called method |
|||
* @return {Promise} |
|||
*/ |
|||
async sendRequest(method, ...params) { |
|||
const batch = [{method: method, params: params}] |
|||
const ret = await this._call(batch, false) |
|||
return ret[0] |
|||
} |
|||
|
|||
/** |
|||
* Send requests (internal method) |
|||
* @param {Object[]} data - array of objects {method: ..., params: ...} |
|||
* @returns {Promise} |
|||
*/ |
|||
async _call(data, batched) { |
|||
return new Promise((resolve, reject) => { |
|||
let methodId = 0 |
|||
let requests = [] |
|||
let responses = [] |
|||
let response = '' |
|||
|
|||
const requestErrorMsg = `Indexer JSON-RPC: host=${this._opts.host} port=${this._opts.port}:` |
|||
|
|||
// Prepare an array of requests
|
|||
requests = data.map(req => { |
|||
return JSON.stringify({ |
|||
jsonrpc: '2.0', |
|||
method: req.method, |
|||
params: req.params || [], |
|||
id: methodId++ |
|||
}) |
|||
}) |
|||
|
|||
// If batch mode
|
|||
// send a single batched request
|
|||
if (batched) |
|||
requests = [`[${requests.join(',')}]`] |
|||
|
|||
// Initialize the connection
|
|||
const conn = net.Socket() |
|||
conn.setTimeout(this._opts.timeout) |
|||
conn.setEncoding('utf8') |
|||
conn.setKeepAlive(true, 0) |
|||
conn.setNoDelay(true) |
|||
|
|||
conn.on('connect', () => { |
|||
// Send the requests
|
|||
for (let request of requests) |
|||
conn.write(request + '\n') |
|||
}) |
|||
|
|||
conn.on('timeout', () => { |
|||
const e = new Error('ETIMEDOUT') |
|||
e.errorno = 'ETIMEDOUT' |
|||
e.code = 'ETIMEDOUT' |
|||
e.connect = false |
|||
conn.emit('error', e) |
|||
}) |
|||
|
|||
conn.on('data', chunk => { |
|||
// Process the received chunk char by char
|
|||
for (let c of chunk) { |
|||
response += c |
|||
// Detect the end of a response
|
|||
if (c == '\n') { |
|||
try { |
|||
// Parse the response
|
|||
let parsed = JSON.parse(response) |
|||
if (parsed.error) |
|||
throw new Error(JSON.stringify(parsed.error)) |
|||
// Add the parsed reponse to the array of responses
|
|||
if (batched) { |
|||
responses = parsed.map(p => { return {idxAddr: p.id, txs: p.result} }) |
|||
} else { |
|||
responses.push({idxAddr: parsed.id, txs: parsed.result}) |
|||
} |
|||
// Reset the response
|
|||
response = '' |
|||
// If all responses have been received
|
|||
// close the connection
|
|||
if (responses.length == data.length) |
|||
conn.end() |
|||
} catch (err) { |
|||
reject( |
|||
new Error(`${requestErrorMsg} Error Parsing JSON: ${err.message}, data: ${response}`) |
|||
) |
|||
} |
|||
} |
|||
} |
|||
}) |
|||
|
|||
conn.on('end', e => { |
|||
// Connection closed
|
|||
// we can return the responses
|
|||
resolve(responses) |
|||
}) |
|||
|
|||
conn.on('error', e => { |
|||
reject(new Error(`${requestErrorMsg} Request error: ${e}`)) |
|||
}) |
|||
|
|||
// Connect to the RPC API
|
|||
conn.connect({ |
|||
host: this._opts.host, |
|||
port: this._opts.port |
|||
}) |
|||
|
|||
}) |
|||
} |
|||
|
|||
} |
|||
|
|||
module.exports = RpcClient |
|||
@ -1,179 +0,0 @@ |
|||
/*! |
|||
* lib/remote-importer\btccom-wrapper.js |
|||
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. |
|||
*/ |
|||
'use strict' |
|||
|
|||
const rp = require('request-promise-native') |
|||
const addrHelper = require('../bitcoin/addresses-helper') |
|||
const util = require('../util') |
|||
const Logger = require('../logger') |
|||
const network = require('../bitcoin/network') |
|||
const keys = require('../../keys')[network.key] |
|||
const Wrapper = require('./wrapper') |
|||
|
|||
|
|||
/** |
|||
* Wrapper for the btc.com block explorer APIs |
|||
*/ |
|||
class BtcComWrapper extends Wrapper { |
|||
|
|||
/** |
|||
* Constructor |
|||
*/ |
|||
constructor(url) { |
|||
super(url, keys.explorers.socks5Proxy) |
|||
} |
|||
|
|||
/** |
|||
* Send a GET request to the API |
|||
* @param {string} route |
|||
* @returns {Promise} |
|||
*/ |
|||
async _get(route) { |
|||
const params = { |
|||
url: `${this.base}${route}`, |
|||
method: 'GET', |
|||
json: true, |
|||
timeout: 15000 |
|||
} |
|||
|
|||
// Sets socks proxy agent if required
|
|||
if (keys.explorers.socks5Proxy != null) |
|||
params['agent'] = this.socksProxyAgent |
|||
|
|||
return rp(params) |
|||
} |
|||
|
|||
/** |
|||
* Get a page of transactions related to a given address |
|||
* @param {string} address - bitcoin address |
|||
* @param {integer} page - page index |
|||
* @returns {Promise} |
|||
*/ |
|||
async _getTxsForAddress(address, page) { |
|||
const uri = `/address/${address}/tx?page=${page}&verbose=1` |
|||
const results = await this._get(uri) |
|||
return results.data.list.map(tx => tx.hash) |
|||
} |
|||
|
|||
/** |
|||
* Retrieve information for a given address |
|||
* @param {string} address - bitcoin address |
|||
* @param {boolean} filterAddr - True if an upper bound should be used |
|||
* for #transactions associated to the address, False otherwise |
|||
* @returns {Promise} returns an object |
|||
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>} |
|||
*/ |
|||
async getAddress(address, filterAddr) { |
|||
const reqAddr = addrHelper.isBech32(address) |
|||
? addrHelper.getScriptHashFromBech32(address) |
|||
: address |
|||
|
|||
const uri = `/address/${reqAddr}` |
|||
const result = await this._get(uri) |
|||
|
|||
const ret = { |
|||
address: address, |
|||
ntx: result.data.tx_count, |
|||
txids: [] |
|||
} |
|||
|
|||
// Check if we should filter this address
|
|||
if (filterAddr && ret.ntx > keys.addrFilterThreshold) { |
|||
Logger.info(` import of ${ret.address} rejected (too many transactions - ${ret.ntx})`) |
|||
return ret |
|||
} |
|||
|
|||
const nbPagesApi = Math.ceil(ret.ntx / BtcComWrapper.NB_TXS_PER_PAGE) |
|||
const nbPages = Math.min(20, nbPagesApi) |
|||
|
|||
const aPages = new Array(nbPages) |
|||
const listPages = Array.from(aPages, (val, idx) => idx + 1) |
|||
|
|||
const results = await util.seriesCall(listPages, idx => { |
|||
return this._getTxsForAddress(reqAddr, idx) |
|||
}) |
|||
|
|||
for (let txids of results) |
|||
ret.txids = ret.txids.concat(txids) |
|||
|
|||
return ret |
|||
} |
|||
|
|||
/** |
|||
* Retrieve information for a given list of addresses |
|||
* @param {string[]} addresses - array of bitcoin addresses |
|||
* @param {boolean} filterAddr - True if an upper bound should be used |
|||
* for #transactions associated to the address, False otherwise |
|||
* @returns {Promise} returns an array of objects |
|||
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>} |
|||
*/ |
|||
async getAddresses(addresses, filterAddr) { |
|||
const ret = [] |
|||
const reqAddresses = [] |
|||
const xlatedBech32Addr = {} |
|||
|
|||
for (let a of addresses) { |
|||
if (addrHelper.isBech32(a)) { |
|||
const scriptHash = addrHelper.getScriptHashFromBech32(a) |
|||
reqAddresses.push(scriptHash) |
|||
xlatedBech32Addr[scriptHash] = a |
|||
} else { |
|||
reqAddresses.push(a) |
|||
} |
|||
} |
|||
|
|||
// Send a batch request for all the addresses
|
|||
const strReqAddresses = reqAddresses.join(',') |
|||
const uri = `/address/${strReqAddresses}` |
|||
const results = await this._get(uri) |
|||
|
|||
const foundAddresses = Array.isArray(results.data) |
|||
? results.data |
|||
: [results.data] |
|||
|
|||
for (let a of foundAddresses) { |
|||
if (a && a.tx_count > 0) { |
|||
// Translate bech32 address
|
|||
const address = xlatedBech32Addr.hasOwnProperty(a.address) |
|||
? xlatedBech32Addr[a.address] |
|||
: a.address |
|||
|
|||
if (a.tx_count <= 2) { |
|||
// Less than 3 transactions for this address
|
|||
// all good
|
|||
const retAddr = { |
|||
address: address, |
|||
ntx: a.tx_count, |
|||
txids: [] |
|||
} |
|||
|
|||
retAddr.txids = (a.tx_count == 1) |
|||
? [a.first_tx] |
|||
: [a.first_tx, a.last_tx] |
|||
|
|||
ret.push(retAddr) |
|||
|
|||
} else { |
|||
// More than 2 transactions for this address
|
|||
// We need more requests to the API
|
|||
if (filterAddr && a.tx_count > keys.addrFilterThreshold) { |
|||
Logger.info(` import of ${address} rejected (too many transactions - ${a.tx_count})`) |
|||
} else { |
|||
const retAddr = await this.getAddress(address) |
|||
ret.push(retAddr) |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
return ret |
|||
} |
|||
|
|||
} |
|||
|
|||
// BTC.COM acepts a max of 50txs per page
|
|||
BtcComWrapper.NB_TXS_PER_PAGE = 50 |
|||
|
|||
module.exports = BtcComWrapper |
|||
@ -1,90 +0,0 @@ |
|||
/*! |
|||
* lib/remote-importer/insight-wrapper.js |
|||
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. |
|||
*/ |
|||
'use strict' |
|||
|
|||
const rp = require('request-promise-native') |
|||
const Logger = require('../logger') |
|||
const network = require('../bitcoin/network') |
|||
const keys = require('../../keys')[network.key] |
|||
const Wrapper = require('./wrapper') |
|||
|
|||
|
|||
/** |
|||
* Wrapper for the Insight block explorer APIs |
|||
*/ |
|||
class InsightWrapper extends Wrapper { |
|||
|
|||
/** |
|||
* Constructor |
|||
*/ |
|||
constructor(url) { |
|||
super(url, keys.explorers.socks5Proxy) |
|||
} |
|||
|
|||
/** |
|||
* Send a GET request to the API |
|||
* @param {string} route |
|||
* @returns {Promise} |
|||
*/ |
|||
async _get(route) { |
|||
const params = { |
|||
url: `${this.base}${route}`, |
|||
method: 'GET', |
|||
json: true, |
|||
timeout: 15000 |
|||
} |
|||
|
|||
// Sets socks proxy agent if required
|
|||
if (keys.explorers.socks5Proxy != null) |
|||
params['agent'] = this.socksProxyAgent |
|||
|
|||
return rp(params) |
|||
} |
|||
|
|||
/** |
|||
* Retrieve information for a given address |
|||
* @param {string} address - bitcoin address |
|||
* @param {boolean} filterAddr - True if an upper bound should be used |
|||
* for #transactions associated to the address, False otherwise |
|||
* @returns {Promise} returns an object |
|||
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>} |
|||
*/ |
|||
async getAddress(address, filterAddr) { |
|||
const uri = `/addr/${address}` |
|||
// Param filterAddr isn't used for insight
|
|||
const result = await this._get(uri) |
|||
|
|||
const ret = { |
|||
address: result.addrStr, |
|||
txids: [], |
|||
ntx: result.txApperances |
|||
} |
|||
|
|||
// Check if we should filter this address
|
|||
if (filterAddr && ret.ntx > keys.addrFilterThreshold) { |
|||
Logger.info(` import of ${ret.address} rejected (too many transactions - ${ret.ntx})`) |
|||
return ret |
|||
} |
|||
|
|||
ret.txids = result.transactions |
|||
return ret |
|||
} |
|||
|
|||
/** |
|||
* Retrieve information for a given list of addresses |
|||
* @param {string} addresses - array of bitcoin addresses |
|||
* @param {boolean} filterAddr - True if an upper bound should be used |
|||
* for #transactions associated to the address, False otherwise |
|||
* @returns {Promise} returns an array of objects |
|||
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>} |
|||
*/ |
|||
async getAddresses(addresses, filterAddr) { |
|||
// Not implemented for this api
|
|||
throw "Not implemented" |
|||
} |
|||
|
|||
} |
|||
|
|||
module.exports = InsightWrapper |
|||
@ -0,0 +1,143 @@ |
|||
/*! |
|||
* lib/remote-importer/local-indexer-wrapper.js |
|||
* Copyright © 2019 – Katana Cryptographic Ltd. All Rights Reserved. |
|||
*/ |
|||
'use strict' |
|||
|
|||
const bitcoin = require('bitcoinjs-lib') |
|||
const Logger = require('../logger') |
|||
const util = require('../util') |
|||
const network = require('../bitcoin/network') |
|||
const activeNet = network.network |
|||
const keys = require('../../keys')[network.key] |
|||
const RpcClient = require('../indexer-rpc/rpc-client') |
|||
const Wrapper = require('./wrapper') |
|||
|
|||
|
|||
/** |
|||
* Wrapper for a local indexer |
|||
* Currently supports indexers |
|||
* providing a RPC API compliant |
|||
* with a subset of the electrum protocol |
|||
*/ |
|||
class LocalIndexerWrapper extends Wrapper { |
|||
|
|||
/** |
|||
* Constructor |
|||
*/ |
|||
constructor() { |
|||
super(null, null) |
|||
// RPC client
|
|||
this.client = new RpcClient() |
|||
} |
|||
|
|||
/** |
|||
* Translate a bitcoin address into a script hash |
|||
* (@see https://electrumx.readthedocs.io/en/latest/protocol-basics.html#script-hashes)
|
|||
* @param {string} address - bitcoin address |
|||
* @returns {string} returns the script hash associated to the address |
|||
*/ |
|||
_getScriptHash(address) { |
|||
const bScriptPubKey = bitcoin.address.toOutputScript(address, activeNet) |
|||
const bScriptHash = bitcoin.crypto.sha256(bScriptPubKey) |
|||
return util.reverseBuffer(bScriptHash).toString('hex') |
|||
} |
|||
|
|||
/** |
|||
* Retrieve information for a given address |
|||
* @param {string} address - bitcoin address |
|||
* @param {boolean} filterAddr - True if an upper bound should be used |
|||
* for #transactions associated to the address, False otherwise |
|||
* @returns {Promise} returns an object |
|||
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>} |
|||
*/ |
|||
async getAddress(address, filterAddr) { |
|||
const ret = { |
|||
address: address, |
|||
ntx: 0, |
|||
txids: [] |
|||
} |
|||
|
|||
const scriptHash = this._getScriptHash(address) |
|||
|
|||
const results = await this.client.sendRequest( |
|||
LocalIndexerWrapper.GET_HISTORY_RPC_CMD, |
|||
scriptHash |
|||
) |
|||
|
|||
for (let r of results.txs) { |
|||
ret.txids.push(r.tx_hash) |
|||
ret.ntx++ |
|||
} |
|||
|
|||
if (filterAddr && ret.ntx > keys.addrFilterThreshold) { |
|||
Logger.info(` import of ${address} rejected (too many transactions - ${ret.ntx})`) |
|||
return { |
|||
address: address, |
|||
ntx: 0, |
|||
txids: [] |
|||
} |
|||
} |
|||
|
|||
return ret |
|||
} |
|||
|
|||
/** |
|||
* Retrieve information for a given list of addresses |
|||
* @param {string} addresses - array of bitcoin addresses |
|||
* @param {boolean} filterAddr - True if an upper bound should be used |
|||
* for #transactions associated to the address, False otherwise |
|||
* @returns {Promise} returns an array of objects |
|||
* { address: <bitcoin_address>, txids: <txids>, ntx: <total_nb_txs>} |
|||
*/ |
|||
async getAddresses(addresses, filterAddr) { |
|||
const ret = {} |
|||
|
|||
// Build an array of script hashes
|
|||
const scriptHashes = addresses.map(a => this._getScriptHash(a)) |
|||
|
|||
// Build an array of commands
|
|||
const commands = scriptHashes.map(s => { |
|||
return { |
|||
method: LocalIndexerWrapper.GET_HISTORY_RPC_CMD, |
|||
params: [s] |
|||
} |
|||
}) |
|||
|
|||
// Send the requests
|
|||
const results = (keys.indexer.localIndexer.batchRequests == 'active') |
|||
? await this.client.sendBatch(commands) |
|||
: await this.client.sendRequests(commands) |
|||
|
|||
for (let r of results) { |
|||
const addr = addresses[r.idxAddr] |
|||
const txids = r.txs.map(t => t.tx_hash) |
|||
|
|||
ret[addr] = { |
|||
address: addr, |
|||
ntx: txids.length, |
|||
txids: txids |
|||
} |
|||
} |
|||
|
|||
const aRet = Object.values(ret) |
|||
|
|||
for (let i in aRet) { |
|||
if (filterAddr && aRet[i].ntx > keys.addrFilterThreshold) { |
|||
Logger.info(` import of ${aRet[i].address} rejected (too many transactions - ${aRet[i].ntx})`) |
|||
aRet.splice(i, 1) |
|||
} |
|||
} |
|||
|
|||
return aRet |
|||
} |
|||
|
|||
} |
|||
|
|||
/** |
|||
* Get history RPC command (Electrum protocol) |
|||
*/ |
|||
LocalIndexerWrapper.GET_HISTORY_RPC_CMD = 'blockchain.scripthash.get_history' |
|||
|
|||
|
|||
module.exports = LocalIndexerWrapper |
|||
Loading…
Reference in new issue