Browse Source
* rebase on top old history * fix dep * more missing deps * updated docs * impl readme api * final tweaks * more tests * revert version * remove dead codev4
Mathias Buus
6 years ago
committed by
GitHub
GitHub
13 changed files with 1410 additions and 1049 deletions
@ -1 +1,2 @@ |
|||
node_modules |
|||
sandbox.js |
|||
|
|||
@ -1,46 +1,50 @@ |
|||
var dht = require('./') |
|||
var blake2b = require('./blake2b') |
|||
|
|||
var node = dht({ |
|||
bootstrap: 'localhost:49737', |
|||
ephemeral: !!process.argv[2] |
|||
}) |
|||
|
|||
var values = {} |
|||
|
|||
node.on('update:store', function (query, cb) { |
|||
console.log('(onupdate)') |
|||
if (!query.value) return cb() |
|||
var key = blake2b(query.value).toString('hex') |
|||
values[key] = query.value |
|||
console.log('Storing', key, '-->', query.value.toString()) |
|||
cb() |
|||
}) |
|||
|
|||
node.on('query:lookup', function (query, cb) { |
|||
console.log('(onquery)') |
|||
var value = values[query.target.toString('hex')] |
|||
cb(null, value) |
|||
}) |
|||
|
|||
if (process.argv.length > 3) { |
|||
var val = process.argv.slice(3).join(' ') |
|||
if (process.argv[2] === 'put') { |
|||
node.update({command: 'store', target: blake2b(Buffer.from(val)), value: val}, function (err) { |
|||
const dht = require('./') |
|||
|
|||
const bootstrap = dht() |
|||
bootstrap.listen(10001) |
|||
|
|||
const nodes = [] |
|||
var swarm = 1000 |
|||
loop(null) |
|||
|
|||
function loop (err) { |
|||
if (err) throw err |
|||
console.log('Inserted', blake2b(Buffer.from(val)).toString('hex')) |
|||
if (swarm--) addNode(loop) |
|||
else done() |
|||
} |
|||
|
|||
function done () { |
|||
console.log('executing hi update') |
|||
|
|||
const i = Math.floor(Math.random() * nodes.length) |
|||
const rs = nodes[i].update('hi', Buffer.alloc(32)) |
|||
|
|||
rs.resume() |
|||
rs.on('end', function () { |
|||
setTimeout(done, 2000) |
|||
}) |
|||
} |
|||
if (process.argv[2] === 'get') { |
|||
node.query({command: 'lookup', target: Buffer.from(val, 'hex')}) |
|||
.on('data', function (data) { |
|||
if (data.value && blake2b(data.value).toString('hex') === val) { |
|||
console.log(val, '-->', data.value.toString()) |
|||
this.destroy() |
|||
} |
|||
|
|||
function addNode (cb) { |
|||
const node = dht({ |
|||
bootstrap: [ |
|||
10001 |
|||
] |
|||
}) |
|||
|
|||
var hits = 0 |
|||
node.command('hi', { |
|||
update (query, cb) { |
|||
console.log('hi', ++hits) |
|||
cb(null) |
|||
}, |
|||
query (query, cb) { |
|||
cb(null) |
|||
} |
|||
}) |
|||
.on('end', function () { |
|||
console.log('(query finished)') |
|||
|
|||
node.once('ready', function () { |
|||
nodes.push(node) |
|||
cb() |
|||
}) |
|||
} |
|||
} |
|||
|
|||
@ -0,0 +1,299 @@ |
|||
const { Message, Holepunch, TYPE } = require('./messages') |
|||
const blake2b = require('./blake2b') |
|||
const peers = require('ipv4-peers') |
|||
const sodium = require('sodium-universal') |
|||
|
|||
const QUERY = Symbol('QUERY') |
|||
const UPDATE = Symbol('UPDATE') |
|||
|
|||
const ECANCELLED = new Error('Request cancelled') |
|||
const ETIMEDOUT = new Error('Request timed out') |
|||
|
|||
ETIMEDOUT.code = 'ETIMEDOUT' |
|||
ECANCELLED.code = 'ECANCELLED' |
|||
|
|||
const TRIES = 3 |
|||
|
|||
class IO { |
|||
constructor (socket, id, ctx) { |
|||
this.id = id |
|||
this.socket = socket |
|||
this.inflight = [] |
|||
|
|||
this._ctx = ctx |
|||
this._rid = (Math.random() * 65536) | 0 |
|||
this._requests = new Array(65536) |
|||
this._pending = [] |
|||
this._secrets = [ randomBytes(32), randomBytes(32) ] |
|||
this._tickInterval = setInterval(this._ontick.bind(this), 250) |
|||
this._rotateInterval = setInterval(this._onrotate.bind(this), 300000) |
|||
|
|||
socket.on('message', this._onmessage.bind(this)) |
|||
} |
|||
|
|||
_token (peer, i) { |
|||
return blake2b.batch([ |
|||
this._secrets[i], |
|||
Buffer.from(peer.host) |
|||
]) |
|||
} |
|||
|
|||
_free () { |
|||
const rid = this._rid++ |
|||
if (this._rid === 65536) this._rid = 0 |
|||
return rid |
|||
} |
|||
|
|||
/* |
|||
|
|||
R |
|||
/ \ |
|||
A B |
|||
|
|||
A sent a message to B that failed |
|||
|
|||
It could be that the message got dropped |
|||
or that it needs holepunching |
|||
|
|||
To retry |
|||
|
|||
resend(req, A -> B) |
|||
fire_and_forget({ _holepunch, to: B }, A -> R) |
|||
|
|||
R.onholepunch { to: B } => fire_and_forget({ _holepunch, from: A }, R -> B) |
|||
B.onholepunch { from: A } => fire_and_forget({ _holepunch }, B -> A) |
|||
|
|||
A and B is now holepunched and the session has been retried as well |
|||
|
|||
*/ |
|||
|
|||
_holepunch (req) { |
|||
const rid = req.message.command === '_holepunch' |
|||
? req.rid |
|||
: this._free() |
|||
|
|||
const punch = { |
|||
type: req.message.type, |
|||
rid, |
|||
command: '_holepunch', |
|||
value: Holepunch.encode({ |
|||
to: peers.encode([ req.peer ]) |
|||
}) |
|||
} |
|||
|
|||
this.send(Message.encode(punch), req.peer.referrer) |
|||
} |
|||
|
|||
_retry (req) { |
|||
req.timeout = 4 |
|||
this.send(req.buffer, req.peer) |
|||
// if referrer is avail, try holepunching automatically
|
|||
if (req.peer.referrer) this._holepunch(req) |
|||
} |
|||
|
|||
_onmessage (buf, rinfo) { |
|||
const message = decodeMessage(buf) |
|||
if (!message) return |
|||
|
|||
const peer = { port: rinfo.port, host: rinfo.address } |
|||
|
|||
switch (message.type) { |
|||
case TYPE.RESPONSE: |
|||
this._ctx.onresponse(message, peer) |
|||
this._finish(message.rid, null, message, peer) |
|||
break |
|||
|
|||
case TYPE.QUERY: |
|||
this._ctx.onrequest(QUERY, message, peer) |
|||
break |
|||
|
|||
case TYPE.UPDATE: |
|||
const rt = message.roundtripToken |
|||
if (!rt || (!rt.equals(this._token(peer, 0)) && !rt.equals(this._token(peer, 1)))) return |
|||
this._ctx.onrequest(UPDATE, message, peer) |
|||
break |
|||
} |
|||
} |
|||
|
|||
_finish (rid, err, val, peer) { |
|||
const req = this._requests[rid] |
|||
if (!req) return |
|||
|
|||
this._requests[rid] = undefined |
|||
const top = this.inflight[this.inflight.length - 1] |
|||
this.inflight[top.index = req.index] = top |
|||
this.inflight.pop() |
|||
|
|||
const type = req.message.type === TYPE.QUERY |
|||
? QUERY |
|||
: UPDATE |
|||
|
|||
req.callback(err, val, peer, req.message, req.peer, type) |
|||
|
|||
while (this._pending.length && this.inflight.length < this._ctx.concurrency) { |
|||
const { message, peer, callback } = this._pending.shift() |
|||
this._requestImmediately(message, peer, callback) |
|||
} |
|||
} |
|||
|
|||
_request (message, peer, callback) { |
|||
// Should we wait to send?
|
|||
if (this._pending.length || (this.inflight.length >= this._ctx.concurrency)) { |
|||
this._pending.push({ message, peer, callback }) |
|||
} else { |
|||
this._requestImmediately(message, peer, callback) |
|||
} |
|||
} |
|||
|
|||
_requestImmediately (message, peer, callback) { |
|||
const rid = message.rid = this._free() |
|||
const buffer = Message.encode(message) |
|||
|
|||
const req = { |
|||
rid, |
|||
index: this.inflight.length, |
|||
callback, |
|||
message, |
|||
buffer, |
|||
peer, |
|||
timeout: 4, |
|||
tries: 0 |
|||
} |
|||
|
|||
this._requests[rid] = req |
|||
this.inflight.push(req) |
|||
this.send(buffer, peer) |
|||
|
|||
// if sending a holepunch cmd, forward it right away
|
|||
if (message.command === '_holepunch') this._holepunch(req) |
|||
} |
|||
|
|||
_cancel (rid, err) { |
|||
this._finish(rid, err || ECANCELLED, null, null) |
|||
} |
|||
|
|||
_onrotate () { |
|||
this._secrets[1] = this._secrets[0] |
|||
this._secrets[0] = randomBytes(32) |
|||
} |
|||
|
|||
_ontick () { |
|||
for (var i = this.inflight.length - 1; i >= 0; i--) { |
|||
const req = this.inflight[i] |
|||
|
|||
if (req.timeout === 2 && ++req.tries < TRIES) { |
|||
this._retry(req) |
|||
continue |
|||
} |
|||
|
|||
if (--req.timeout) { |
|||
continue |
|||
} |
|||
|
|||
this._cancel(req.rid, ETIMEDOUT) |
|||
} |
|||
} |
|||
|
|||
send (buffer, peer) { |
|||
this.socket.send(buffer, 0, buffer.length, peer.port, peer.host) |
|||
} |
|||
|
|||
destroy () { |
|||
clearInterval(this._rotateInterval) |
|||
clearInterval(this._tickInterval) |
|||
|
|||
this.socket.close() |
|||
|
|||
const pending = this._pending |
|||
this._pending = [] |
|||
|
|||
for (const req of pending) req.callback(ECANCELLED) |
|||
for (const req of this.inflight) this._cancel(req.rid) |
|||
} |
|||
|
|||
response (request, value, closerNodes, peer) { |
|||
const message = { |
|||
type: TYPE.RESPONSE, |
|||
rid: request.rid, |
|||
id: this.id, |
|||
closerNodes, |
|||
roundtripToken: this._token(peer, 0), |
|||
value |
|||
} |
|||
|
|||
this.send(Message.encode(message), peer) |
|||
} |
|||
|
|||
error (request, error, closerNodes, peer) { |
|||
const message = { |
|||
type: TYPE.RESPONSE, |
|||
rid: request.rid, |
|||
id: this.id, |
|||
closerNodes, |
|||
error: error.message |
|||
} |
|||
|
|||
this.send(Message.encode(message), peer) |
|||
} |
|||
|
|||
query (command, target, value, peer, callback) { |
|||
if (!callback) callback = noop |
|||
|
|||
this._request({ |
|||
type: TYPE.QUERY, |
|||
rid: 0, |
|||
id: this.id, |
|||
target, |
|||
command, |
|||
value |
|||
}, peer, callback) |
|||
} |
|||
|
|||
queryImmediately (command, target, value, peer, callback) { |
|||
if (!callback) callback = noop |
|||
|
|||
this._requestImmediately({ |
|||
type: TYPE.QUERY, |
|||
rid: 0, |
|||
id: this.id, |
|||
target, |
|||
command, |
|||
value |
|||
}, peer, callback) |
|||
} |
|||
|
|||
update (command, target, value, peer, callback) { |
|||
if (!callback) callback = noop |
|||
|
|||
this._request({ |
|||
type: TYPE.UPDATE, |
|||
rid: 0, |
|||
id: this.id, |
|||
roundtripToken: peer.roundtripToken, |
|||
target, |
|||
command, |
|||
value |
|||
}, peer, callback) |
|||
} |
|||
} |
|||
|
|||
IO.QUERY = QUERY |
|||
IO.UPDATE = UPDATE |
|||
|
|||
module.exports = IO |
|||
|
|||
function noop () {} |
|||
|
|||
function decodeMessage (buf) { |
|||
try { |
|||
return Message.decode(buf) |
|||
} catch (err) { |
|||
return null |
|||
} |
|||
} |
|||
|
|||
function randomBytes (n) { |
|||
const buf = Buffer.allocUnsafe(32) |
|||
sodium.randombytes_buf(buf) |
|||
return buf |
|||
} |
|||
@ -0,0 +1,264 @@ |
|||
const { Readable } = require('stream') |
|||
const nodes = require('ipv4-peers').idLength(32) |
|||
const QueryTable = require('./query-table') |
|||
|
|||
const BOOTSTRAPPING = Symbol('BOOTSTRAPPING') |
|||
const MOVING_CLOSER = Symbol('MOVING_CLOSER') |
|||
const UPDATING = Symbol('UPDATING') |
|||
const FINALIZED = Symbol('FINALIZED') |
|||
|
|||
class QueryStream extends Readable { |
|||
constructor (node, command, target, value, opts) { |
|||
if (!opts) opts = {} |
|||
if (!opts.concurrency) opts.concurrency = opts.highWaterMark || node.concurrency |
|||
|
|||
super({ |
|||
objectMode: true, |
|||
highWaterMark: opts.concurrency |
|||
}) |
|||
|
|||
const cmd = node._commands.get(command) |
|||
|
|||
this.command = command |
|||
this.target = target |
|||
this.value = cmd ? cmd.inputEncoding.encode(value) : (value || null) |
|||
this.update = !!opts.update |
|||
this.query = !!opts.query || !opts.update |
|||
this.destroyed = false |
|||
this.inflight = 0 |
|||
this.responses = 0 |
|||
this.updates = 0 |
|||
this.table = opts.table || new QueryTable(node.id, target) |
|||
|
|||
node.inflightQueries++ |
|||
|
|||
this._status = opts.table ? MOVING_CLOSER : BOOTSTRAPPING |
|||
this._node = node |
|||
this._concurrency = opts.concurrency |
|||
this._callback = this._onresponse.bind(this) |
|||
this._map = identity |
|||
this._outputEncoding = cmd ? cmd.outputEncoding : null |
|||
} |
|||
|
|||
map (fn) { |
|||
this._map = fn |
|||
return this |
|||
} |
|||
|
|||
_onresponse (err, message, peer, request, to, type) { |
|||
this.inflight-- |
|||
|
|||
if (err && to.id) { |
|||
// Request, including retries, failed completely
|
|||
// Remove the "to" node.
|
|||
const node = this._node.bucket.get(to.id) |
|||
if (node) this._node._removeNode(node) |
|||
} |
|||
|
|||
if (this._status === FINALIZED) { |
|||
if (!this.inflight) { |
|||
if (this.destroyed) this.emit('close') |
|||
else this.destroy() |
|||
} |
|||
return |
|||
} |
|||
|
|||
if (err) { |
|||
this.emit('warning', err) |
|||
this._readMaybe() |
|||
return |
|||
} |
|||
|
|||
this.responses++ |
|||
this.table.addVerified(message, peer) |
|||
|
|||
if (this._status === MOVING_CLOSER) { |
|||
const candidates = decodeNodes(message.closerNodes) |
|||
for (var i = 0; i < candidates.length; i++) { |
|||
this.table.addUnverified(candidates[i], peer) |
|||
} |
|||
} else if (this._status === UPDATING) { |
|||
this.updates++ |
|||
} |
|||
|
|||
if (message.error) { |
|||
this.emit('warning', new Error(message.error)) |
|||
this._readMaybe() |
|||
return |
|||
} |
|||
|
|||
if (!this.query && this._status === MOVING_CLOSER) { |
|||
this._readMaybe() |
|||
return |
|||
} |
|||
|
|||
const value = this._outputEncoding |
|||
? this._decodeOutput(message.value) |
|||
: message.value |
|||
|
|||
const data = this._map({ |
|||
type, |
|||
node: { |
|||
id: message.id, |
|||
port: peer.port, |
|||
host: peer.host |
|||
}, |
|||
value |
|||
}) |
|||
|
|||
if (!data) { |
|||
this._readMaybe() |
|||
return |
|||
} |
|||
|
|||
this.push(data) |
|||
} |
|||
|
|||
_decodeOutput (val) { |
|||
try { |
|||
return val && this._outputEncoding.decode(val) |
|||
} catch (err) { |
|||
return null |
|||
} |
|||
} |
|||
|
|||
_bootstrap () { |
|||
const table = this.table |
|||
const bootstrap = this._node.bucket.closest(table.target, table.k) |
|||
var i = 0 |
|||
|
|||
for (; i < bootstrap.length; i++) { |
|||
const b = bootstrap[i] |
|||
const node = { id: b.id, port: b.port, host: b.host } |
|||
table.addUnverified(node, null) |
|||
} |
|||
|
|||
const bootstrapNodes = this._node.bootstrapNodes |
|||
if (bootstrap.length < bootstrapNodes.length) { |
|||
for (i = 0; i < bootstrapNodes.length; i++) { |
|||
this._send(bootstrapNodes[i], true, false) |
|||
} |
|||
} |
|||
|
|||
this._status = MOVING_CLOSER |
|||
this._moveCloser() |
|||
} |
|||
|
|||
_sendAll (nodes, force, sendToken) { |
|||
var free = Math.max(0, this._concurrency - this._node._io.inflight.length) |
|||
var sent = 0 |
|||
|
|||
if (!free && !this.inflight) free = 1 |
|||
if (!free) return 0 |
|||
|
|||
for (var i = 0; i < nodes.length; i++) { |
|||
if (this._send(nodes[i], force, sendToken)) { |
|||
if (++sent === free) break |
|||
} |
|||
} |
|||
|
|||
return sent |
|||
} |
|||
|
|||
_send (node, force, isUpdate) { |
|||
if (!force) { |
|||
if (node.queried) return false |
|||
node.queried = true |
|||
} |
|||
|
|||
this.inflight++ |
|||
const io = this._node._io |
|||
|
|||
if (isUpdate) { |
|||
if (!node.roundtripToken) return this._callback(new Error('Roundtrip token is required')) |
|||
io.update(this.command, this.target, this.value, node, this._callback) |
|||
} else if (this.query) { |
|||
io.query(this.command, this.target, this.value, node, this._callback) |
|||
} else { |
|||
io.query('_find_node', this.target, null, node, this._callback) |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
_sendUpdate () { |
|||
const sent = this._sendAll(this.table.closest, false, true) |
|||
if (sent || this.inflight) return |
|||
|
|||
this._finalize() |
|||
} |
|||
|
|||
_moveCloser () { |
|||
const table = this.table |
|||
const sent = this._sendAll(table.unverified, false, false) |
|||
if (sent || this.inflight) return |
|||
|
|||
if (this.update) { |
|||
for (var i = 0; i < table.closest.length; i++) { |
|||
table.closest[i].queried = false |
|||
} |
|||
this._status = UPDATING |
|||
this._sendUpdate() |
|||
} else { |
|||
this._finalize() |
|||
} |
|||
} |
|||
|
|||
_finalize () { |
|||
const status = this._status |
|||
if (status === FINALIZED) return |
|||
|
|||
this._status = FINALIZED |
|||
this._node.inflightQueries-- |
|||
|
|||
if (!this.responses && !this.destroyed) { |
|||
this.destroy(new Error('No nodes responded')) |
|||
} |
|||
if (status === UPDATING && !this.updates && !this.destroyed) { |
|||
this.destroy(new Error('No close nodes responded')) |
|||
} |
|||
|
|||
this.push(null) |
|||
} |
|||
|
|||
_readMaybe () { |
|||
if (!this.inflight || this._readableState.flowing === true) this._read() |
|||
} |
|||
|
|||
_read () { |
|||
if (this._node.destroyed) return |
|||
|
|||
switch (this._status) { |
|||
case BOOTSTRAPPING: return this._bootstrap() |
|||
case MOVING_CLOSER: return this._moveCloser() |
|||
case UPDATING: return this._sendUpdate() |
|||
case FINALIZED: return |
|||
} |
|||
|
|||
throw new Error('Unknown status: ' + this._status) |
|||
} |
|||
|
|||
destroy (err) { |
|||
if (this.destroyed) return |
|||
this.destroyed = true |
|||
|
|||
if (err) this.emit('error', err) |
|||
this._finalize() |
|||
if (!this.inflight) this.emit('close') |
|||
} |
|||
} |
|||
|
|||
module.exports = QueryStream |
|||
|
|||
function decodeNodes (buf) { |
|||
if (!buf) return [] |
|||
try { |
|||
return nodes.decode(buf) |
|||
} catch (err) { |
|||
return [] |
|||
} |
|||
} |
|||
|
|||
function identity (a) { |
|||
return a |
|||
} |
|||
@ -0,0 +1,69 @@ |
|||
const xor = require('xor-distance') |
|||
|
|||
class QueryTable { |
|||
constructor (id, target) { |
|||
this.k = 20 |
|||
this.id = id |
|||
this.target = target |
|||
this.closest = [] |
|||
this.unverified = [] |
|||
} |
|||
|
|||
addUnverified (node, referrer) { |
|||
if (node.id.equals(this.id)) return |
|||
|
|||
node.distance = xor(this.target, node.id) |
|||
node.referrer = referrer |
|||
|
|||
insertSorted(node, this.k, this.unverified) |
|||
} |
|||
|
|||
addVerified (message, peer) { |
|||
if (!message.id || !message.roundtripToken || message.id.equals(this.id)) { |
|||
return |
|||
} |
|||
|
|||
var prev = getNode(message.id, this.unverified) |
|||
|
|||
if (!prev) { |
|||
prev = { |
|||
id: message.id, |
|||
host: peer.host, |
|||
port: peer.port, |
|||
distance: xor(message.id, this.target) |
|||
} |
|||
} |
|||
|
|||
prev.roundtripToken = message.roundtripToken |
|||
insertSorted(prev, this.k, this.closest) |
|||
} |
|||
} |
|||
|
|||
module.exports = QueryTable |
|||
|
|||
function getNode (id, list) { |
|||
// find id in the list.
|
|||
// technically this would be faster with binary search (against distance)
|
|||
// but this list is always small, so meh
|
|||
|
|||
for (var i = 0; i < list.length; i++) { |
|||
if (list[i].id.equals(id)) return list[i] |
|||
} |
|||
|
|||
return null |
|||
} |
|||
|
|||
function insertSorted (node, max, list) { |
|||
if (list.length === max && !xor.lt(node.distance, list[max - 1].distance)) return |
|||
if (getNode(node.id, list)) return |
|||
|
|||
if (list.length < max) list.push(node) |
|||
else list[max - 1] = node |
|||
|
|||
var pos = list.length - 1 |
|||
while (pos && xor.gt(list[pos - 1].distance, node.distance)) { |
|||
list[pos] = list[pos - 1] |
|||
list[pos - 1] = node |
|||
pos-- |
|||
} |
|||
} |
|||
@ -1,38 +1,35 @@ |
|||
{ |
|||
"name": "dht-rpc", |
|||
"version": "3.0.1", |
|||
"description": "Make RPC calls over a Kademlia based DHT.", |
|||
"description": "Make RPC calls over a Kademlia based DHT", |
|||
"main": "index.js", |
|||
"dependencies": { |
|||
"duplexify": "^3.5.0", |
|||
"inherits": "^2.0.3", |
|||
"ipv4-peers": "^1.1.1", |
|||
"k-bucket": "^5.0.0", |
|||
"protocol-buffers-encodings": "^1.1.0", |
|||
"readable-stream": "^2.1.5", |
|||
"sodium-universal": "^2.0.0", |
|||
"stream-collector": "^1.0.1", |
|||
"time-ordered-set": "^1.0.1", |
|||
"udp-request": "^1.3.0", |
|||
"xor-distance": "^1.0.0" |
|||
}, |
|||
"devDependencies": { |
|||
"protocol-buffers": "^3.2.1", |
|||
"standard": "^11.0.1", |
|||
"tape": "^4.9.0" |
|||
"scripts": { |
|||
"test": "standard && tape test.js", |
|||
"protobuf": "protocol-buffers schema.proto -o lib/messages.js" |
|||
}, |
|||
"repository": { |
|||
"type": "git", |
|||
"url": "https://github.com/mafintosh/dht-rpc.git" |
|||
}, |
|||
"scripts": { |
|||
"test": "standard && tape test.js", |
|||
"protobuf": "protocol-buffers schema.proto -o messages.js" |
|||
"url": "git+https://github.com/mafintosh/dht-rpc.git" |
|||
}, |
|||
"author": "Mathias Buus (@mafintosh)", |
|||
"license": "MIT", |
|||
"bugs": { |
|||
"url": "https://github.com/mafintosh/dht-rpc/issues" |
|||
}, |
|||
"homepage": "https://github.com/mafintosh/dht-rpc" |
|||
"homepage": "https://github.com/mafintosh/dht-rpc#readme", |
|||
"devDependencies": { |
|||
"protocol-buffers": "^4.1.0", |
|||
"standard": "^12.0.1", |
|||
"tape": "^4.9.1" |
|||
}, |
|||
"dependencies": { |
|||
"codecs": "^1.2.1", |
|||
"ipv4-peers": "^1.1.1", |
|||
"k-bucket": "^5.0.0", |
|||
"protocol-buffers-encodings": "^1.1.0", |
|||
"sodium-universal": "^2.0.0", |
|||
"stream-collector": "^1.0.1", |
|||
"time-ordered-set": "^1.0.1", |
|||
"xor-distance": "^1.0.0" |
|||
} |
|||
} |
|||
|
|||
@ -1,304 +0,0 @@ |
|||
var stream = require('readable-stream') |
|||
var inherits = require('inherits') |
|||
var nodes = require('ipv4-peers').idLength(32) |
|||
var xor = require('xor-distance') |
|||
|
|||
var BLANK = Buffer.from([ |
|||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
|||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 |
|||
]) |
|||
|
|||
module.exports = QueryStream |
|||
|
|||
function QueryStream (dht, query, opts) { |
|||
if (!(this instanceof QueryStream)) return new QueryStream(dht, query, opts) |
|||
if (!opts) opts = {} |
|||
if (!opts.concurrency) opts.concurrency = opts.highWaterMark || dht.concurrency |
|||
if (!query.target) throw new Error('query.target is required') |
|||
|
|||
stream.Readable.call(this, {objectMode: true, highWaterMark: opts.concurrency}) |
|||
|
|||
var self = this |
|||
var nodes = opts.node || opts.nodes |
|||
|
|||
this.query = query |
|||
this.query.id = dht._queryId |
|||
this.target = query.target |
|||
this.token = !!opts.token |
|||
this.holepunching = opts.holepunching !== false |
|||
this.commits = 0 |
|||
this.responses = 0 |
|||
this.errors = 0 |
|||
this.destroyed = false |
|||
this.verbose = !!opts.verbose |
|||
|
|||
dht.inflightQueries++ |
|||
|
|||
this._dht = dht |
|||
this._committing = false |
|||
this._finalized = false |
|||
this._closest = opts.closest || [] |
|||
this._concurrency = opts.concurrency |
|||
this._updating = false |
|||
this._pending = nodes ? [].concat(nodes).map(copyNode) : [] |
|||
this._k = nodes ? Infinity : opts.k || 20 |
|||
this._inflight = 0 |
|||
this._moveCloser = !nodes |
|||
this._map = opts.map || echo |
|||
this._bootstrapped = !this._moveCloser |
|||
this._onresponse = onresponse |
|||
this._onresponseholepunch = onresponseholepunch |
|||
|
|||
function onresponseholepunch (err, res, peer, query) { |
|||
if (!err || !peer || !peer.referrer) self._callback(err, res, peer) |
|||
else self._holepunch(peer, query) |
|||
} |
|||
|
|||
function onresponse (err, res, peer) { |
|||
self._callback(err, res, peer) |
|||
} |
|||
} |
|||
|
|||
inherits(QueryStream, stream.Readable) |
|||
|
|||
QueryStream.prototype.destroy = function (err) { |
|||
if (this.destroyed) return |
|||
this.destroyed = true |
|||
this._finalize() |
|||
if (err) this.emit('error', err) |
|||
this.emit('close') |
|||
} |
|||
|
|||
QueryStream.prototype._finalize = function () { |
|||
if (this._finalized) return |
|||
this._finalized = true |
|||
this._dht.inflightQueries-- |
|||
if (!this.responses && !this.destroyed) this.destroy(new Error('No nodes responded')) |
|||
if (!this.commits && this._committing && !this.destroyed) this.destroy(new Error('No close nodes responded')) |
|||
this.push(null) |
|||
} |
|||
|
|||
QueryStream.prototype._bootstrap = function () { |
|||
this._bootstrapped = true |
|||
|
|||
var bootstrap = this._dht.bucket.closest(this.target, this._k) |
|||
var i = 0 |
|||
|
|||
for (i = 0; i < bootstrap.length; i++) { |
|||
var b = bootstrap[i] |
|||
this._addPending({id: b.id, port: b.port, host: b.host}, null) |
|||
} |
|||
|
|||
if (bootstrap.length < this._dht._bootstrap.length) { |
|||
for (i = 0; i < this._dht._bootstrap.length; i++) { |
|||
this._send(this._dht._bootstrap[i], true, false) |
|||
} |
|||
} |
|||
} |
|||
|
|||
QueryStream.prototype._readMaybe = function () { |
|||
if (this._readableState.flowing === true) this._read() |
|||
} |
|||
|
|||
QueryStream.prototype._sendTokens = function () { |
|||
if (this.destroyed) return |
|||
|
|||
var sent = this._sendAll(this._closest, false, true) |
|||
if (sent || this._inflight) return |
|||
|
|||
this._finalize() |
|||
} |
|||
|
|||
QueryStream.prototype._sendPending = function () { |
|||
if (this.destroyed) return |
|||
if (!this._bootstrapped) this._bootstrap() |
|||
|
|||
var sent = this._sendAll(this._pending, false, false) |
|||
if (sent || this._inflight) return |
|||
|
|||
if (this.token) { |
|||
for (var i = 0; i < this._closest.length; i++) this._closest[i].queried = false |
|||
this._committing = true |
|||
this._sendTokens() |
|||
} else { |
|||
this._finalize() |
|||
} |
|||
} |
|||
|
|||
QueryStream.prototype._read = function () { |
|||
if (this._committing) this._sendTokens() |
|||
else this._sendPending() |
|||
} |
|||
|
|||
QueryStream.prototype._holepunch = function (peer, query) { |
|||
var self = this |
|||
|
|||
this._dht._holepunch(peer, peer.referrer, function (err) { |
|||
if (err) return self._callback(err, null, peer) |
|||
self._dht._request(query, peer, false, self._onresponse) |
|||
}) |
|||
} |
|||
|
|||
QueryStream.prototype._callback = function (err, res, peer) { |
|||
this._inflight-- |
|||
if (this.destroyed) return |
|||
|
|||
if (err) { |
|||
if (res && res.id) { |
|||
var node = this._dht.bucket.get(res.id) |
|||
if (node) this._dht._removeNode(node) |
|||
} |
|||
this.errors++ |
|||
this.emit('warning', err) |
|||
this._readMaybe() |
|||
return |
|||
} |
|||
|
|||
this.responses++ |
|||
if (this._committing) this.commits++ |
|||
this._addClosest(res, peer) |
|||
|
|||
if (this._moveCloser) { |
|||
var candidates = decodeNodes(res.nodes) |
|||
for (var i = 0; i < candidates.length; i++) this._addPending(candidates[i], peer) |
|||
} |
|||
|
|||
if (!validateId(res.id) || (this.token && !this.verbose && !this._committing)) { |
|||
this._readMaybe() |
|||
return |
|||
} |
|||
|
|||
var data = this._map({ |
|||
node: { |
|||
id: res.id, |
|||
port: peer.port, |
|||
host: peer.host |
|||
}, |
|||
value: res.value |
|||
}) |
|||
|
|||
if (!data) { |
|||
this._readMaybe() |
|||
return |
|||
} |
|||
|
|||
this.push(data) |
|||
} |
|||
|
|||
QueryStream.prototype._sendAll = function (nodes, force, useToken) { |
|||
var sent = 0 |
|||
var free = Math.max(0, this._concurrency - this._dht.socket.inflight) |
|||
|
|||
if (!free && !this._inflight) free = 1 |
|||
if (!free) return 0 |
|||
|
|||
for (var i = 0; i < nodes.length; i++) { |
|||
if (this._send(nodes[i], force, useToken)) { |
|||
if (++sent === free) break |
|||
} |
|||
} |
|||
|
|||
return sent |
|||
} |
|||
|
|||
QueryStream.prototype._send = function (node, force, useToken) { |
|||
if (!force) { |
|||
if (node.queried) return false |
|||
node.queried = true |
|||
} |
|||
|
|||
this._inflight++ |
|||
|
|||
var query = this.query |
|||
|
|||
if (useToken && node.roundtripToken) { |
|||
query = { |
|||
command: this.query.command, |
|||
id: this.query.id, |
|||
target: this.query.target, |
|||
value: this.query.value, |
|||
roundtripToken: node.roundtripToken |
|||
} |
|||
} |
|||
|
|||
this._dht._request(query, node, false, this.holepunching ? this._onresponseholepunch : this._onresponse) |
|||
return true |
|||
} |
|||
|
|||
QueryStream.prototype._addPending = function (node, ref) { |
|||
if (node.id.equals(this._dht.id)) return |
|||
node.distance = xor(this.target, node.id) |
|||
node.referrer = ref |
|||
insertSorted(node, this._k, this._pending) |
|||
} |
|||
|
|||
QueryStream.prototype._addClosest = function (res, peer) { |
|||
if (!res.id || !res.roundtripToken || res.id.equals(this._dht.id)) return |
|||
|
|||
var prev = getNode(res.id, this._pending) |
|||
|
|||
if (!prev) { |
|||
prev = { |
|||
id: res.id, |
|||
port: peer.port, |
|||
host: peer.host, |
|||
distance: xor(res.id, this.target) |
|||
} |
|||
} |
|||
|
|||
prev.roundtripToken = res.roundtripToken |
|||
insertSorted(prev, this._k, this._closest) |
|||
} |
|||
|
|||
function decodeNodes (buf) { |
|||
if (!buf) return [] |
|||
try { |
|||
return nodes.decode(buf) |
|||
} catch (err) { |
|||
return [] |
|||
} |
|||
} |
|||
|
|||
function getNode (id, list) { |
|||
// find id in the list.
|
|||
// technically this would be faster with binary search (against distance)
|
|||
// but this list is always small, so meh
|
|||
for (var i = 0; i < list.length; i++) { |
|||
if (list[i].id.equals(id)) return list[i] |
|||
} |
|||
|
|||
return null |
|||
} |
|||
|
|||
function validateId (id) { |
|||
return id && id.length === 32 |
|||
} |
|||
|
|||
function insertSorted (node, max, list) { |
|||
if (list.length === max && !xor.lt(node.distance, list[max - 1].distance)) return |
|||
if (getNode(node.id, list)) return |
|||
|
|||
if (list.length < max) list.push(node) |
|||
else list[max - 1] = node |
|||
|
|||
var pos = list.length - 1 |
|||
while (pos && xor.gt(list[pos - 1].distance, node.distance)) { |
|||
list[pos] = list[pos - 1] |
|||
list[pos - 1] = node |
|||
pos-- |
|||
} |
|||
} |
|||
|
|||
function copyNode (node) { |
|||
return { |
|||
id: node.id || BLANK, |
|||
port: node.port, |
|||
host: node.host, |
|||
roundtripToken: node.roundtripToken, |
|||
referrer: node.referrer || node.referer |
|||
} |
|||
} |
|||
|
|||
function echo (a) { |
|||
return a |
|||
} |
|||
@ -1,16 +1,27 @@ |
|||
message Request { |
|||
required string command = 1; |
|||
optional bytes id = 2; |
|||
optional bytes target = 3; |
|||
optional bytes forwardRequest = 4; |
|||
optional bytes forwardResponse = 5; |
|||
optional bytes roundtripToken = 6; |
|||
optional bytes value = 7; |
|||
message Holepunch { |
|||
optional bytes from = 2; |
|||
optional bytes to = 3; |
|||
} |
|||
|
|||
enum TYPE { |
|||
QUERY = 1; |
|||
UPDATE = 2; |
|||
RESPONSE = 3; |
|||
} |
|||
|
|||
message Response { |
|||
optional bytes id = 1; |
|||
optional bytes nodes = 2; |
|||
optional bytes value = 3; |
|||
optional bytes roundtripToken = 4; |
|||
message Message { |
|||
// request/response type + id |
|||
required TYPE type = 1; |
|||
required uint64 rid = 2; |
|||
|
|||
// kademlia stuff |
|||
optional bytes id = 3; |
|||
optional bytes target = 4; |
|||
optional bytes closerNodes = 5; |
|||
optional bytes roundtripToken = 6; |
|||
|
|||
// rpc stuff |
|||
optional string command = 7; |
|||
optional string error = 8; |
|||
optional bytes value = 9; |
|||
} |
|||
|
|||
Loading…
Reference in new issue