Browse Source
* rebase on top old history * fix dep * more missing deps * updated docs * impl readme api * final tweaks * more tests * revert version * remove dead codev4
Mathias Buus
7 years ago
committed by
GitHub
GitHub
13 changed files with 1410 additions and 1049 deletions
@ -1 +1,2 @@ |
|||||
node_modules |
node_modules |
||||
|
sandbox.js |
||||
|
|||||
@ -1,46 +1,50 @@ |
|||||
var dht = require('./') |
const dht = require('./') |
||||
var blake2b = require('./blake2b') |
|
||||
|
const bootstrap = dht() |
||||
var node = dht({ |
bootstrap.listen(10001) |
||||
bootstrap: 'localhost:49737', |
|
||||
ephemeral: !!process.argv[2] |
const nodes = [] |
||||
}) |
var swarm = 1000 |
||||
|
loop(null) |
||||
var values = {} |
|
||||
|
function loop (err) { |
||||
node.on('update:store', function (query, cb) { |
|
||||
console.log('(onupdate)') |
|
||||
if (!query.value) return cb() |
|
||||
var key = blake2b(query.value).toString('hex') |
|
||||
values[key] = query.value |
|
||||
console.log('Storing', key, '-->', query.value.toString()) |
|
||||
cb() |
|
||||
}) |
|
||||
|
|
||||
node.on('query:lookup', function (query, cb) { |
|
||||
console.log('(onquery)') |
|
||||
var value = values[query.target.toString('hex')] |
|
||||
cb(null, value) |
|
||||
}) |
|
||||
|
|
||||
if (process.argv.length > 3) { |
|
||||
var val = process.argv.slice(3).join(' ') |
|
||||
if (process.argv[2] === 'put') { |
|
||||
node.update({command: 'store', target: blake2b(Buffer.from(val)), value: val}, function (err) { |
|
||||
if (err) throw err |
if (err) throw err |
||||
console.log('Inserted', blake2b(Buffer.from(val)).toString('hex')) |
if (swarm--) addNode(loop) |
||||
|
else done() |
||||
|
} |
||||
|
|
||||
|
function done () { |
||||
|
console.log('executing hi update') |
||||
|
|
||||
|
const i = Math.floor(Math.random() * nodes.length) |
||||
|
const rs = nodes[i].update('hi', Buffer.alloc(32)) |
||||
|
|
||||
|
rs.resume() |
||||
|
rs.on('end', function () { |
||||
|
setTimeout(done, 2000) |
||||
}) |
}) |
||||
} |
} |
||||
if (process.argv[2] === 'get') { |
|
||||
node.query({command: 'lookup', target: Buffer.from(val, 'hex')}) |
function addNode (cb) { |
||||
.on('data', function (data) { |
const node = dht({ |
||||
if (data.value && blake2b(data.value).toString('hex') === val) { |
bootstrap: [ |
||||
console.log(val, '-->', data.value.toString()) |
10001 |
||||
this.destroy() |
] |
||||
|
}) |
||||
|
|
||||
|
var hits = 0 |
||||
|
node.command('hi', { |
||||
|
update (query, cb) { |
||||
|
console.log('hi', ++hits) |
||||
|
cb(null) |
||||
|
}, |
||||
|
query (query, cb) { |
||||
|
cb(null) |
||||
} |
} |
||||
}) |
}) |
||||
.on('end', function () { |
|
||||
console.log('(query finished)') |
node.once('ready', function () { |
||||
|
nodes.push(node) |
||||
|
cb() |
||||
}) |
}) |
||||
} |
|
||||
} |
} |
||||
|
|||||
@ -0,0 +1,299 @@ |
|||||
|
const { Message, Holepunch, TYPE } = require('./messages') |
||||
|
const blake2b = require('./blake2b') |
||||
|
const peers = require('ipv4-peers') |
||||
|
const sodium = require('sodium-universal') |
||||
|
|
||||
|
const QUERY = Symbol('QUERY') |
||||
|
const UPDATE = Symbol('UPDATE') |
||||
|
|
||||
|
const ECANCELLED = new Error('Request cancelled') |
||||
|
const ETIMEDOUT = new Error('Request timed out') |
||||
|
|
||||
|
ETIMEDOUT.code = 'ETIMEDOUT' |
||||
|
ECANCELLED.code = 'ECANCELLED' |
||||
|
|
||||
|
const TRIES = 3 |
||||
|
|
||||
|
class IO { |
||||
|
constructor (socket, id, ctx) { |
||||
|
this.id = id |
||||
|
this.socket = socket |
||||
|
this.inflight = [] |
||||
|
|
||||
|
this._ctx = ctx |
||||
|
this._rid = (Math.random() * 65536) | 0 |
||||
|
this._requests = new Array(65536) |
||||
|
this._pending = [] |
||||
|
this._secrets = [ randomBytes(32), randomBytes(32) ] |
||||
|
this._tickInterval = setInterval(this._ontick.bind(this), 250) |
||||
|
this._rotateInterval = setInterval(this._onrotate.bind(this), 300000) |
||||
|
|
||||
|
socket.on('message', this._onmessage.bind(this)) |
||||
|
} |
||||
|
|
||||
|
_token (peer, i) { |
||||
|
return blake2b.batch([ |
||||
|
this._secrets[i], |
||||
|
Buffer.from(peer.host) |
||||
|
]) |
||||
|
} |
||||
|
|
||||
|
_free () { |
||||
|
const rid = this._rid++ |
||||
|
if (this._rid === 65536) this._rid = 0 |
||||
|
return rid |
||||
|
} |
||||
|
|
||||
|
/* |
||||
|
|
||||
|
R |
||||
|
/ \ |
||||
|
A B |
||||
|
|
||||
|
A sent a message to B that failed |
||||
|
|
||||
|
It could be that the message got dropped |
||||
|
or that it needs holepunching |
||||
|
|
||||
|
To retry |
||||
|
|
||||
|
resend(req, A -> B) |
||||
|
fire_and_forget({ _holepunch, to: B }, A -> R) |
||||
|
|
||||
|
R.onholepunch { to: B } => fire_and_forget({ _holepunch, from: A }, R -> B) |
||||
|
B.onholepunch { from: A } => fire_and_forget({ _holepunch }, B -> A) |
||||
|
|
||||
|
A and B is now holepunched and the session has been retried as well |
||||
|
|
||||
|
*/ |
||||
|
|
||||
|
_holepunch (req) { |
||||
|
const rid = req.message.command === '_holepunch' |
||||
|
? req.rid |
||||
|
: this._free() |
||||
|
|
||||
|
const punch = { |
||||
|
type: req.message.type, |
||||
|
rid, |
||||
|
command: '_holepunch', |
||||
|
value: Holepunch.encode({ |
||||
|
to: peers.encode([ req.peer ]) |
||||
|
}) |
||||
|
} |
||||
|
|
||||
|
this.send(Message.encode(punch), req.peer.referrer) |
||||
|
} |
||||
|
|
||||
|
_retry (req) { |
||||
|
req.timeout = 4 |
||||
|
this.send(req.buffer, req.peer) |
||||
|
// if referrer is avail, try holepunching automatically
|
||||
|
if (req.peer.referrer) this._holepunch(req) |
||||
|
} |
||||
|
|
||||
|
_onmessage (buf, rinfo) { |
||||
|
const message = decodeMessage(buf) |
||||
|
if (!message) return |
||||
|
|
||||
|
const peer = { port: rinfo.port, host: rinfo.address } |
||||
|
|
||||
|
switch (message.type) { |
||||
|
case TYPE.RESPONSE: |
||||
|
this._ctx.onresponse(message, peer) |
||||
|
this._finish(message.rid, null, message, peer) |
||||
|
break |
||||
|
|
||||
|
case TYPE.QUERY: |
||||
|
this._ctx.onrequest(QUERY, message, peer) |
||||
|
break |
||||
|
|
||||
|
case TYPE.UPDATE: |
||||
|
const rt = message.roundtripToken |
||||
|
if (!rt || (!rt.equals(this._token(peer, 0)) && !rt.equals(this._token(peer, 1)))) return |
||||
|
this._ctx.onrequest(UPDATE, message, peer) |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
_finish (rid, err, val, peer) { |
||||
|
const req = this._requests[rid] |
||||
|
if (!req) return |
||||
|
|
||||
|
this._requests[rid] = undefined |
||||
|
const top = this.inflight[this.inflight.length - 1] |
||||
|
this.inflight[top.index = req.index] = top |
||||
|
this.inflight.pop() |
||||
|
|
||||
|
const type = req.message.type === TYPE.QUERY |
||||
|
? QUERY |
||||
|
: UPDATE |
||||
|
|
||||
|
req.callback(err, val, peer, req.message, req.peer, type) |
||||
|
|
||||
|
while (this._pending.length && this.inflight.length < this._ctx.concurrency) { |
||||
|
const { message, peer, callback } = this._pending.shift() |
||||
|
this._requestImmediately(message, peer, callback) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
_request (message, peer, callback) { |
||||
|
// Should we wait to send?
|
||||
|
if (this._pending.length || (this.inflight.length >= this._ctx.concurrency)) { |
||||
|
this._pending.push({ message, peer, callback }) |
||||
|
} else { |
||||
|
this._requestImmediately(message, peer, callback) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
_requestImmediately (message, peer, callback) { |
||||
|
const rid = message.rid = this._free() |
||||
|
const buffer = Message.encode(message) |
||||
|
|
||||
|
const req = { |
||||
|
rid, |
||||
|
index: this.inflight.length, |
||||
|
callback, |
||||
|
message, |
||||
|
buffer, |
||||
|
peer, |
||||
|
timeout: 4, |
||||
|
tries: 0 |
||||
|
} |
||||
|
|
||||
|
this._requests[rid] = req |
||||
|
this.inflight.push(req) |
||||
|
this.send(buffer, peer) |
||||
|
|
||||
|
// if sending a holepunch cmd, forward it right away
|
||||
|
if (message.command === '_holepunch') this._holepunch(req) |
||||
|
} |
||||
|
|
||||
|
_cancel (rid, err) { |
||||
|
this._finish(rid, err || ECANCELLED, null, null) |
||||
|
} |
||||
|
|
||||
|
_onrotate () { |
||||
|
this._secrets[1] = this._secrets[0] |
||||
|
this._secrets[0] = randomBytes(32) |
||||
|
} |
||||
|
|
||||
|
_ontick () { |
||||
|
for (var i = this.inflight.length - 1; i >= 0; i--) { |
||||
|
const req = this.inflight[i] |
||||
|
|
||||
|
if (req.timeout === 2 && ++req.tries < TRIES) { |
||||
|
this._retry(req) |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
if (--req.timeout) { |
||||
|
continue |
||||
|
} |
||||
|
|
||||
|
this._cancel(req.rid, ETIMEDOUT) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
send (buffer, peer) { |
||||
|
this.socket.send(buffer, 0, buffer.length, peer.port, peer.host) |
||||
|
} |
||||
|
|
||||
|
destroy () { |
||||
|
clearInterval(this._rotateInterval) |
||||
|
clearInterval(this._tickInterval) |
||||
|
|
||||
|
this.socket.close() |
||||
|
|
||||
|
const pending = this._pending |
||||
|
this._pending = [] |
||||
|
|
||||
|
for (const req of pending) req.callback(ECANCELLED) |
||||
|
for (const req of this.inflight) this._cancel(req.rid) |
||||
|
} |
||||
|
|
||||
|
response (request, value, closerNodes, peer) { |
||||
|
const message = { |
||||
|
type: TYPE.RESPONSE, |
||||
|
rid: request.rid, |
||||
|
id: this.id, |
||||
|
closerNodes, |
||||
|
roundtripToken: this._token(peer, 0), |
||||
|
value |
||||
|
} |
||||
|
|
||||
|
this.send(Message.encode(message), peer) |
||||
|
} |
||||
|
|
||||
|
error (request, error, closerNodes, peer) { |
||||
|
const message = { |
||||
|
type: TYPE.RESPONSE, |
||||
|
rid: request.rid, |
||||
|
id: this.id, |
||||
|
closerNodes, |
||||
|
error: error.message |
||||
|
} |
||||
|
|
||||
|
this.send(Message.encode(message), peer) |
||||
|
} |
||||
|
|
||||
|
query (command, target, value, peer, callback) { |
||||
|
if (!callback) callback = noop |
||||
|
|
||||
|
this._request({ |
||||
|
type: TYPE.QUERY, |
||||
|
rid: 0, |
||||
|
id: this.id, |
||||
|
target, |
||||
|
command, |
||||
|
value |
||||
|
}, peer, callback) |
||||
|
} |
||||
|
|
||||
|
queryImmediately (command, target, value, peer, callback) { |
||||
|
if (!callback) callback = noop |
||||
|
|
||||
|
this._requestImmediately({ |
||||
|
type: TYPE.QUERY, |
||||
|
rid: 0, |
||||
|
id: this.id, |
||||
|
target, |
||||
|
command, |
||||
|
value |
||||
|
}, peer, callback) |
||||
|
} |
||||
|
|
||||
|
update (command, target, value, peer, callback) { |
||||
|
if (!callback) callback = noop |
||||
|
|
||||
|
this._request({ |
||||
|
type: TYPE.UPDATE, |
||||
|
rid: 0, |
||||
|
id: this.id, |
||||
|
roundtripToken: peer.roundtripToken, |
||||
|
target, |
||||
|
command, |
||||
|
value |
||||
|
}, peer, callback) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
IO.QUERY = QUERY |
||||
|
IO.UPDATE = UPDATE |
||||
|
|
||||
|
module.exports = IO |
||||
|
|
||||
|
function noop () {} |
||||
|
|
||||
|
function decodeMessage (buf) { |
||||
|
try { |
||||
|
return Message.decode(buf) |
||||
|
} catch (err) { |
||||
|
return null |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function randomBytes (n) { |
||||
|
const buf = Buffer.allocUnsafe(32) |
||||
|
sodium.randombytes_buf(buf) |
||||
|
return buf |
||||
|
} |
||||
@ -0,0 +1,264 @@ |
|||||
|
const { Readable } = require('stream') |
||||
|
const nodes = require('ipv4-peers').idLength(32) |
||||
|
const QueryTable = require('./query-table') |
||||
|
|
||||
|
const BOOTSTRAPPING = Symbol('BOOTSTRAPPING') |
||||
|
const MOVING_CLOSER = Symbol('MOVING_CLOSER') |
||||
|
const UPDATING = Symbol('UPDATING') |
||||
|
const FINALIZED = Symbol('FINALIZED') |
||||
|
|
||||
|
class QueryStream extends Readable { |
||||
|
constructor (node, command, target, value, opts) { |
||||
|
if (!opts) opts = {} |
||||
|
if (!opts.concurrency) opts.concurrency = opts.highWaterMark || node.concurrency |
||||
|
|
||||
|
super({ |
||||
|
objectMode: true, |
||||
|
highWaterMark: opts.concurrency |
||||
|
}) |
||||
|
|
||||
|
const cmd = node._commands.get(command) |
||||
|
|
||||
|
this.command = command |
||||
|
this.target = target |
||||
|
this.value = cmd ? cmd.inputEncoding.encode(value) : (value || null) |
||||
|
this.update = !!opts.update |
||||
|
this.query = !!opts.query || !opts.update |
||||
|
this.destroyed = false |
||||
|
this.inflight = 0 |
||||
|
this.responses = 0 |
||||
|
this.updates = 0 |
||||
|
this.table = opts.table || new QueryTable(node.id, target) |
||||
|
|
||||
|
node.inflightQueries++ |
||||
|
|
||||
|
this._status = opts.table ? MOVING_CLOSER : BOOTSTRAPPING |
||||
|
this._node = node |
||||
|
this._concurrency = opts.concurrency |
||||
|
this._callback = this._onresponse.bind(this) |
||||
|
this._map = identity |
||||
|
this._outputEncoding = cmd ? cmd.outputEncoding : null |
||||
|
} |
||||
|
|
||||
|
map (fn) { |
||||
|
this._map = fn |
||||
|
return this |
||||
|
} |
||||
|
|
||||
|
_onresponse (err, message, peer, request, to, type) { |
||||
|
this.inflight-- |
||||
|
|
||||
|
if (err && to.id) { |
||||
|
// Request, including retries, failed completely
|
||||
|
// Remove the "to" node.
|
||||
|
const node = this._node.bucket.get(to.id) |
||||
|
if (node) this._node._removeNode(node) |
||||
|
} |
||||
|
|
||||
|
if (this._status === FINALIZED) { |
||||
|
if (!this.inflight) { |
||||
|
if (this.destroyed) this.emit('close') |
||||
|
else this.destroy() |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
if (err) { |
||||
|
this.emit('warning', err) |
||||
|
this._readMaybe() |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
this.responses++ |
||||
|
this.table.addVerified(message, peer) |
||||
|
|
||||
|
if (this._status === MOVING_CLOSER) { |
||||
|
const candidates = decodeNodes(message.closerNodes) |
||||
|
for (var i = 0; i < candidates.length; i++) { |
||||
|
this.table.addUnverified(candidates[i], peer) |
||||
|
} |
||||
|
} else if (this._status === UPDATING) { |
||||
|
this.updates++ |
||||
|
} |
||||
|
|
||||
|
if (message.error) { |
||||
|
this.emit('warning', new Error(message.error)) |
||||
|
this._readMaybe() |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
if (!this.query && this._status === MOVING_CLOSER) { |
||||
|
this._readMaybe() |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
const value = this._outputEncoding |
||||
|
? this._decodeOutput(message.value) |
||||
|
: message.value |
||||
|
|
||||
|
const data = this._map({ |
||||
|
type, |
||||
|
node: { |
||||
|
id: message.id, |
||||
|
port: peer.port, |
||||
|
host: peer.host |
||||
|
}, |
||||
|
value |
||||
|
}) |
||||
|
|
||||
|
if (!data) { |
||||
|
this._readMaybe() |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
this.push(data) |
||||
|
} |
||||
|
|
||||
|
_decodeOutput (val) { |
||||
|
try { |
||||
|
return val && this._outputEncoding.decode(val) |
||||
|
} catch (err) { |
||||
|
return null |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
_bootstrap () { |
||||
|
const table = this.table |
||||
|
const bootstrap = this._node.bucket.closest(table.target, table.k) |
||||
|
var i = 0 |
||||
|
|
||||
|
for (; i < bootstrap.length; i++) { |
||||
|
const b = bootstrap[i] |
||||
|
const node = { id: b.id, port: b.port, host: b.host } |
||||
|
table.addUnverified(node, null) |
||||
|
} |
||||
|
|
||||
|
const bootstrapNodes = this._node.bootstrapNodes |
||||
|
if (bootstrap.length < bootstrapNodes.length) { |
||||
|
for (i = 0; i < bootstrapNodes.length; i++) { |
||||
|
this._send(bootstrapNodes[i], true, false) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
this._status = MOVING_CLOSER |
||||
|
this._moveCloser() |
||||
|
} |
||||
|
|
||||
|
_sendAll (nodes, force, sendToken) { |
||||
|
var free = Math.max(0, this._concurrency - this._node._io.inflight.length) |
||||
|
var sent = 0 |
||||
|
|
||||
|
if (!free && !this.inflight) free = 1 |
||||
|
if (!free) return 0 |
||||
|
|
||||
|
for (var i = 0; i < nodes.length; i++) { |
||||
|
if (this._send(nodes[i], force, sendToken)) { |
||||
|
if (++sent === free) break |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return sent |
||||
|
} |
||||
|
|
||||
|
_send (node, force, isUpdate) { |
||||
|
if (!force) { |
||||
|
if (node.queried) return false |
||||
|
node.queried = true |
||||
|
} |
||||
|
|
||||
|
this.inflight++ |
||||
|
const io = this._node._io |
||||
|
|
||||
|
if (isUpdate) { |
||||
|
if (!node.roundtripToken) return this._callback(new Error('Roundtrip token is required')) |
||||
|
io.update(this.command, this.target, this.value, node, this._callback) |
||||
|
} else if (this.query) { |
||||
|
io.query(this.command, this.target, this.value, node, this._callback) |
||||
|
} else { |
||||
|
io.query('_find_node', this.target, null, node, this._callback) |
||||
|
} |
||||
|
|
||||
|
return true |
||||
|
} |
||||
|
|
||||
|
_sendUpdate () { |
||||
|
const sent = this._sendAll(this.table.closest, false, true) |
||||
|
if (sent || this.inflight) return |
||||
|
|
||||
|
this._finalize() |
||||
|
} |
||||
|
|
||||
|
_moveCloser () { |
||||
|
const table = this.table |
||||
|
const sent = this._sendAll(table.unverified, false, false) |
||||
|
if (sent || this.inflight) return |
||||
|
|
||||
|
if (this.update) { |
||||
|
for (var i = 0; i < table.closest.length; i++) { |
||||
|
table.closest[i].queried = false |
||||
|
} |
||||
|
this._status = UPDATING |
||||
|
this._sendUpdate() |
||||
|
} else { |
||||
|
this._finalize() |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
_finalize () { |
||||
|
const status = this._status |
||||
|
if (status === FINALIZED) return |
||||
|
|
||||
|
this._status = FINALIZED |
||||
|
this._node.inflightQueries-- |
||||
|
|
||||
|
if (!this.responses && !this.destroyed) { |
||||
|
this.destroy(new Error('No nodes responded')) |
||||
|
} |
||||
|
if (status === UPDATING && !this.updates && !this.destroyed) { |
||||
|
this.destroy(new Error('No close nodes responded')) |
||||
|
} |
||||
|
|
||||
|
this.push(null) |
||||
|
} |
||||
|
|
||||
|
_readMaybe () { |
||||
|
if (!this.inflight || this._readableState.flowing === true) this._read() |
||||
|
} |
||||
|
|
||||
|
_read () { |
||||
|
if (this._node.destroyed) return |
||||
|
|
||||
|
switch (this._status) { |
||||
|
case BOOTSTRAPPING: return this._bootstrap() |
||||
|
case MOVING_CLOSER: return this._moveCloser() |
||||
|
case UPDATING: return this._sendUpdate() |
||||
|
case FINALIZED: return |
||||
|
} |
||||
|
|
||||
|
throw new Error('Unknown status: ' + this._status) |
||||
|
} |
||||
|
|
||||
|
destroy (err) { |
||||
|
if (this.destroyed) return |
||||
|
this.destroyed = true |
||||
|
|
||||
|
if (err) this.emit('error', err) |
||||
|
this._finalize() |
||||
|
if (!this.inflight) this.emit('close') |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
module.exports = QueryStream |
||||
|
|
||||
|
function decodeNodes (buf) { |
||||
|
if (!buf) return [] |
||||
|
try { |
||||
|
return nodes.decode(buf) |
||||
|
} catch (err) { |
||||
|
return [] |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function identity (a) { |
||||
|
return a |
||||
|
} |
||||
@ -0,0 +1,69 @@ |
|||||
|
const xor = require('xor-distance') |
||||
|
|
||||
|
class QueryTable { |
||||
|
constructor (id, target) { |
||||
|
this.k = 20 |
||||
|
this.id = id |
||||
|
this.target = target |
||||
|
this.closest = [] |
||||
|
this.unverified = [] |
||||
|
} |
||||
|
|
||||
|
addUnverified (node, referrer) { |
||||
|
if (node.id.equals(this.id)) return |
||||
|
|
||||
|
node.distance = xor(this.target, node.id) |
||||
|
node.referrer = referrer |
||||
|
|
||||
|
insertSorted(node, this.k, this.unverified) |
||||
|
} |
||||
|
|
||||
|
addVerified (message, peer) { |
||||
|
if (!message.id || !message.roundtripToken || message.id.equals(this.id)) { |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
var prev = getNode(message.id, this.unverified) |
||||
|
|
||||
|
if (!prev) { |
||||
|
prev = { |
||||
|
id: message.id, |
||||
|
host: peer.host, |
||||
|
port: peer.port, |
||||
|
distance: xor(message.id, this.target) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
prev.roundtripToken = message.roundtripToken |
||||
|
insertSorted(prev, this.k, this.closest) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
module.exports = QueryTable |
||||
|
|
||||
|
function getNode (id, list) { |
||||
|
// find id in the list.
|
||||
|
// technically this would be faster with binary search (against distance)
|
||||
|
// but this list is always small, so meh
|
||||
|
|
||||
|
for (var i = 0; i < list.length; i++) { |
||||
|
if (list[i].id.equals(id)) return list[i] |
||||
|
} |
||||
|
|
||||
|
return null |
||||
|
} |
||||
|
|
||||
|
function insertSorted (node, max, list) { |
||||
|
if (list.length === max && !xor.lt(node.distance, list[max - 1].distance)) return |
||||
|
if (getNode(node.id, list)) return |
||||
|
|
||||
|
if (list.length < max) list.push(node) |
||||
|
else list[max - 1] = node |
||||
|
|
||||
|
var pos = list.length - 1 |
||||
|
while (pos && xor.gt(list[pos - 1].distance, node.distance)) { |
||||
|
list[pos] = list[pos - 1] |
||||
|
list[pos - 1] = node |
||||
|
pos-- |
||||
|
} |
||||
|
} |
||||
@ -1,38 +1,35 @@ |
|||||
{ |
{ |
||||
"name": "dht-rpc", |
"name": "dht-rpc", |
||||
"version": "3.0.1", |
"version": "3.0.1", |
||||
"description": "Make RPC calls over a Kademlia based DHT.", |
"description": "Make RPC calls over a Kademlia based DHT", |
||||
"main": "index.js", |
"main": "index.js", |
||||
"dependencies": { |
"scripts": { |
||||
"duplexify": "^3.5.0", |
"test": "standard && tape test.js", |
||||
"inherits": "^2.0.3", |
"protobuf": "protocol-buffers schema.proto -o lib/messages.js" |
||||
"ipv4-peers": "^1.1.1", |
|
||||
"k-bucket": "^5.0.0", |
|
||||
"protocol-buffers-encodings": "^1.1.0", |
|
||||
"readable-stream": "^2.1.5", |
|
||||
"sodium-universal": "^2.0.0", |
|
||||
"stream-collector": "^1.0.1", |
|
||||
"time-ordered-set": "^1.0.1", |
|
||||
"udp-request": "^1.3.0", |
|
||||
"xor-distance": "^1.0.0" |
|
||||
}, |
|
||||
"devDependencies": { |
|
||||
"protocol-buffers": "^3.2.1", |
|
||||
"standard": "^11.0.1", |
|
||||
"tape": "^4.9.0" |
|
||||
}, |
}, |
||||
"repository": { |
"repository": { |
||||
"type": "git", |
"type": "git", |
||||
"url": "https://github.com/mafintosh/dht-rpc.git" |
"url": "git+https://github.com/mafintosh/dht-rpc.git" |
||||
}, |
|
||||
"scripts": { |
|
||||
"test": "standard && tape test.js", |
|
||||
"protobuf": "protocol-buffers schema.proto -o messages.js" |
|
||||
}, |
}, |
||||
"author": "Mathias Buus (@mafintosh)", |
"author": "Mathias Buus (@mafintosh)", |
||||
"license": "MIT", |
"license": "MIT", |
||||
"bugs": { |
"bugs": { |
||||
"url": "https://github.com/mafintosh/dht-rpc/issues" |
"url": "https://github.com/mafintosh/dht-rpc/issues" |
||||
}, |
}, |
||||
"homepage": "https://github.com/mafintosh/dht-rpc" |
"homepage": "https://github.com/mafintosh/dht-rpc#readme", |
||||
|
"devDependencies": { |
||||
|
"protocol-buffers": "^4.1.0", |
||||
|
"standard": "^12.0.1", |
||||
|
"tape": "^4.9.1" |
||||
|
}, |
||||
|
"dependencies": { |
||||
|
"codecs": "^1.2.1", |
||||
|
"ipv4-peers": "^1.1.1", |
||||
|
"k-bucket": "^5.0.0", |
||||
|
"protocol-buffers-encodings": "^1.1.0", |
||||
|
"sodium-universal": "^2.0.0", |
||||
|
"stream-collector": "^1.0.1", |
||||
|
"time-ordered-set": "^1.0.1", |
||||
|
"xor-distance": "^1.0.0" |
||||
|
} |
||||
} |
} |
||||
|
|||||
@ -1,304 +0,0 @@ |
|||||
var stream = require('readable-stream') |
|
||||
var inherits = require('inherits') |
|
||||
var nodes = require('ipv4-peers').idLength(32) |
|
||||
var xor = require('xor-distance') |
|
||||
|
|
||||
var BLANK = Buffer.from([ |
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 |
|
||||
]) |
|
||||
|
|
||||
module.exports = QueryStream |
|
||||
|
|
||||
function QueryStream (dht, query, opts) { |
|
||||
if (!(this instanceof QueryStream)) return new QueryStream(dht, query, opts) |
|
||||
if (!opts) opts = {} |
|
||||
if (!opts.concurrency) opts.concurrency = opts.highWaterMark || dht.concurrency |
|
||||
if (!query.target) throw new Error('query.target is required') |
|
||||
|
|
||||
stream.Readable.call(this, {objectMode: true, highWaterMark: opts.concurrency}) |
|
||||
|
|
||||
var self = this |
|
||||
var nodes = opts.node || opts.nodes |
|
||||
|
|
||||
this.query = query |
|
||||
this.query.id = dht._queryId |
|
||||
this.target = query.target |
|
||||
this.token = !!opts.token |
|
||||
this.holepunching = opts.holepunching !== false |
|
||||
this.commits = 0 |
|
||||
this.responses = 0 |
|
||||
this.errors = 0 |
|
||||
this.destroyed = false |
|
||||
this.verbose = !!opts.verbose |
|
||||
|
|
||||
dht.inflightQueries++ |
|
||||
|
|
||||
this._dht = dht |
|
||||
this._committing = false |
|
||||
this._finalized = false |
|
||||
this._closest = opts.closest || [] |
|
||||
this._concurrency = opts.concurrency |
|
||||
this._updating = false |
|
||||
this._pending = nodes ? [].concat(nodes).map(copyNode) : [] |
|
||||
this._k = nodes ? Infinity : opts.k || 20 |
|
||||
this._inflight = 0 |
|
||||
this._moveCloser = !nodes |
|
||||
this._map = opts.map || echo |
|
||||
this._bootstrapped = !this._moveCloser |
|
||||
this._onresponse = onresponse |
|
||||
this._onresponseholepunch = onresponseholepunch |
|
||||
|
|
||||
function onresponseholepunch (err, res, peer, query) { |
|
||||
if (!err || !peer || !peer.referrer) self._callback(err, res, peer) |
|
||||
else self._holepunch(peer, query) |
|
||||
} |
|
||||
|
|
||||
function onresponse (err, res, peer) { |
|
||||
self._callback(err, res, peer) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
inherits(QueryStream, stream.Readable) |
|
||||
|
|
||||
QueryStream.prototype.destroy = function (err) { |
|
||||
if (this.destroyed) return |
|
||||
this.destroyed = true |
|
||||
this._finalize() |
|
||||
if (err) this.emit('error', err) |
|
||||
this.emit('close') |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._finalize = function () { |
|
||||
if (this._finalized) return |
|
||||
this._finalized = true |
|
||||
this._dht.inflightQueries-- |
|
||||
if (!this.responses && !this.destroyed) this.destroy(new Error('No nodes responded')) |
|
||||
if (!this.commits && this._committing && !this.destroyed) this.destroy(new Error('No close nodes responded')) |
|
||||
this.push(null) |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._bootstrap = function () { |
|
||||
this._bootstrapped = true |
|
||||
|
|
||||
var bootstrap = this._dht.bucket.closest(this.target, this._k) |
|
||||
var i = 0 |
|
||||
|
|
||||
for (i = 0; i < bootstrap.length; i++) { |
|
||||
var b = bootstrap[i] |
|
||||
this._addPending({id: b.id, port: b.port, host: b.host}, null) |
|
||||
} |
|
||||
|
|
||||
if (bootstrap.length < this._dht._bootstrap.length) { |
|
||||
for (i = 0; i < this._dht._bootstrap.length; i++) { |
|
||||
this._send(this._dht._bootstrap[i], true, false) |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._readMaybe = function () { |
|
||||
if (this._readableState.flowing === true) this._read() |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._sendTokens = function () { |
|
||||
if (this.destroyed) return |
|
||||
|
|
||||
var sent = this._sendAll(this._closest, false, true) |
|
||||
if (sent || this._inflight) return |
|
||||
|
|
||||
this._finalize() |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._sendPending = function () { |
|
||||
if (this.destroyed) return |
|
||||
if (!this._bootstrapped) this._bootstrap() |
|
||||
|
|
||||
var sent = this._sendAll(this._pending, false, false) |
|
||||
if (sent || this._inflight) return |
|
||||
|
|
||||
if (this.token) { |
|
||||
for (var i = 0; i < this._closest.length; i++) this._closest[i].queried = false |
|
||||
this._committing = true |
|
||||
this._sendTokens() |
|
||||
} else { |
|
||||
this._finalize() |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._read = function () { |
|
||||
if (this._committing) this._sendTokens() |
|
||||
else this._sendPending() |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._holepunch = function (peer, query) { |
|
||||
var self = this |
|
||||
|
|
||||
this._dht._holepunch(peer, peer.referrer, function (err) { |
|
||||
if (err) return self._callback(err, null, peer) |
|
||||
self._dht._request(query, peer, false, self._onresponse) |
|
||||
}) |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._callback = function (err, res, peer) { |
|
||||
this._inflight-- |
|
||||
if (this.destroyed) return |
|
||||
|
|
||||
if (err) { |
|
||||
if (res && res.id) { |
|
||||
var node = this._dht.bucket.get(res.id) |
|
||||
if (node) this._dht._removeNode(node) |
|
||||
} |
|
||||
this.errors++ |
|
||||
this.emit('warning', err) |
|
||||
this._readMaybe() |
|
||||
return |
|
||||
} |
|
||||
|
|
||||
this.responses++ |
|
||||
if (this._committing) this.commits++ |
|
||||
this._addClosest(res, peer) |
|
||||
|
|
||||
if (this._moveCloser) { |
|
||||
var candidates = decodeNodes(res.nodes) |
|
||||
for (var i = 0; i < candidates.length; i++) this._addPending(candidates[i], peer) |
|
||||
} |
|
||||
|
|
||||
if (!validateId(res.id) || (this.token && !this.verbose && !this._committing)) { |
|
||||
this._readMaybe() |
|
||||
return |
|
||||
} |
|
||||
|
|
||||
var data = this._map({ |
|
||||
node: { |
|
||||
id: res.id, |
|
||||
port: peer.port, |
|
||||
host: peer.host |
|
||||
}, |
|
||||
value: res.value |
|
||||
}) |
|
||||
|
|
||||
if (!data) { |
|
||||
this._readMaybe() |
|
||||
return |
|
||||
} |
|
||||
|
|
||||
this.push(data) |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._sendAll = function (nodes, force, useToken) { |
|
||||
var sent = 0 |
|
||||
var free = Math.max(0, this._concurrency - this._dht.socket.inflight) |
|
||||
|
|
||||
if (!free && !this._inflight) free = 1 |
|
||||
if (!free) return 0 |
|
||||
|
|
||||
for (var i = 0; i < nodes.length; i++) { |
|
||||
if (this._send(nodes[i], force, useToken)) { |
|
||||
if (++sent === free) break |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
return sent |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._send = function (node, force, useToken) { |
|
||||
if (!force) { |
|
||||
if (node.queried) return false |
|
||||
node.queried = true |
|
||||
} |
|
||||
|
|
||||
this._inflight++ |
|
||||
|
|
||||
var query = this.query |
|
||||
|
|
||||
if (useToken && node.roundtripToken) { |
|
||||
query = { |
|
||||
command: this.query.command, |
|
||||
id: this.query.id, |
|
||||
target: this.query.target, |
|
||||
value: this.query.value, |
|
||||
roundtripToken: node.roundtripToken |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
this._dht._request(query, node, false, this.holepunching ? this._onresponseholepunch : this._onresponse) |
|
||||
return true |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._addPending = function (node, ref) { |
|
||||
if (node.id.equals(this._dht.id)) return |
|
||||
node.distance = xor(this.target, node.id) |
|
||||
node.referrer = ref |
|
||||
insertSorted(node, this._k, this._pending) |
|
||||
} |
|
||||
|
|
||||
QueryStream.prototype._addClosest = function (res, peer) { |
|
||||
if (!res.id || !res.roundtripToken || res.id.equals(this._dht.id)) return |
|
||||
|
|
||||
var prev = getNode(res.id, this._pending) |
|
||||
|
|
||||
if (!prev) { |
|
||||
prev = { |
|
||||
id: res.id, |
|
||||
port: peer.port, |
|
||||
host: peer.host, |
|
||||
distance: xor(res.id, this.target) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
prev.roundtripToken = res.roundtripToken |
|
||||
insertSorted(prev, this._k, this._closest) |
|
||||
} |
|
||||
|
|
||||
function decodeNodes (buf) { |
|
||||
if (!buf) return [] |
|
||||
try { |
|
||||
return nodes.decode(buf) |
|
||||
} catch (err) { |
|
||||
return [] |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
function getNode (id, list) { |
|
||||
// find id in the list.
|
|
||||
// technically this would be faster with binary search (against distance)
|
|
||||
// but this list is always small, so meh
|
|
||||
for (var i = 0; i < list.length; i++) { |
|
||||
if (list[i].id.equals(id)) return list[i] |
|
||||
} |
|
||||
|
|
||||
return null |
|
||||
} |
|
||||
|
|
||||
function validateId (id) { |
|
||||
return id && id.length === 32 |
|
||||
} |
|
||||
|
|
||||
function insertSorted (node, max, list) { |
|
||||
if (list.length === max && !xor.lt(node.distance, list[max - 1].distance)) return |
|
||||
if (getNode(node.id, list)) return |
|
||||
|
|
||||
if (list.length < max) list.push(node) |
|
||||
else list[max - 1] = node |
|
||||
|
|
||||
var pos = list.length - 1 |
|
||||
while (pos && xor.gt(list[pos - 1].distance, node.distance)) { |
|
||||
list[pos] = list[pos - 1] |
|
||||
list[pos - 1] = node |
|
||||
pos-- |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
function copyNode (node) { |
|
||||
return { |
|
||||
id: node.id || BLANK, |
|
||||
port: node.port, |
|
||||
host: node.host, |
|
||||
roundtripToken: node.roundtripToken, |
|
||||
referrer: node.referrer || node.referer |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
function echo (a) { |
|
||||
return a |
|
||||
} |
|
||||
@ -1,16 +1,27 @@ |
|||||
message Request { |
message Holepunch { |
||||
required string command = 1; |
optional bytes from = 2; |
||||
optional bytes id = 2; |
optional bytes to = 3; |
||||
optional bytes target = 3; |
} |
||||
optional bytes forwardRequest = 4; |
|
||||
optional bytes forwardResponse = 5; |
enum TYPE { |
||||
optional bytes roundtripToken = 6; |
QUERY = 1; |
||||
optional bytes value = 7; |
UPDATE = 2; |
||||
|
RESPONSE = 3; |
||||
} |
} |
||||
|
|
||||
message Response { |
message Message { |
||||
optional bytes id = 1; |
// request/response type + id |
||||
optional bytes nodes = 2; |
required TYPE type = 1; |
||||
optional bytes value = 3; |
required uint64 rid = 2; |
||||
optional bytes roundtripToken = 4; |
|
||||
|
// kademlia stuff |
||||
|
optional bytes id = 3; |
||||
|
optional bytes target = 4; |
||||
|
optional bytes closerNodes = 5; |
||||
|
optional bytes roundtripToken = 6; |
||||
|
|
||||
|
// rpc stuff |
||||
|
optional string command = 7; |
||||
|
optional string error = 8; |
||||
|
optional bytes value = 9; |
||||
} |
} |
||||
|
|||||
Loading…
Reference in new issue