dht-rpc/index.js

const dns = require('dns')
const RPC = require('./lib/rpc')
const Query = require('./lib/query')
const Table = require('kademlia-routing-table')
const TOS = require('time-ordered-set')
const FIFO = require('fast-fifo/fixed-size')
const sodium = require('sodium-universal')
const { EventEmitter } = require('events')

const TICK_INTERVAL = 5000
const SLEEPING_INTERVAL = 3 * TICK_INTERVAL
const STABLE_TICKS = 240 // if nothing major bad happens in ~20mins we can consider this node stable (if nat is friendly)
const REFRESH_TICKS = 60 // refresh every ~5min when idle
const RECENT_NODE = 20 // we've heard from a node less than 1min ago
const OLD_NODE = 360 // if an node has been around more than 30 min we consider it old...

class Request {
  constructor (dht, m) {
    this.rpc = dht.rpc
    this.dht = dht
    this.tid = m.tid
    this.from = m.from
    this.to = m.to
    this.nodeId = m.nodeId
    this.target = m.target
    this.closerNodes = m.closerNodes
    this.status = m.status
    this.token = m.token
    this.command = m.command
    this.value = m.value
  }

  get update () {
    return this.token !== null
  }

  error (code) {
    this.dht._reply(this.rpc, this.tid, this.target, code, null, false, this.from)
  }

  reply (value, token = false) {
    this.dht._reply(this.rpc, this.tid, this.target, 0, value, token, this.from)
  }
}

class DHT extends EventEmitter {
  constructor (opts = {}) {
    super()

    const id = opts.id || randomBytes(32)

    this.bootstrapNodes = opts.bootstrap === false ? [] : (opts.bootstrap || []).map(parseNode)
    this.nodes = new TOS()
    this.table = new Table(id)
    this.rpc = new RPC({
      socket: opts.socket,
      onwarning: opts.onwarning,
      onrequest: this._onrequest.bind(this),
      onresponse: this._onresponse.bind(this)
    })

    this.bootstrapped = false
    this.concurrency = opts.concurrency || 16
    this.ephemeral = !!opts.ephemeral

    this._repinging = 0
    this._reping = new FIFO(128)
    this._bootstrapping = this.bootstrap()
    this._secrets = [randomBytes(32), randomBytes(32)]
    this._tick = (Math.random() * 1024) | 0 // random offset it
    this._rotateSecrets = false
    this._lastTick = Date.now()
    this._refreshTick = this._tick + REFRESH_TICKS
    this._stableTick = this._tick + STABLE_TICKS
    this._tickInterval = setInterval(this._ontick.bind(this), TICK_INTERVAL)

    this.table.on('row', (row) => row.on('full', (node) => this._onfullrow(node, row)))
  }

  get id () {
    return this.table.id
  }

  static createRPCSocket (opts) {
    return new RPC(opts)
  }

  ready () {
    return this._bootstrapping
  }

  query (target, command, value, opts) {
    this._refreshTick = this._tick + REFRESH_TICKS
    return new Query(this, target, command, value, opts)
  }

  ping (node) {
    return this.request(null, 'ping', null, node)
  }

  request (target, command, value, to) {
    return this.rpc.request({
      version: 1,
      tid: 0,
      from: null,
      to,
      token: to.token || null,
      nodeId: this.ephemeral ? null : this.table.id,
      target,
      closerNodes: null,
      command,
      status: 0,
      value
    })
  }

  requestAll (target, command, value, nodes, opts = {}) {
    if (nodes instanceof Table) nodes = nodes.closest(nodes.id)
    if (nodes instanceof Query) nodes = nodes.table.closest(nodes.table.id)
    if (nodes.length === 0) return Promise.resolve([])

    const p = []
    for (const node of nodes) p.push(this.request(target, command, value, node))

    let errors = 0
    const results = []
    const min = typeof opts.min === 'number' ? opts.min : 1
    const max = typeof opts.max === 'number' ? opts.max : p.length

    return new Promise((resolve, reject) => {
      for (let i = 0; i < p.length; i++) p[i].then(ondone, onerror)

      function ondone (res) {
        if (results.length < max) results.push(res)
        if (results.length >= max) return resolve(results)
        if (results.length + errors === p.length) return resolve(results)
      }

      function onerror () {
        if ((p.length - ++errors) < min) reject(new Error('Too many requests failed'))
      }
    })
  }

  destroy () {
    this.rpc.destroy()
    clearInterval(this._tickInterval)
  }

  async bootstrap () {
    return new Promise((resolve) => {
      this._backgroundQuery(this.table.id, 'find_node', null)
        .on('close', () => {
          if (!this.bootstrapped) {
            this.bootstrapped = true
            this.emit('ready')
          }
          resolve()
        })
    })
  }

  _backgroundQuery (target, command, value) {
    const backgroundCon = Math.min(this.concurrency, Math.max(2, (this.concurrency / 8) | 0))
    const q = this.query(target, command, value, {
      concurrency: backgroundCon
    })

    q.on('data', () => {
      // yield to other traffic
      q.concurrency = this.rpc.inflightRequests < 3
        ? this.concurrency
        : backgroundCon
    })

    return q
  }

  refresh () {
    const node = this.table.random()
    this._backgroundQuery(node ? node.id : this.table.id, 'find_node', null)
  }

  _pingSome () {
    let cnt = this.rpc.inflightRequests > 2 ? 3 : 5
    let oldest = this.nodes.oldest

    // tiny dht, ping the bootstrap again
    if (!oldest) {
      this.refresh()
      return
    }

    // we've recently pinged the oldest one, so only trigger a couple of repings
    if ((this._tick - oldest.seen) < RECENT_NODE) {
      cnt = 2
    }

    while (cnt--) {
      if (!oldest || this._tick === oldest.seen) continue
      this._check(oldest)
      oldest = oldest.next
    }
  }

  _check (node) {
    this.ping(node)
      .then(
        m => this._maybeRemoveNode(node, m.nodeId),
        () => this._removeNode(node)
      )
  }

  _token (peer, i) {
    this._rotateSecrets = true
    const out = Buffer.allocUnsafe(32)
    sodium.crypto_generichash(out, Buffer.from(peer.host), this._secrets[i])
    return out
  }

  _ontick () {
    if (this._rotateSecrets) {
      const tmp = this._secrets[0]
      this._secrets[0] = this._secrets[1]
      this._secrets[1] = tmp
      sodium.randombytes_buf(tmp)
    }

    const time = Date.now()

    if (time - this._lastTick > SLEEPING_INTERVAL) {
      this._stableTick = 0 // never stable
      this._tick += 2 * OLD_NODE // bump the tick enough that everything appears old.
      this._tick += 8 - (this._tick & 7) - 2 // triggers a series of pings in two ticks
      this._refreshTick = this._tick + 1 // triggers a refresh next tick (allow network time to wake up also)
      this.emit('wakeup')
    } else {
      this._tick++
    }

    this._lastTick = time

    if (!this.bootstrapped) return

    if (this._tick === this._stableTick) {
      this.emit('stable')
    }

    if ((this._tick & 7) === 0) {
      this._pingSome()
    }

    if (((this._tick & 63) === 0 && this.nodes.length < this.table.k) || this._tick >= this._refreshTick) {
      this.refresh()
    }
  }

  _onfullrow (newNode, row) {
    if (this.bootstrapped && this._reping.push({ newNode, row })) this._repingMaybe()
  }

  _repingMaybe () {
    while (this._repinging < 3 && this._reping.isEmpty() === false) {
      const { newNode, row } = this._reping.shift()
      if (this.table.get(newNode.id)) continue

      let oldest = null
      for (const node of row.nodes) {
        if (node.seen === this._tick) continue
        if (oldest === null || oldest.seen > node.seen || (oldest.seen === node.seen && oldest.added > node.added)) oldest = node
      }

      if (oldest === null) continue
      if ((this._tick - oldest.seen) < RECENT_NODE && (this._tick - oldest.added) > OLD_NODE) continue

      this._repingAndSwap(newNode, oldest)
    }
  }

  _repingAndSwap (newNode, oldNode) {
    const self = this

    this._repinging++
    this.ping(oldNode).then(onsuccess, onswap)

    function onsuccess (m) {
      if (m.nodeId === null || !m.nodeId.equals(oldNode.id)) return onswap()
      self._repinging--
      self._repingMaybe()
    }

    function onswap () {
      self._repinging--
      self._repingMaybe()
      self._removeNode(oldNode)
      self._addNode(newNode)
    }
  }

  _resolveBootstrapNodes (done) {
    if (!this.bootstrapNodes.length) return done([])

    let missing = this.bootstrapNodes.length
    const nodes = []

    for (const node of this.bootstrapNodes) {
      dns.lookup(node.host, (_, host) => {
        if (host) nodes.push({ id: node.id || null, host, port: node.port })
        if (--missing === 0) done(nodes)
      })
    }
  }

  _addNode (node) {
    if (this.nodes.has(node) || node.id.equals(this.table.id)) return

    node.added = node.seen = this._tick

    if (this.table.add(node)) this.nodes.add(node)

    this.emit('add-node', node)
  }

  _maybeRemoveNode (node, expectedId) {
    if (expectedId !== null && expectedId.equals(node.id)) return
    this._removeNode(node)
  }

  _removeNode (node) {
    if (!this.nodes.has(node)) return

    this.table.remove(node.id)
    this.nodes.remove(node)

    this.emit('remove-node', node)
  }

  _addNodeFromMessage (m) {
    const oldNode = this.table.get(m.nodeId)

    if (oldNode) {
      if (oldNode.port === m.from.port && oldNode.host === m.from.host) {
        // refresh it
        oldNode.to = m.to
        oldNode.seen = this._tick
        this.nodes.add(oldNode)
      }
      return
    }

    this._addNode({
      id: m.nodeId,
      token: null,
      port: m.from.port,
      host: m.from.host,
      to: m.to,
      added: this._tick,
      seen: this._tick,
      prev: null,
      next: null
    })
  }

  _onrequest (req) {
    if (req.nodeId !== null) this._addNodeFromMessage(req)

    if (req.token !== null) {
      if (!req.token.equals(this._token(req.from, 1)) && !req.token.equals(this._token(req.from, 0))) {
        req.token = null
      }
    }

    // empty reply back
    if (req.command === 'ping') {
      this._reply(this.rpc, req.tid, null, 0, null, false, req.from)
      return
    }

    // empty dht reply back
    if (req.command === 'find_node') {
      this._reply(this.rpc, req.tid, req.target, 0, null, false, req.from)
      return
    }

    if (this.emit('request', new Request(this, req)) === false) {
      this._reply(this.rpc, req.tid, req.target, 1, null, false, req.from)
    }
  }

  _onresponse (res) {
    if (res.nodeId !== null) this._addNodeFromMessage(res)
  }

  bind (...args) {
    return this.rpc.bind(...args)
  }

  address () {
    return this.rpc.address()
  }

  _reply (rpc, tid, target, status, value, token, to) {
    const closerNodes = target ? this.table.closest(target) : null
    const persistent = !this.ephemeral && rpc === this.rpc

    rpc.send({
      version: 1,
      tid,
      from: null,
      to,
      token: token ? this._token(to, 1) : null,
      nodeId: persistent ? this.table.id : null,
      target: null,
      closerNodes,
      command: null,
      status,
      value
    })
  }
}

DHT.OK = 0
DHT.UNKNOWN_COMMAND = 1
DHT.BAD_TOKEN = 2

module.exports = DHT

function parseNode (s) {
  if (typeof s === 'object') return s
  const [, id, host, port] = s.match(/([a-f0-9]{64}@)?([^:@]+)(:\d+)?$/i)
  if (!port) throw new Error('Node format is id@?host:port')

  return {
    id: id ? Buffer.from(id.slice(0, -1), 'hex') : null,
    host,
    port: Number(port.slice(1))
  }
}

function randomBytes (n) {
  const b = Buffer.alloc(n)
  sodium.randombytes_buf(b)
  return b
}