node/deps/npm/lib/install/deps.js

'use strict'
var assert = require('assert')
var path = require('path')
var url = require('url')
var semver = require('semver')
var asyncMap = require('slide').asyncMap
var chain = require('slide').chain
var union = require('lodash.union')
var iferr = require('iferr')
var npa = require('npm-package-arg')
var validate = require('aproba')
var realizePackageSpecifier = require('realize-package-specifier')
var dezalgo = require('dezalgo')
var fetchPackageMetadata = require('../fetch-package-metadata.js')
var andAddParentToErrors = require('./and-add-parent-to-errors.js')
var addShrinkwrap = require('../fetch-package-metadata.js').addShrinkwrap
var addBundled = require('../fetch-package-metadata.js').addBundled
var readShrinkwrap = require('./read-shrinkwrap.js')
var inflateShrinkwrap = require('./inflate-shrinkwrap.js')
var inflateBundled = require('./inflate-bundled.js')
var andFinishTracker = require('./and-finish-tracker.js')
var npm = require('../npm.js')
var flatName = require('./flatten-tree.js').flatName
var createChild = require('./node.js').create
var resetMetadata = require('./node.js').reset
var andIgnoreErrors = require('./and-ignore-errors.js')
var isInstallable = require('./validate-args.js').isInstallable
var packageId = require('../utils/package-id.js')
var moduleName = require('../utils/module-name.js')

exports.test = {} // used to hold functions for testing by unit tests

// The export functions in this module mutate a dependency tree, adding
// items to them.

function isDep (tree, child) {
  if (child.fromShrinkwrap) return true
  var name = moduleName(child)
  var requested = isProdDep(tree, name)
  var matches
  if (requested) matches = doesChildVersionMatch(child, requested, tree)
  if (matches) return matches
  requested = isDevDep(tree, name)
  if (!requested) return
  return doesChildVersionMatch(child, requested, tree)
}

function isDevDep (tree, name) {
  var devDeps = tree.package.devDependencies || {}
  var reqVer = devDeps[name]
  if (reqVer == null) return
  return npa(name + '@' + reqVer)
}

function isProdDep (tree, name) {
  var deps = tree.package.dependencies || {}
  var reqVer = deps[name]
  if (reqVer == null) return false
  return npa(name + '@' + reqVer)
}

var registryTypes = { range: true, version: true }

function doesChildVersionMatch (child, requested, requestor) {
  // we always consider deps provided by a shrinkwrap as "correct" or else
  // we'll subvert them if they're intentionally "invalid"
  if (child.parent === requestor && child.fromShrinkwrap) return true
  // ranges of * ALWAYS count as a match, because when downloading we allow
  // prereleases to match * if there are ONLY prereleases
  if (requested.spec === '*') return true

  var childReq = child.package._requested
  if (childReq) {
    if (childReq.rawSpec === requested.rawSpec) return true
    if (childReq.type === requested.type && childReq.spec === requested.spec) return true
  }
  if (!registryTypes[requested.type]) return requested.rawSpec === child.package._from
  return semver.satisfies(child.package.version, requested.spec)
}

exports.recalculateMetadata = function (tree, log, next) {
  recalculateMetadata(tree, log, {}, next)
}

function recalculateMetadata (tree, log, seen, next) {
  validate('OOOF', arguments)
  if (seen[tree.path]) return next()
  seen[tree.path] = true
  if (tree.parent == null) resetMetadata(tree)
  function markDeps (spec, done) {
    validate('SF', arguments)
    realizePackageSpecifier(spec, packageRelativePath(tree), function (er, req) {
      if (er || !req.name) return done()
      var child = findRequirement(tree, req.name, req)
      if (child) {
        resolveWithExistingModule(child, tree, log, andIgnoreErrors(done))
      } else if (tree.package.dependencies[req.name] != null) {
        tree.missingDeps[req.name] = req.rawSpec
        done()
      } else if (tree.package.devDependencies[req.name] != null) {
        tree.missingDevDeps[req.name] = req.rawSpec
        done()
      } else {
        done()
      }
    })
  }
  function specs (deps) {
    return Object.keys(deps).map(function (depname) { return depname + '@' + deps[depname] })
  }
  var tomark = specs(tree.package.dependencies)
  if (!tree.parent && (npm.config.get('dev') || !npm.config.get('production'))) {
    tomark = union(tomark, specs(tree.package.devDependencies))
  }
  tree.children = tree.children.filter(function (child) { return !child.failed })
  chain([
    [asyncMap, tomark, markDeps],
    [asyncMap, tree.children, function (child, done) { recalculateMetadata(child, log, seen, done) }]
  ], function () {
    tree.userRequired = tree.package._requiredBy.some(function (req) { req === '#USER' })
    tree.existing = tree.package._requiredBy.some(function (req) { req === '#EXISTING' })
    tree.package._location = flatNameFromTree(tree)
    next(null, tree)
  })
}

function addRequiredDep (tree, child) {
  if (!isDep(tree, child)) return false
  var name = isProdDep(tree, moduleName(child)) ? flatNameFromTree(tree) : '#DEV:' + flatNameFromTree(tree)
  replaceModuleName(child.package, '_requiredBy', name)
  replaceModule(child, 'requiredBy', tree)
  replaceModule(tree, 'requires', child)
  return true
}

function removeObsoleteDep (child) {
  if (child.removed) return
  child.removed = true
  var requires = child.requires || []
  requires.forEach(function (requirement) {
    requirement.requiredBy = requirement.requiredBy.filter(function (reqBy) { reqBy !== child })
    if (requirement.requiredBy.length === 0) removeObsoleteDep(requirement)
  })
}

function matchingDep (tree, name) {
  if (tree.package.dependencies && tree.package.dependencies[name]) return tree.package.dependencies[name]
  if (tree.package.devDependencies && tree.package.devDependencies[name]) return tree.package.devDependencies[name]
  return
}

function packageRelativePath (tree) {
  if (!tree) return ''
  var requested = tree.package._requested || {}
  var isLocal = requested.type === 'directory' || requested.type === 'local'
  return isLocal ? requested.spec : tree.path
}

function getShrinkwrap (tree, name) {
  return tree.package._shrinkwrap && tree.package._shrinkwrap.dependencies && tree.package._shrinkwrap.dependencies[name]
}

exports.getAllMetadata = function (args, tree, next) {
  asyncMap(args, function (spec, done) {
    if (tree && spec.lastIndexOf('@') <= 0) {
      var sw = getShrinkwrap(tree, spec)
      if (sw) {
        // FIXME: This is duplicated in inflate-shrinkwrap and should be factoed
        // into a shared function
        spec = sw.resolved
             ? spec + '@' + sw.resolved
             : (sw.from && url.parse(sw.from).protocol)
             ? spec + '@' + sw.from
             : spec + '@' + sw.version
      } else {
        var version = matchingDep(tree, spec)
        if (version != null) {
          spec += '@' + version
        }
      }
    }
    fetchPackageMetadata(spec, packageRelativePath(tree), done)
  }, next)
}

// Add a list of args to tree's top level dependencies
exports.loadRequestedDeps = function (args, tree, saveToDependencies, log, next) {
  validate('AOOF', [args, tree, log, next])
  asyncMap(args, function (pkg, done) {
    var depLoaded = andAddParentToErrors(tree, done)
    resolveWithNewModule(pkg, tree, log.newGroup('loadRequestedDeps'), iferr(depLoaded, function (child, tracker) {
      validate('OO', arguments)
      if (npm.config.get('global')) {
        child.isGlobal = true
      }
      var childName = moduleName(child)
      if (saveToDependencies) {
        tree.package[saveToDependencies][childName] =
          child.package._requested.rawSpec || child.package._requested.spec
      }
      if (saveToDependencies && saveToDependencies !== 'devDependencies') {
        tree.package.dependencies[childName] =
          child.package._requested.rawSpec || child.package._requested.spec
      }
      child.directlyRequested = true
      child.save = saveToDependencies

      // For things the user asked to install, that aren't a dependency (or
      // won't be when we're done), flag it as "depending" on the user
      // themselves, so we don't remove it as a dep that no longer exists
      if (!addRequiredDep(tree, child)) {
        replaceModuleName(child.package, '_requiredBy', '#USER')
        child.directlyRequested = true
      }
      depLoaded(null, child, tracker)
    }))
  }, andForEachChild(loadDeps, andFinishTracker(log, next)))
}

function moduleNameMatches (name) {
  return function (child) { return moduleName(child) === name }
}

function noModuleNameMatches (name) {
  return function (child) { return moduleName(child) !== name }
}

// while this implementation does not require async calling, doing so
// gives this a consistent interface with loadDeps et al
exports.removeDeps = function (args, tree, saveToDependencies, log, next) {
  validate('AOOF', [args, tree, log, next])
  args.forEach(function (pkg) {
    var pkgName = moduleName(pkg)
    if (saveToDependencies) {
      var toRemove = tree.children.filter(moduleNameMatches(pkgName))
      replaceModule(tree, 'removed', toRemove[0])
      toRemove.forEach(function (parent) {
        parent.save = saveToDependencies
      })
    }
    tree.children = tree.children.filter(noModuleNameMatches(pkgName))
  })
  log.finish()
  next()
}

function andForEachChild (load, next) {
  validate('F', [next])
  next = dezalgo(next)
  return function (er, children, logs) {
    // when children is empty, logs won't be passed in at all (asyncMap is weird)
    // so shortcircuit before arg validation
    if (!er && (!children || children.length === 0)) return next()
    validate('EAA', arguments)
    if (er) return next(er)
    assert(children.length === logs.length)
    var cmds = []
    for (var ii = 0; ii < children.length; ++ii) {
      cmds.push([load, children[ii], logs[ii]])
    }
    var sortedCmds = cmds.sort(function installOrder (aa, bb) {
      return moduleName(aa[1]).localeCompare(moduleName(bb[1]))
    })
    chain(sortedCmds, next)
  }
}

function isDepOptional (tree, name) {
  if (!tree.package.optionalDependencies) return false
  if (tree.package.optionalDependencies[name] != null) return true
  return false
}

var failedDependency = exports.failedDependency = function (tree, name_pkg) {
  var name, pkg
  if (typeof name_pkg === 'string') {
    name = name_pkg
  } else {
    pkg = name_pkg
    name = moduleName(pkg)
  }

  tree.children = tree.children.filter(noModuleNameMatches(name))

  if (isDepOptional(tree, name)) {
    return false
  }

  tree.failed = true

  if (!tree.parent) return true

  for (var ii = 0; ii < tree.requiredBy.length; ++ii) {
    var requireParent = tree.requiredBy[ii]
    if (failedDependency(requireParent, tree.package)) {
      return true
    }
  }
  return false
}

function andHandleOptionalErrors (log, tree, name, done) {
  validate('OOSF', arguments)
  return function (er, child, childLog) {
    if (!er) validate('OO', [child, childLog])
    if (!er) return done(er, child, childLog)
    var isFatal = failedDependency(tree, name)
    if (er && !isFatal) {
      tree.children = tree.children.filter(noModuleNameMatches(name))
      log.warn('install', "Couldn't install optional dependency:", er.message)
      log.verbose('install', er.stack)
      return done()
    } else {
      return done(er, child, childLog)
    }
  }
}

// Load any missing dependencies in the given tree
exports.loadDeps = loadDeps
function loadDeps (tree, log, next) {
  validate('OOF', arguments)
  if (tree.loaded || (tree.parent && tree.parent.failed)) return andFinishTracker.now(log, next)
  if (tree.parent) tree.loaded = true
  if (!tree.package.dependencies) tree.package.dependencies = {}
  asyncMap(Object.keys(tree.package.dependencies), function (dep, done) {
    var version = tree.package.dependencies[dep]
    if (tree.package.optionalDependencies &&
        tree.package.optionalDependencies[dep] &&
        !npm.config.get('optional')) {
      return done()
    }

    addDependency(dep, version, tree, log.newGroup('loadDep:' + dep), andHandleOptionalErrors(log, tree, dep, done))
  }, andForEachChild(loadDeps, andFinishTracker(log, next)))
}

// Load development dependencies into the given tree
exports.loadDevDeps = function (tree, log, next) {
  validate('OOF', arguments)
  if (!tree.package.devDependencies) return andFinishTracker.now(log, next)
  asyncMap(Object.keys(tree.package.devDependencies), function (dep, done) {
    // things defined as both dev dependencies and regular dependencies are treated
    // as the former
    if (tree.package.dependencies[dep]) return done()

    var logGroup = log.newGroup('loadDevDep:' + dep)
    addDependency(dep, tree.package.devDependencies[dep], tree, logGroup, done)
  }, andForEachChild(loadDeps, andFinishTracker(log, next)))
}

exports.loadExtraneous = function loadExtraneous (tree, log, next) {
  var seen = {}
  function loadExtraneous (tree, log, next) {
    validate('OOF', arguments)
    if (seen[tree.path]) return next()
    seen[tree.path] = true
    asyncMap(tree.children.filter(function (child) { return !child.loaded }), function (child, done) {
      resolveWithExistingModule(child, tree, log, done)
    }, andForEachChild(loadExtraneous, andFinishTracker(log, next)))
  }
  loadExtraneous(tree, log, next)
}

exports.loadExtraneous.andResolveDeps = function (tree, log, next) {
  validate('OOF', arguments)
  asyncMap(tree.children.filter(function (child) { return !child.loaded }), function (child, done) {
    resolveWithExistingModule(child, tree, log, done)
  }, andForEachChild(loadDeps, andFinishTracker(log, next)))
}

function addDependency (name, versionSpec, tree, log, done) {
  validate('SSOOF', arguments)
  var next = andAddParentToErrors(tree, done)
  var spec = name + '@' + versionSpec
  realizePackageSpecifier(spec, packageRelativePath(tree), iferr(done, function (req) {
    var child = findRequirement(tree, name, req)
    if (child) {
      resolveWithExistingModule(child, tree, log, iferr(next, function (child, log) {
        if (child.package._shrinkwrap === undefined) {
          readShrinkwrap.andInflate(child, function (er) { next(er, child, log) })
        } else {
          next(null, child, log)
        }
      }))
    } else {
      resolveWithNewModule(req, tree, log, next)
    }
  }))
}

function resolveWithExistingModule (child, tree, log, next) {
  validate('OOOF', arguments)
  addRequiredDep(tree, child)

  if (tree.parent && child.parent !== tree) updatePhantomChildren(tree.parent, child)

  next(null, child, log)
}

var updatePhantomChildren = exports.updatePhantomChildren = function (current, child) {
  validate('OO', arguments)
  while (current && current !== child.parent) {
    // FIXME: phantomChildren doesn't actually belong in the package.json
    if (!current.package._phantomChildren) current.package._phantomChildren = {}
    current.package._phantomChildren[moduleName(child)] = child.package.version
    current = current.parent
  }
}

function flatNameFromTree (tree) {
  validate('O', arguments)
  if (!tree.parent) return '/'
  var path = flatNameFromTree(tree.parent)
  if (path !== '/') path += '/'
  return flatName(path, tree)
}

exports.test.replaceModuleName = replaceModuleName
function replaceModuleName (obj, key, name) {
  validate('OSS', arguments)
  obj[key] = union(obj[key] || [], [name])
}

exports.test.replaceModule = replaceModule
function replaceModule (obj, key, child) {
  validate('OSO', arguments)
  if (!obj[key]) obj[key] = []
  // we replace children with a new array object instead of mutating it
  // because mutating it results in weird failure states.
  // I would very much like to know _why_ this is. =/
  var children = [].concat(obj[key])
  var childName = moduleName(child)
  for (var replaceAt = 0; replaceAt < children.length; ++replaceAt) {
    if (moduleName(children[replaceAt]) === childName) break
  }
  var replacing = children.splice(replaceAt, 1, child)
  obj[key] = children
  return replacing[0]
}

function resolveWithNewModule (pkg, tree, log, next) {
  validate('OOOF', arguments)
  if (pkg.type) {
    return fetchPackageMetadata(pkg, packageRelativePath(tree), log.newItem('fetchMetadata'), iferr(next, function (pkg) {
      resolveWithNewModule(pkg, tree, log, next)
    }))
  }

  if (!pkg._installable) {
    log.silly('resolveWithNewModule', packageId(pkg), 'checking installable status')
    return isInstallable(pkg, iferr(next, function () {
      pkg._installable = true
      resolveWithNewModule(pkg, tree, log, next)
    }))
  }

  if (!pkg._from) {
    pkg._from = pkg._requested.name + '@' + pkg._requested.spec
  }
  addShrinkwrap(pkg, iferr(next, function () {
    addBundled(pkg, iferr(next, function () {
      var parent = earliestInstallable(tree, tree, pkg) || tree
      var child = createChild({
        package: pkg,
        parent: parent,
        path: path.join(parent.path, 'node_modules', pkg.name),
        realpath: path.resolve(parent.realpath, 'node_modules', pkg.name),
        children: pkg._bundled || [],
        isLink: tree.isLink
      })

      var replaced = replaceModule(parent, 'children', child)
      if (replaced) removeObsoleteDep(replaced)
      addRequiredDep(tree, child)
      pkg._location = flatNameFromTree(child)

      if (tree.parent && parent !== tree) updatePhantomChildren(tree.parent, child)

      if (pkg._bundled) {
        inflateBundled(child, child.children)
      }

      if (pkg._shrinkwrap && pkg._shrinkwrap.dependencies) {
        return inflateShrinkwrap(child, pkg._shrinkwrap.dependencies, function (er) {
          next(er, child, log)
        })
      }

      next(null, child, log)
    }))
  }))
}

var validatePeerDeps = exports.validatePeerDeps = function (tree, onInvalid) {
  if (!tree.package.peerDependencies) return
  Object.keys(tree.package.peerDependencies).forEach(function (pkgname) {
    var version = tree.package.peerDependencies[pkgname]
    var match = findRequirement(tree.parent || tree, pkgname, npa(pkgname + '@' + version))
    if (!match) onInvalid(tree, pkgname, version)
  })
}

exports.validateAllPeerDeps = function (tree, onInvalid) {
  validateAllPeerDeps(tree, onInvalid, {})
}

function validateAllPeerDeps (tree, onInvalid, seen) {
  validate('OFO', arguments)
  if (seen[tree.path]) return
  seen[tree.path] = true
  validatePeerDeps(tree, onInvalid)
  tree.children.forEach(function (child) { validateAllPeerDeps(child, onInvalid, seen) })
}

// Determine if a module requirement is already met by the tree at or above
// our current location in the tree.
var findRequirement = exports.findRequirement = function (tree, name, requested, requestor) {
  validate('OSO', [tree, name, requested])
  if (!requestor) requestor = tree
  var nameMatch = function (child) {
    return moduleName(child) === name && child.parent && !child.removed
  }
  var versionMatch = function (child) {
    return doesChildVersionMatch(child, requested, requestor)
  }
  if (nameMatch(tree)) {
    // this *is* the module, but it doesn't match the version, so a
    // new copy will have to be installed
    return versionMatch(tree) ? tree : null
  }

  var matches = tree.children.filter(nameMatch)
  if (matches.length) {
    matches = matches.filter(versionMatch)
    // the module exists as a dependent, but the version doesn't match, so
    // a new copy will have to be installed above here
    if (matches.length) return matches[0]
    return null
  }
  if (!tree.parent) return null
  return findRequirement(tree.parent, name, requested, requestor)
}

// Find the highest level in the tree that we can install this module in.
// If the module isn't installed above us yet, that'd be the very top.
// If it is, then it's the level below where its installed.
var earliestInstallable = exports.earliestInstallable = function (requiredBy, tree, pkg) {
  validate('OOO', arguments)

  function undeletedModuleMatches (child) {
    return !child.removed && moduleName(child) === pkg.name
  }
  if (tree.children.some(undeletedModuleMatches)) return null

  // If any of the children of this tree have conflicting
  // binaries then we need to decline to install this package here.
  var binaryMatches = typeof pkg.bin === 'object' && tree.children.some(function (child) {
    if (child.removed) return false
    if (typeof child.package.bin !== 'object') return false
    return Object.keys(child.package.bin).some(function (bin) {
      return pkg.bin[bin]
    })
  })
  if (binaryMatches) return null

  // if this tree location requested the same module then we KNOW it
  // isn't compatible because if it were findRequirement would have
  // found that version.
  var deps = tree.package.dependencies || {}
  if (!tree.removed && requiredBy !== tree && deps[pkg.name]) {
    return null
  }

  // FIXME: phantomChildren doesn't actually belong in the package.json
  if (tree.package._phantomChildren && tree.package._phantomChildren[pkg.name]) return null

  if (!tree.parent) return tree
  if (tree.isGlobal) return tree

  return (earliestInstallable(requiredBy, tree.parent, pkg) || tree)
}