@ -51,14 +51,16 @@ adding a local tarball:
* / * /
exports = module . exports = cache exports = module . exports = cache
exports . read = read cach e. read = read
exports . clean = clean cach e. clean = clean
exports . unpack = unpack cach e. unpack = unpack
exports . lock = lock cach e. lock = lock
exports . unlock = unlock cach e. unlock = unlock
var mkdir = require ( "mkdirp" ) var mkdir = require ( "mkdirp" )
, exec = require ( "./utils/exec.js" ) , exec = require ( "./utils/exec.js" )
, spawn = require ( "child_process" ) . spawn
, once = require ( "once" )
, fetch = require ( "./utils/fetch.js" ) , fetch = require ( "./utils/fetch.js" )
, npm = require ( "./npm.js" ) , npm = require ( "./npm.js" )
, fs = require ( "graceful-fs" ) , fs = require ( "graceful-fs" )
@ -77,6 +79,7 @@ var mkdir = require("mkdirp")
, lockFile = require ( "lockfile" ) , lockFile = require ( "lockfile" )
, crypto = require ( "crypto" ) , crypto = require ( "crypto" )
, retry = require ( "retry" ) , retry = require ( "retry" )
, zlib = require ( "zlib" )
cache . usage = "npm cache add <tarball file>" cache . usage = "npm cache add <tarball file>"
+ "\nnpm cache add <folder>" + "\nnpm cache add <folder>"
@ -151,7 +154,7 @@ function ls (args, cb) {
if ( 0 === prefix . indexOf ( process . env . HOME ) ) { if ( 0 === prefix . indexOf ( process . env . HOME ) ) {
prefix = "~" + prefix . substr ( process . env . HOME . length ) prefix = "~" + prefix . substr ( process . env . HOME . length )
} }
ls_ ( args , npm . config . get ( "depth" ) , function ( er , files ) { ls_ ( args , npm . config . get ( "depth" ) , function ( er , files ) {
console . log ( files . map ( function ( f ) { console . log ( files . map ( function ( f ) {
return path . join ( prefix , f ) return path . join ( prefix , f )
} ) . join ( "\n" ) . trim ( ) ) } ) . join ( "\n" ) . trim ( ) )
@ -188,7 +191,7 @@ function clean (args, cb) {
// npm cache add <pkg> <ver>
// npm cache add <pkg> <ver>
// npm cache add <tarball>
// npm cache add <tarball>
// npm cache add <folder>
// npm cache add <folder>
exports . add = function ( pkg , ver , scrub , cb ) { cach e. add = function ( pkg , ver , scrub , cb ) {
if ( typeof cb !== "function" ) cb = scrub , scrub = false if ( typeof cb !== "function" ) cb = scrub , scrub = false
if ( typeof cb !== "function" ) cb = ver , ver = null if ( typeof cb !== "function" ) cb = ver , ver = null
if ( scrub ) { if ( scrub ) {
@ -230,7 +233,7 @@ function add (args, cb) {
spec = args [ 0 ] spec = args [ 0 ]
} }
log . silly ( "cache add" , "name=%j spec=%j args=%j" , name , spec , args ) log . verbose ( "cache add" , "name=%j spec=%j args=%j" , name , spec , args )
if ( ! name && ! spec ) return cb ( usage ) if ( ! name && ! spec ) return cb ( usage )
@ -298,6 +301,10 @@ function addRemoteTarball (u, shasum, name, cb_) {
if ( iF . length > 1 ) return if ( iF . length > 1 ) return
function cb ( er , data ) { function cb ( er , data ) {
if ( data ) {
data . _ from = u
data . _ resolved = u
}
unlock ( u , function ( ) { unlock ( u , function ( ) {
var c var c
while ( c = iF . shift ( ) ) c ( er , data ) while ( c = iF . shift ( ) ) c ( er , data )
@ -305,46 +312,55 @@ function addRemoteTarball (u, shasum, name, cb_) {
} ) } )
} }
var tmp = path . join ( npm . tmp , Date . now ( ) + "-" + Math . random ( ) , "tmp.tgz" )
lock ( u , function ( er ) { lock ( u , function ( er ) {
if ( er ) return cb ( er ) if ( er ) return cb ( er )
log . verbose ( "addRemoteTarball" , [ u , shasum ] ) log . verbose ( "addRemoteTarball" , [ u , shasum ] )
var tmp = path . join ( npm . tmp , Date . now ( ) + "-" + Math . random ( ) , "tmp.tgz" )
mkdir ( path . dirname ( tmp ) , function ( er ) { mkdir ( path . dirname ( tmp ) , function ( er ) {
if ( er ) return cb ( er ) if ( er ) return cb ( er )
// Tuned to spread 3 attempts over about a minute.
addRemoteTarball_ ( u , tmp , shasum , done )
// See formula at <https://github.com/tim-kos/node-retry>.
} )
var operation = retry . operation } )
( { retries : npm . config . get ( "fetch-retries" )
, factor : npm . config . get ( "fetch-retry-factor" ) function done ( er ) {
, minTimeout : npm . config . get ( "fetch-retry-mintimeout" ) if ( er ) return cb ( er )
, maxTimeout : npm . config . get ( "fetch-retry-maxtimeout" ) } ) addLocalTarball ( tmp , name , cb )
}
operation . attempt ( function ( currentAttempt ) { }
log . info ( "retry" , "fetch attempt " + currentAttempt
+ " at " + ( new Date ( ) ) . toLocaleTimeString ( ) ) function addRemoteTarball_ ( u , tmp , shasum , cb ) {
fetchAndShaCheck ( u , tmp , shasum , function ( er , response ) { // Tuned to spread 3 attempts over about a minute.
// Only retry on 408, 5xx or no `response`.
// See formula at <https://github.com/tim-kos/node-retry>.
var statusCode = response && response . statusCode var operation = retry . operation
var statusRetry = ! statusCode || ( statusCode === 408 || statusCode >= 500 ) ( { retries : npm . config . get ( "fetch-retries" )
if ( er && statusRetry && operation . retry ( er ) ) { , factor : npm . config . get ( "fetch-retry-factor" )
log . info ( "retry" , "will retry, error on last attempt: " + er ) , minTimeout : npm . config . get ( "fetch-retry-mintimeout" )
return , maxTimeout : npm . config . get ( "fetch-retry-maxtimeout" ) } )
}
done ( er ) operation . attempt ( function ( currentAttempt ) {
} ) log . info ( "retry" , "fetch attempt " + currentAttempt
} ) + " at " + ( new Date ( ) ) . toLocaleTimeString ( ) )
fetchAndShaCheck ( u , tmp , shasum , function ( er , response ) {
// Only retry on 408, 5xx or no `response`.
var sc = response && response . statusCode
var statusRetry = ! sc || ( sc === 408 || sc >= 500 )
if ( er && statusRetry && operation . retry ( er ) ) {
log . info ( "retry" , "will retry, error on last attempt: " + er )
return
}
cb ( er )
} ) } )
function done ( er ) {
if ( er ) return cb ( er )
addLocalTarball ( tmp , name , cb )
}
} ) } )
} }
// For now, this is kind of dumb. Just basically treat git as
// 1. cacheDir = path.join(cache,'_git-remotes',sha1(u))
// yet another "fetch and scrub" kind of thing.
// 2. checkGitDir(cacheDir) ? 4. : 3. (rm cacheDir if necessary)
// Clone to temp folder, then proceed with the addLocal stuff.
// 3. git clone --mirror u cacheDir
// 4. cd cacheDir && git fetch -a origin
// 5. git archive /tmp/random.tgz
// 6. addLocalTarball(/tmp/random.tgz) <gitref> --format=tar --prefix=package/
function addRemoteGit ( u , parsed , name , cb_ ) { function addRemoteGit ( u , parsed , name , cb_ ) {
if ( typeof cb_ !== "function" ) cb_ = name , name = null if ( typeof cb_ !== "function" ) cb_ = name , name = null
@ -361,6 +377,8 @@ function addRemoteGit (u, parsed, name, cb_) {
} ) } )
} }
var p , co // cachePath, git-ref we want to check out
lock ( u , function ( er ) { lock ( u , function ( er ) {
if ( er ) return cb ( er ) if ( er ) return cb ( er )
@ -379,34 +397,125 @@ function addRemoteGit (u, parsed, name, cb_) {
u = u . replace ( /^ssh:\/\// , "" ) u = u . replace ( /^ssh:\/\// , "" )
} }
var v = crypto . createHash ( "sha1" ) . update ( u ) . digest ( "hex" ) . slice ( 0 , 8 )
v = u . replace ( /[^a-zA-Z0-9]+/g , '-' ) + '-' + v
log . verbose ( "addRemoteGit" , [ u , co ] ) log . verbose ( "addRemoteGit" , [ u , co ] )
var tmp = path . join ( npm . tmp , Date . now ( ) + "-" + Math . random ( ) ) p = path . join ( npm . config . get ( "cache" ) , "_git-remotes" , v )
mkdir ( path . dirname ( tmp ) , function ( er ) {
checkGitDir ( p , u , co , cb )
} )
}
function checkGitDir ( p , u , co , cb ) {
fs . stat ( p , function ( er , s ) {
if ( er ) return cloneGitRemote ( p , u , co , cb )
if ( ! s . isDirectory ( ) ) return rm ( p , function ( er ) {
if ( er ) return cb ( er ) if ( er ) return cb ( er )
exec ( npm . config . get ( "git" ) , [ "clone" , u , tmp ] , gitEnv ( ) , false cloneGitRemote ( p , u , co , cb )
, function ( er , code , stdout , stderr ) { } )
stdout = ( stdout + "\n" + stderr ) . trim ( )
if ( er ) { var git = npm . config . get ( "git" )
log . error ( "git clone " + u , stdout ) var args = [ "config" , "--get" , "remote.origin.url" ]
return cb ( er ) var env = gitEnv ( )
}
log . verbose ( "git clone " + u , stdout ) exec ( git , args , env , false , p , function ( er , code , stdout , stderr ) {
exec ( npm . config . get ( "git" ) , [ "checkout" , co ] , gitEnv ( ) , false , tmp stdoutTrimmed = ( stdout + "\n" + stderr ) . trim ( )
, function ( er , code , stdout , stderr ) { if ( er || u !== stdout . trim ( ) ) {
stdout = ( stdout + "\n" + stderr ) . trim ( ) log . warn ( "`git config --get remote.origin.url` returned "
if ( er ) { + "wrong result (" + u + ")" , stdoutTrimmed )
log . error ( "git checkout " + co , stdout ) return rm ( p , function ( er ) {
return cb ( er ) if ( er ) return cb ( er )
} cloneGitRemote ( p , u , co , cb )
log . verbose ( "git checkout " + co , stdout )
addLocalDirectory ( tmp , cb )
} ) } )
} ) }
log . verbose ( "git remote.origin.url" , stdoutTrimmed )
archiveGitRemote ( p , u , co , cb )
} )
} )
}
function cloneGitRemote ( p , u , co , cb ) {
mkdir ( p , function ( er ) {
if ( er ) return cb ( er )
exec ( npm . config . get ( "git" ) , [ "clone" , "--mirror" , u , p ] , gitEnv ( ) , false
, function ( er , code , stdout , stderr ) {
stdout = ( stdout + "\n" + stderr ) . trim ( )
if ( er ) {
log . error ( "git clone " + u , stdout )
return cb ( er )
}
log . verbose ( "git clone " + u , stdout )
archiveGitRemote ( p , u , co , cb )
} ) } )
} ) } )
} }
function archiveGitRemote ( p , u , co , cb ) {
var git = npm . config . get ( "git" )
var archive = [ "fetch" , "-a" , "origin" ]
var resolve = [ "rev-list" , "-n1" , co ]
var env = gitEnv ( )
var errState = null
var n = 0
var resolved = null
var tmp
exec ( git , archive , env , false , p , function ( er , code , stdout , stderr ) {
stdout = ( stdout + "\n" + stderr ) . trim ( )
if ( er ) {
log . error ( "git fetch -a origin (" + u + ")" , stdout )
return next ( er )
}
log . verbose ( "git fetch -a origin (" + u + ")" , stdout )
tmp = path . join ( npm . tmp , Date . now ( ) + "-" + Math . random ( ) , "tmp.tgz" )
next ( )
} )
exec ( git , resolve , env , false , p , function ( er , code , stdout , stderr ) {
stdout = ( stdout + "\n" + stderr ) . trim ( )
if ( er ) {
log . error ( "Failed resolving git HEAD (" + u + ")" , stderr )
return next ( er )
}
log . verbose ( "git rev-list -n1 " + co , stdout )
var parsed = url . parse ( u )
parsed . hash = stdout
resolved = url . format ( parsed )
next ( )
} )
function next ( er ) {
if ( errState ) return
if ( er ) return cb ( errState = er )
if ( ++ n < 2 ) return
mkdir ( path . dirname ( tmp ) , function ( er ) {
if ( er ) return cb ( er )
var gzip = zlib . createGzip ( { level : 9 } )
var git = npm . config . get ( "git" )
var args = [ "archive" , co , "--format=tar" , "--prefix=package/" ]
var out = fs . createWriteStream ( tmp )
var env = gitEnv ( )
cb = once ( cb )
var cp = spawn ( git , args , { env : env , cwd : p } )
cp . on ( "error" , cb )
cp . stderr . on ( "data" , function ( chunk ) {
log . silly ( chunk . toString ( ) , "git archive" )
} )
cp . stdout . pipe ( gzip ) . pipe ( out ) . on ( "close" , function ( ) {
addLocalTarball ( tmp , function ( er , data ) {
if ( data ) data . _ resolved = resolved
cb ( er , data )
} )
} )
} )
}
}
var gitEnv_ var gitEnv_
function gitEnv ( ) { function gitEnv ( ) {
@ -436,6 +545,7 @@ function addNamed (name, x, data, cb_) {
if ( iF . length > 1 ) return if ( iF . length > 1 ) return
function cb ( er , data ) { function cb ( er , data ) {
if ( data && ! data . _ fromGithub ) data . _ from = k
unlock ( k , function ( ) { unlock ( k , function ( ) {
var c var c
while ( c = iF . shift ( ) ) c ( er , data ) while ( c = iF . shift ( ) ) c ( er , data )
@ -643,6 +753,7 @@ function addLocal (p, name, cb_) {
log . error ( "addLocal" , "Could not install %s" , p ) log . error ( "addLocal" , "Could not install %s" , p )
return cb_ ( er ) return cb_ ( er )
} }
if ( data && ! data . _ fromGithub ) data . _ from = p
return cb_ ( er , data ) return cb_ ( er , data )
} ) } )
} }
@ -666,31 +777,33 @@ function addLocal (p, name, cb_) {
} }
function maybeGithub ( p , name , er , cb ) { function maybeGithub ( p , name , er , cb ) {
var u = "https ://github.com/" + p var u = "git ://github.com/" + p
, up = url . parse ( u ) , up = url . parse ( u )
if ( up . hash && up . hash [ 0 ] === "#" )
up . hash = up . hash . slice ( 1 )
var ref = encodeURIComponent ( up . hash || "master" )
up . pathname = path . join ( up . pathname , "tarball" , ref ) . replace ( /\\/g , "/" )
u = url . format ( up )
log . info ( "maybeGithub" , "Attempting to fetch %s from %s" , p , u ) log . info ( "maybeGithub" , "Attempting to fetch %s from %s" , p , u )
return addRemoteTarball ( u , null , name , function ( er2 , data ) {
return addRemoteGit ( u , up , name , function ( er2 , data ) {
if ( er2 ) return cb ( er ) if ( er2 ) return cb ( er )
data . _ from = u
data . _ fromGithub = true
return cb ( null , data ) return cb ( null , data )
} ) } )
} }
function addLocalTarball ( p , name , cb ) { function addLocalTarball ( p , name , cb_ ) {
if ( typeof cb !== "function" ) cb = name , name = "" if ( typeof cb_ !== "function" ) cb_ = name , name = ""
// if it's a tar, and not in place,
// if it's a tar, and not in place,
// then unzip to .tmp, add the tmp folder, and clean up tmp
// then unzip to .tmp, add the tmp folder, and clean up tmp
if ( p . indexOf ( npm . tmp ) === 0 ) return addTmpTarball ( p , name , cb ) if ( p . indexOf ( npm . tmp ) === 0 ) return addTmpTarball ( p , name , cb_ )
if ( p . indexOf ( npm . cache ) === 0 ) { if ( p . indexOf ( npm . cache ) === 0 ) {
if ( path . basename ( p ) !== "package.tgz" ) return cb ( new Error ( if ( path . basename ( p ) !== "package.tgz" ) return cb_ ( new Error (
"Not a valid cache tarball name: " + p ) ) "Not a valid cache tarball name: " + p ) )
return addPlacedTarball ( p , name , cb ) return addPlacedTarball ( p , name , cb_ )
}
function cb ( er , data ) {
if ( data ) data . _ resolved = p
return cb_ ( er , data )
} }
// just copy it over and then add the temp tarball file.
// just copy it over and then add the temp tarball file.
isaacs
12 years ago