From 9bed5dcb2c58e9169ec44bcec44b47fd92b67e37 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 8 Feb 2011 16:54:25 -0800 Subject: [PATCH] Support caching for realpath, use in module load This adds support for a cache object to be passed to the fs.realpath and fs.realpathSync functions. The Module loader keeps an object around which caches the resulting realpaths that it looks up in the process of loading modules. This means that (at least as a result of loading modules) the same files and folders are never lstat()ed more than once. To reset the cache, set require("module")._realpathCache to an empty object. To disable the caching behavior, set it to null. --- lib/fs.js | 98 +++++++++++++++++++++++++-------- lib/module.js | 7 ++- test/simple/test-fs-realpath.js | 28 +++++++++- 3 files changed, 109 insertions(+), 24 deletions(-) diff --git a/lib/fs.js b/lib/fs.js index 9645b7679a..eca2c8a615 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -514,15 +514,27 @@ if (isWindows) { // windows version fs.realpathSync = function realpathSync(p) { var p = path.resolve(p); + if (cache && Object.prototype.hasOwnProperty.call(cache, p)) { + return cache[p]; + } fs.statSync(p); + if (cache) cache[p] = p; return p; }; // windows version - fs.realpath = function(p, cb) { + fs.realpath = function(p, cache, cb) { + if (typeof cb !== 'function') { + cb = cache; + cache = null; + } var p = path.resolve(p); + if (cache && Object.prototype.hasOwnProperty.call(cache, p)) { + return cb(null, cache[p]); + } fs.stat(p, function(err) { if (err) cb(err); + if (cache) cache[p] = p; cb(null, p); }); }; @@ -535,11 +547,16 @@ if (isWindows) { var nextPartRe = /(.*?)(?:[\/]+|$)/g; // posix version - fs.realpathSync = function realpathSync(p) { + fs.realpathSync = function realpathSync(p, cache) { // make p is absolute p = path.resolve(p); - var seenLinks = {}, + if (cache && Object.prototype.hasOwnProperty.call(cache, p)) { + return cache[p]; + } + + var original = p, + seenLinks = {}, knownHard = {}; // current character position in p @@ -564,38 +581,61 @@ if (isWindows) { pos = nextPartRe.lastIndex; // continue if not a symlink, or if root - if (!base || knownHard[base]) { - continue; - } - var stat = fs.lstatSync(base); - if (!stat.isSymbolicLink()) { - knownHard[base] = true; + if (!base || knownHard[base] || (cache && cache[base] === base)) { continue; } - // read the link if it wasn't read before - var id = stat.dev.toString(32) + ':' + stat.ino.toString(32); - if (!seenLinks[id]) { - fs.statSync(base); - seenLinks[id] = fs.readlinkSync(base); + var resolvedLink; + if (cache && Object.prototype.hasOwnProperty.call(cache, base)) { + // some known symbolic link. no need to stat again. + resolvedLink = cache[base]; + } else { + var stat = fs.lstatSync(base); + if (!stat.isSymbolicLink()) { + knownHard[base] = true; + if (cache) cache[base] = base; + continue; + } + + // read the link if it wasn't read before + var id = stat.dev.toString(32) + ':' + stat.ino.toString(32); + if (!seenLinks[id]) { + fs.statSync(base); + seenLinks[id] = fs.readlinkSync(base); + resolvedLink = path.resolve(previous, seenLinks[id]); + // track this, if given a cache. + if (cache) cache[base] = resolvedLink; + } } // resolve the link, then start over - p = path.resolve(previous, seenLinks[id], p.slice(pos)); + p = path.resolve(resolvedLink, p.slice(pos)); pos = 0; previous = base = current = ''; } + if (cache) cache[original] = p; + return p; }; // posix version - fs.realpath = function realpath(p, cb) { + fs.realpath = function realpath(p, cache, cb) { + if (typeof cb !== 'function') { + cb = cache; + cache = null; + } + // make p is absolute p = path.resolve(p); - var seenLinks = {}, + if (cache && Object.prototype.hasOwnProperty.call(cache, p)) { + return cb(null, cache[p]); + } + + var original = p, + seenLinks = {}, knownHard = {}; // current character position in p @@ -613,6 +653,7 @@ if (isWindows) { function LOOP() { // stop if scanned past end of path if (pos >= p.length) { + if (cache) cache[original] = p; return cb(null, p); } @@ -624,11 +665,16 @@ if (isWindows) { base = previous + result[1]; pos = nextPartRe.lastIndex; - // continue if known to be hard or if root - if (!base || knownHard[base]) { + // continue if known to be hard or if root or in cache already. + if (!base || knownHard[base] || (cache && cache[base] === base)) { return process.nextTick(LOOP); } + if (cache && Object.prototype.hasOwnProperty.call(cache, base)) { + // known symbolic link. no need to stat again. + return gotResolvedLink(cache[base]); + } + return fs.lstat(base, gotStat); } @@ -638,6 +684,7 @@ if (isWindows) { // if not a symlink, skip to the next path part if (!stat.isSymbolicLink()) { knownHard[base] = true; + if (cache) cache[base] = base; return process.nextTick(LOOP); } @@ -645,7 +692,7 @@ if (isWindows) { // call gotTarget as soon as the link target is known var id = stat.dev.toString(32) + ':' + stat.ino.toString(32); if (seenLinks[id]) { - return gotTarget(null, seenLinks[id]); + return gotTarget(null, seenLinks[id], base); } fs.stat(base, function(err) { if (err) return cb(err); @@ -656,11 +703,18 @@ if (isWindows) { }); } - function gotTarget(err, target) { + function gotTarget(err, target, base) { if (err) return cb(err); + var resolvedLink = path.resolve(previous, target); + if (cache) cache[base] = resolvedLink; + gotResolvedLink(resolvedLink); + } + + function gotResolvedLink(resolvedLink) { + // resolve the link, then start over - p = path.resolve(previous, target, p.slice(pos)); + p = path.resolve(resolvedLink, p.slice(pos)); pos = 0; previous = base = current = ''; diff --git a/lib/module.js b/lib/module.js index 99e839ee36..b5724ffaf6 100644 --- a/lib/module.js +++ b/lib/module.js @@ -88,12 +88,17 @@ function tryPackage(requestPath, exts) { return tryFile(filename) || tryExtensions(filename, exts); } +// In order to minimize unnecessary lstat() calls, +// this cache is a list of known-real paths. +// Set to an empty object to reset. +Module._realpathCache = {} + // check if the file exists and is not a directory function tryFile(requestPath) { var fs = NativeModule.require('fs'); var stats = statPath(requestPath); if (stats && !stats.isDirectory()) { - return fs.realpathSync(requestPath); + return fs.realpathSync(requestPath, Module._realpathCache); } return false; } diff --git a/test/simple/test-fs-realpath.js b/test/simple/test-fs-realpath.js index fec0bc321e..5d412785db 100644 --- a/test/simple/test-fs-realpath.js +++ b/test/simple/test-fs-realpath.js @@ -363,6 +363,31 @@ function test_abs_with_kids(cb) { }); } +function test_lying_cache_liar(cb) { + // this should not require *any* stat calls, since everything + // checked by realpath will be found in the cache. + console.log('test_lying_cache_liar'); + var cache = { '/foo/bar/baz/bluff' : '/foo/bar/bluff', + '/1/2/3/4/5/6/7' : '/1', + '/a' : '/a', + '/a/b' : '/a/b', + '/a/b/c' : '/a/b', + '/a/b/d' : '/a/b/d' }; + var rps = fs.realpathSync('/foo/bar/baz/bluff', cache); + assert.equal(cache['/foo/bar/baz/bluff'], rps); + fs.realpath('/1/2/3/4/5/6/7', cache, function(er, rp) { + assert.equal(cache['/1/2/3/4/5/6/7'], rp); + }); + + var test = '/a/b/c/d', + expect = '/a/b/d'; + var actual = fs.realpathSync(test, cache); + assert.equal(expect, actual); + fs.realpath(test, cache, function(er, actual) { + assert.equal(expect, actual); + }); +} + // ---------------------------------------------------------------------------- var tests = [ @@ -376,7 +401,8 @@ var tests = [ test_deep_symlink_mix, test_non_symlinks, test_escape_cwd, - test_abs_with_kids + test_abs_with_kids, + test_lying_cache_liar ]; var numtests = tests.length; function runNextTest(err) {