Browse Source

fs: allow WHATWG URL and file: URLs as paths

Updates the fs module APIs to allow 'file://' URL objects
to be passed as the path.

For example:

```js
const URL = require('url').URL;
const myURL = new URL('file:///C:/path/to/file');
fs.readFile(myURL, (err, data) => {});
```

On Windows, file: URLs with a hostname convert to UNC paths,
while file: URLs with drive letters convert to local absolute
paths:

```
file://hostname/a/b/c => \\hostname\a\b\c
file:///c:/a/b/c => c:\a\b\c
```

On all other platforms, file: URLs with a hostname are unsupported
and will result in a throw:

```
file://hostname/a/b/c => throw!
file:///a/b/c => /a/b/c
```

The documentation for the fs API is intentionally not updated in
this commit because the URL API is still considered experimental
and is not officially documented *at this time*

Note that file: URLs are *required* by spec to always be absolute
paths from the file system root.

This is a semver-major commit because it changes error handling
on the fs APIs.

PR-URL: https://github.com/nodejs/node/pull/10739
Ref: https://github.com/nodejs/node/issues/10703
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>
v6
James M Snell 8 years ago
parent
commit
9549329158
  1. 92
      lib/fs.js
  2. 64
      lib/internal/url.js
  3. 60
      test/parallel/test-fs-null-bytes.js
  4. 70
      test/parallel/test-fs-whatwg-url.js

92
lib/fs.js

@ -16,9 +16,11 @@ const EventEmitter = require('events');
const FSReqWrap = binding.FSReqWrap;
const FSEvent = process.binding('fs_event_wrap').FSEvent;
const internalFS = require('internal/fs');
const internalURL = require('internal/url');
const assertEncoding = internalFS.assertEncoding;
const stringToFlags = internalFS.stringToFlags;
const SyncWriteStream = internalFS.SyncWriteStream;
const getPathFromURL = internalURL.getPathFromURL;
Object.defineProperty(exports, 'constants', {
configurable: false,
@ -202,6 +204,16 @@ fs.Stats.prototype.isSocket = function() {
});
});
function handleError(val, callback) {
if (val instanceof Error) {
if (typeof callback === 'function') {
process.nextTick(callback, val);
return true;
} else throw val;
}
return false;
}
fs.access = function(path, mode, callback) {
if (typeof mode === 'function') {
callback = mode;
@ -210,6 +222,9 @@ fs.access = function(path, mode, callback) {
throw new TypeError('"callback" argument must be a function');
}
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback))
return;
@ -220,6 +235,7 @@ fs.access = function(path, mode, callback) {
};
fs.accessSync = function(path, mode) {
handleError((path = getPathFromURL(path)));
nullCheck(path);
if (mode === undefined)
@ -231,6 +247,8 @@ fs.accessSync = function(path, mode) {
};
fs.exists = function(path, callback) {
if (handleError((path = getPathFromURL(path)), cb))
return;
if (!nullCheck(path, cb)) return;
var req = new FSReqWrap();
req.oncomplete = cb;
@ -242,6 +260,7 @@ fs.exists = function(path, callback) {
fs.existsSync = function(path) {
try {
handleError((path = getPathFromURL(path)));
nullCheck(path);
binding.stat(pathModule._makeLong(path));
return true;
@ -254,6 +273,8 @@ fs.readFile = function(path, options, callback) {
callback = maybeCallback(arguments[arguments.length - 1]);
options = getOptions(options, { flag: 'r' });
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback))
return;
@ -537,6 +558,8 @@ fs.open = function(path, flags, mode, callback_) {
var callback = makeCallback(arguments[arguments.length - 1]);
mode = modeNum(mode, 0o666);
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
@ -550,6 +573,7 @@ fs.open = function(path, flags, mode, callback_) {
fs.openSync = function(path, flags, mode) {
mode = modeNum(mode, 0o666);
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.open(pathModule._makeLong(path), stringToFlags(flags), mode);
};
@ -645,6 +669,12 @@ fs.writeSync = function(fd, buffer, offset, length, position) {
fs.rename = function(oldPath, newPath, callback) {
callback = makeCallback(callback);
if (handleError((oldPath = getPathFromURL(oldPath)), callback))
return;
if (handleError((newPath = getPathFromURL(newPath)), callback))
return;
if (!nullCheck(oldPath, callback)) return;
if (!nullCheck(newPath, callback)) return;
var req = new FSReqWrap();
@ -655,6 +685,8 @@ fs.rename = function(oldPath, newPath, callback) {
};
fs.renameSync = function(oldPath, newPath) {
handleError((oldPath = getPathFromURL(oldPath)));
handleError((newPath = getPathFromURL(newPath)));
nullCheck(oldPath);
nullCheck(newPath);
return binding.rename(pathModule._makeLong(oldPath),
@ -726,6 +758,8 @@ fs.ftruncateSync = function(fd, len) {
fs.rmdir = function(path, callback) {
callback = maybeCallback(callback);
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -733,6 +767,7 @@ fs.rmdir = function(path, callback) {
};
fs.rmdirSync = function(path) {
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.rmdir(pathModule._makeLong(path));
};
@ -760,6 +795,8 @@ fs.fsyncSync = function(fd) {
fs.mkdir = function(path, mode, callback) {
if (typeof mode === 'function') callback = mode;
callback = makeCallback(callback);
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -769,6 +806,7 @@ fs.mkdir = function(path, mode, callback) {
};
fs.mkdirSync = function(path, mode) {
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.mkdir(pathModule._makeLong(path),
modeNum(mode, 0o777));
@ -777,6 +815,8 @@ fs.mkdirSync = function(path, mode) {
fs.readdir = function(path, options, callback) {
callback = makeCallback(typeof options === 'function' ? options : callback);
options = getOptions(options, {});
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -785,6 +825,7 @@ fs.readdir = function(path, options, callback) {
fs.readdirSync = function(path, options) {
options = getOptions(options, {});
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.readdir(pathModule._makeLong(path), options.encoding);
};
@ -797,6 +838,8 @@ fs.fstat = function(fd, callback) {
fs.lstat = function(path, callback) {
callback = makeCallback(callback);
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -805,6 +848,8 @@ fs.lstat = function(path, callback) {
fs.stat = function(path, callback) {
callback = makeCallback(callback);
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -816,11 +861,13 @@ fs.fstatSync = function(fd) {
};
fs.lstatSync = function(path) {
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.lstat(pathModule._makeLong(path));
};
fs.statSync = function(path) {
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.stat(pathModule._makeLong(path));
};
@ -828,6 +875,8 @@ fs.statSync = function(path) {
fs.readlink = function(path, options, callback) {
callback = makeCallback(typeof options === 'function' ? options : callback);
options = getOptions(options, {});
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -836,6 +885,7 @@ fs.readlink = function(path, options, callback) {
fs.readlinkSync = function(path, options) {
options = getOptions(options, {});
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.readlink(pathModule._makeLong(path), options.encoding);
};
@ -859,6 +909,12 @@ fs.symlink = function(target, path, type_, callback_) {
var type = (typeof type_ === 'string' ? type_ : null);
var callback = makeCallback(arguments[arguments.length - 1]);
if (handleError((target = getPathFromURL(target)), callback))
return;
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(target, callback)) return;
if (!nullCheck(path, callback)) return;
@ -873,7 +929,8 @@ fs.symlink = function(target, path, type_, callback_) {
fs.symlinkSync = function(target, path, type) {
type = (typeof type === 'string' ? type : null);
handleError((target = getPathFromURL(target)));
handleError((path = getPathFromURL(path)));
nullCheck(target);
nullCheck(path);
@ -884,6 +941,13 @@ fs.symlinkSync = function(target, path, type) {
fs.link = function(existingPath, newPath, callback) {
callback = makeCallback(callback);
if (handleError((existingPath = getPathFromURL(existingPath)), callback))
return;
if (handleError((newPath = getPathFromURL(newPath)), callback))
return;
if (!nullCheck(existingPath, callback)) return;
if (!nullCheck(newPath, callback)) return;
@ -896,6 +960,8 @@ fs.link = function(existingPath, newPath, callback) {
};
fs.linkSync = function(existingPath, newPath) {
handleError((existingPath = getPathFromURL(existingPath)));
handleError((newPath = getPathFromURL(newPath)));
nullCheck(existingPath);
nullCheck(newPath);
return binding.link(pathModule._makeLong(existingPath),
@ -904,6 +970,8 @@ fs.linkSync = function(existingPath, newPath) {
fs.unlink = function(path, callback) {
callback = makeCallback(callback);
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -911,6 +979,7 @@ fs.unlink = function(path, callback) {
};
fs.unlinkSync = function(path) {
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.unlink(pathModule._makeLong(path));
};
@ -967,6 +1036,8 @@ if (constants.hasOwnProperty('O_SYMLINK')) {
fs.chmod = function(path, mode, callback) {
callback = makeCallback(callback);
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -976,6 +1047,7 @@ fs.chmod = function(path, mode, callback) {
};
fs.chmodSync = function(path, mode) {
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.chmod(pathModule._makeLong(path), modeNum(mode));
};
@ -1010,6 +1082,8 @@ fs.fchownSync = function(fd, uid, gid) {
fs.chown = function(path, uid, gid, callback) {
callback = makeCallback(callback);
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -1017,6 +1091,7 @@ fs.chown = function(path, uid, gid, callback) {
};
fs.chownSync = function(path, uid, gid) {
handleError((path = getPathFromURL(path)));
nullCheck(path);
return binding.chown(pathModule._makeLong(path), uid, gid);
};
@ -1044,6 +1119,8 @@ fs._toUnixTimestamp = toUnixTimestamp;
fs.utimes = function(path, atime, mtime, callback) {
callback = makeCallback(callback);
if (handleError((path = getPathFromURL(path)), callback))
return;
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
@ -1054,6 +1131,7 @@ fs.utimes = function(path, atime, mtime, callback) {
};
fs.utimesSync = function(path, atime, mtime) {
handleError((path = getPathFromURL(path)));
nullCheck(path);
atime = toUnixTimestamp(atime);
mtime = toUnixTimestamp(mtime);
@ -1214,6 +1292,7 @@ FSWatcher.prototype.start = function(filename,
persistent,
recursive,
encoding) {
handleError((filename = getPathFromURL(filename)));
nullCheck(filename);
var err = this._handle.start(pathModule._makeLong(filename),
persistent,
@ -1232,6 +1311,7 @@ FSWatcher.prototype.close = function() {
};
fs.watch = function(filename, options, listener) {
handleError((filename = getPathFromURL(filename)));
nullCheck(filename);
if (typeof options === 'function') {
@ -1292,6 +1372,7 @@ util.inherits(StatWatcher, EventEmitter);
StatWatcher.prototype.start = function(filename, persistent, interval) {
handleError((filename = getPathFromURL(filename)));
nullCheck(filename);
this._handle.start(pathModule._makeLong(filename), persistent, interval);
};
@ -1305,6 +1386,7 @@ StatWatcher.prototype.stop = function() {
const statWatchers = new Map();
fs.watchFile = function(filename, options, listener) {
handleError((filename = getPathFromURL(filename)));
nullCheck(filename);
filename = pathModule.resolve(filename);
var stat;
@ -1341,6 +1423,7 @@ fs.watchFile = function(filename, options, listener) {
};
fs.unwatchFile = function(filename, listener) {
handleError((filename = getPathFromURL(filename)));
nullCheck(filename);
filename = pathModule.resolve(filename);
var stat = statWatchers.get(filename);
@ -1384,6 +1467,7 @@ function encodeRealpathResult(result, options) {
fs.realpathSync = function realpathSync(p, options) {
options = getOptions(options, {});
handleError((p = getPathFromURL(p)));
nullCheck(p);
p = p.toString('utf8');
@ -1487,6 +1571,8 @@ fs.realpathSync = function realpathSync(p, options) {
fs.realpath = function realpath(p, options, callback) {
callback = maybeCallback(typeof options === 'function' ? options : callback);
options = getOptions(options, {});
if (handleError((p = getPathFromURL(p)), callback))
return;
if (!nullCheck(p, callback))
return;
@ -1645,7 +1731,7 @@ function ReadStream(path, options) {
Readable.call(this, options);
this.path = path;
handleError((this.path = getPathFromURL(path)));
this.fd = options.fd === undefined ? null : options.fd;
this.flags = options.flags === undefined ? 'r' : options.flags;
this.mode = options.mode === undefined ? 0o666 : options.mode;
@ -1808,7 +1894,7 @@ function WriteStream(path, options) {
Writable.call(this, options);
this.path = path;
handleError((this.path = getPathFromURL(path)));
this.fd = options.fd === undefined ? null : options.fd;
this.flags = options.flags === undefined ? 'w' : options.flags;
this.mode = options.mode === undefined ? 0o666 : options.mode;

64
lib/internal/url.js

@ -7,6 +7,9 @@ const cannotBeBase = Symbol('cannot-be-base');
const special = Symbol('special');
const searchParams = Symbol('query');
const querystring = require('querystring');
const os = require('os');
const isWindows = process.platform === 'win32';
const kScheme = Symbol('scheme');
const kHost = Symbol('host');
@ -1112,6 +1115,67 @@ function urlToOptions(url) {
return options;
}
function getPathFromURLWin32(url) {
var hostname = url.hostname;
var pathname = url.pathname;
for (var n = 0; n < pathname.length; n++) {
if (pathname[n] === '%') {
var third = pathname.codePointAt(n + 2) | 0x20;
if ((pathname[n + 1] === '2' && third === 102) || // 2f 2F /
(pathname[n + 1] === '5' && third === 99)) { // 5c 5C \
return new TypeError(
'Path must not include encoded \\ or / characters');
}
}
}
pathname = decodeURIComponent(pathname);
if (hostname !== '') {
// If hostname is set, then we have a UNC path
// Pass the hostname through domainToUnicode just in case
// it is an IDN using punycode encoding. We do not need to worry
// about percent encoding because the URL parser will have
// already taken care of that for us. Note that this only
// causes IDNs with an appropriate `xn--` prefix to be decoded.
return `//${domainToUnicode(hostname)}${pathname}`;
} else {
// Otherwise, it's a local path that requires a drive letter
var letter = pathname.codePointAt(1) | 0x20;
var sep = pathname[2];
if (letter < 97 || letter > 122 || // a..z A..Z
(sep !== ':')) {
return new TypeError('File URLs must specify absolute paths');
}
return pathname.slice(1);
}
}
function getPathFromURLPosix(url) {
if (url.hostname !== '') {
return new TypeError(
`File URLs on ${os.platform()} must use hostname 'localhost'` +
' or not specify any hostname');
}
var pathname = url.pathname;
for (var n = 0; n < pathname.length; n++) {
if (pathname[n] === '%') {
var third = pathname.codePointAt(n + 2) | 0x20;
if (pathname[n + 1] === '2' && third === 102) {
return new TypeError('Path must not include encoded / characters');
}
}
}
return decodeURIComponent(pathname);
}
function getPathFromURL(path) {
if (!(path instanceof URL))
return path;
if (path.protocol !== 'file:')
return new TypeError('Only `file:` URLs are supported');
return isWindows ? getPathFromURLWin32(path) : getPathFromURLPosix(path);
}
exports.getPathFromURL = getPathFromURL;
exports.URL = URL;
exports.URLSearchParams = URLSearchParams;
exports.domainToASCII = domainToASCII;

60
test/parallel/test-fs-null-bytes.js

@ -2,6 +2,7 @@
const common = require('../common');
const assert = require('assert');
const fs = require('fs');
const URL = require('url').URL;
function check(async, sync) {
const expected = /Path must be a string without null bytes/;
@ -48,6 +49,65 @@ check(null, fs.watch, 'foo\u0000bar', common.fail);
check(null, fs.watchFile, 'foo\u0000bar', common.fail);
check(fs.writeFile, fs.writeFileSync, 'foo\u0000bar');
const fileUrl = new URL('file:///C:/foo\u0000bar');
const fileUrl2 = new URL('file:///C:/foo%00bar');
check(fs.access, fs.accessSync, fileUrl);
check(fs.access, fs.accessSync, fileUrl, fs.F_OK);
check(fs.appendFile, fs.appendFileSync, fileUrl);
check(fs.chmod, fs.chmodSync, fileUrl, '0644');
check(fs.chown, fs.chownSync, fileUrl, 12, 34);
check(fs.link, fs.linkSync, fileUrl, 'foobar');
check(fs.link, fs.linkSync, 'foobar', fileUrl);
check(fs.lstat, fs.lstatSync, fileUrl);
check(fs.mkdir, fs.mkdirSync, fileUrl, '0755');
check(fs.open, fs.openSync, fileUrl, 'r');
check(fs.readFile, fs.readFileSync, fileUrl);
check(fs.readdir, fs.readdirSync, fileUrl);
check(fs.readlink, fs.readlinkSync, fileUrl);
check(fs.realpath, fs.realpathSync, fileUrl);
check(fs.rename, fs.renameSync, fileUrl, 'foobar');
check(fs.rename, fs.renameSync, 'foobar', fileUrl);
check(fs.rmdir, fs.rmdirSync, fileUrl);
check(fs.stat, fs.statSync, fileUrl);
check(fs.symlink, fs.symlinkSync, fileUrl, 'foobar');
check(fs.symlink, fs.symlinkSync, 'foobar', fileUrl);
check(fs.truncate, fs.truncateSync, fileUrl);
check(fs.unlink, fs.unlinkSync, fileUrl);
check(null, fs.unwatchFile, fileUrl, common.fail);
check(fs.utimes, fs.utimesSync, fileUrl, 0, 0);
check(null, fs.watch, fileUrl, common.fail);
check(null, fs.watchFile, fileUrl, common.fail);
check(fs.writeFile, fs.writeFileSync, fileUrl);
check(fs.access, fs.accessSync, fileUrl2);
check(fs.access, fs.accessSync, fileUrl2, fs.F_OK);
check(fs.appendFile, fs.appendFileSync, fileUrl2);
check(fs.chmod, fs.chmodSync, fileUrl2, '0644');
check(fs.chown, fs.chownSync, fileUrl2, 12, 34);
check(fs.link, fs.linkSync, fileUrl2, 'foobar');
check(fs.link, fs.linkSync, 'foobar', fileUrl2);
check(fs.lstat, fs.lstatSync, fileUrl2);
check(fs.mkdir, fs.mkdirSync, fileUrl2, '0755');
check(fs.open, fs.openSync, fileUrl2, 'r');
check(fs.readFile, fs.readFileSync, fileUrl2);
check(fs.readdir, fs.readdirSync, fileUrl2);
check(fs.readlink, fs.readlinkSync, fileUrl2);
check(fs.realpath, fs.realpathSync, fileUrl2);
check(fs.rename, fs.renameSync, fileUrl2, 'foobar');
check(fs.rename, fs.renameSync, 'foobar', fileUrl2);
check(fs.rmdir, fs.rmdirSync, fileUrl2);
check(fs.stat, fs.statSync, fileUrl2);
check(fs.symlink, fs.symlinkSync, fileUrl2, 'foobar');
check(fs.symlink, fs.symlinkSync, 'foobar', fileUrl2);
check(fs.truncate, fs.truncateSync, fileUrl2);
check(fs.unlink, fs.unlinkSync, fileUrl2);
check(null, fs.unwatchFile, fileUrl2, common.fail);
check(fs.utimes, fs.utimesSync, fileUrl2, 0, 0);
check(null, fs.watch, fileUrl2, common.fail);
check(null, fs.watchFile, fileUrl2, common.fail);
check(fs.writeFile, fs.writeFileSync, fileUrl2);
// an 'error' for exists means that it doesn't exist.
// one of many reasons why this file is the absolute worst.
fs.exists('foo\u0000bar', common.mustCall((exists) => {

70
test/parallel/test-fs-whatwg-url.js

@ -0,0 +1,70 @@
'use strict';
const common = require('../common');
const assert = require('assert');
const path = require('path');
const fs = require('fs');
const os = require('os');
const URL = require('url').URL;
const Buffer = require('buffer').Buffer;
function pathToFileURL(p) {
if (!path.isAbsolute(p))
throw new Error('Path must be absolute');
if (common.isWindows && p.startsWith('\\\\'))
p = p.slice(2);
return new URL(`file://${p}`);
}
const p = path.resolve(common.fixturesDir, 'a.js');
const url = pathToFileURL(p);
assert(url instanceof URL);
// Check that we can pass in a URL object successfully
fs.readFile(url, common.mustCall((err, data) => {
assert.ifError(err);
assert(Buffer.isBuffer(data));
}));
// Check that using a non file:// URL reports an error
const httpUrl = new URL('http://example.org');
fs.readFile(httpUrl, common.mustCall((err) => {
assert(err);
assert.strictEqual(err.message,
'Only `file:` URLs are supported');
}));
// pct-encoded characters in the path will be decoded and checked
fs.readFile(new URL('file:///c:/tmp/%00test'), common.mustCall((err) => {
assert(err);
assert.strictEqual(err.message,
'Path must be a string without null bytes');
}));
if (common.isWindows) {
// encoded back and forward slashes are not permitted on windows
['%2f', '%2F', '%5c', '%5C'].forEach((i) => {
fs.readFile(new URL(`file:///c:/tmp/${i}`), common.mustCall((err) => {
assert(err);
assert.strictEqual(err.message,
'Path must not include encoded \\ or / characters');
}));
});
} else {
// encoded forward slashes are not permitted on other platforms
['%2f', '%2F'].forEach((i) => {
fs.readFile(new URL(`file:///c:/tmp/${i}`), common.mustCall((err) => {
assert(err);
assert.strictEqual(err.message,
'Path must not include encoded / characters');
}));
});
fs.readFile(new URL('file://hostname/a/b/c'), common.mustCall((err) => {
assert(err);
assert.strictEqual(err.message,
`File URLs on ${os.platform()} must use ` +
'hostname \'localhost\' or not specify any hostname');
}));
}
Loading…
Cancel
Save