From c8a389e19f172edbada83f59944cad7cc802d9d5 Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Mon, 5 Jun 2017 19:44:56 -0500 Subject: [PATCH] module: Allow runMain to be ESM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This follows the EPS an allows the node CLI to have ESM as an entry point. `node ./example.mjs`. A newer V8 is needed for `import()` so that is not included. `import.meta` is still in specification stage so that also is not included. PR-URL: https://github.com/nodejs/node/pull/14369 Author: Bradley Farias Author: Guy Bedford Author: Jan Krems Author: Timothy Gu Author: Michaël Zasso Author: Anna Henningsen Reviewed-By: James M Snell Reviewed-By: Jeremiah Senkpiel --- .eslintrc.yaml | 5 + Makefile | 4 +- doc/api/esm.md | 88 +++ lib/internal/bootstrap_node.js | 7 + lib/internal/errors.js | 4 + lib/internal/loader/Loader.js | 75 +++ lib/internal/loader/ModuleJob.js | 116 ++++ lib/internal/loader/ModuleMap.js | 33 ++ lib/internal/loader/ModuleWrap.js | 61 ++ lib/internal/loader/resolveRequestUrl.js | 104 ++++ lib/internal/loader/search.js | 33 ++ lib/internal/safe_globals.js | 26 + lib/internal/url.js | 7 + lib/module.js | 58 +- node.gyp | 9 + src/module_wrap.cc | 531 ++++++++++++++++++ src/module_wrap.h | 58 ++ src/node.cc | 8 + src/node_config.cc | 3 + src/node_internals.h | 4 + src/node_url.cc | 63 +++ src/node_url.h | 4 + test/cctest/test_url.cc | 25 + test/es-module/es-module.status | 7 + test/es-module/esm-snapshot-mutator.js | 5 + test/es-module/esm-snapshot.js | 3 + test/es-module/test-esm-basic-imports.mjs | 8 + .../es-module/test-esm-encoded-path-native.js | 10 + test/es-module/test-esm-encoded-path.mjs | 7 + test/es-module/test-esm-forbidden-globals.mjs | 24 + test/es-module/test-esm-namespace.mjs | 7 + test/es-module/test-esm-ok.mjs | 5 + test/es-module/test-esm-pkg-over-ext.mjs | 8 + test/es-module/test-esm-preserve-symlinks.js | 38 ++ test/es-module/test-esm-require-cache.mjs | 7 + test/es-module/test-esm-shebang.mjs | 6 + test/es-module/test-esm-snapshot.mjs | 7 + test/es-module/test-esm-symlink.js | 48 ++ test/es-module/testcfg.py | 6 + .../es-module-require-cache/counter.js | 2 + .../es-module-require-cache/preload.js | 1 + test/fixtures/es-module-url/empty.js | 0 test/fixtures/es-module-url/native.mjs | 2 + test/testpy/__init__.py | 13 +- tools/eslint-rules/required-modules.js | 60 +- tools/test.py | 18 +- 46 files changed, 1578 insertions(+), 40 deletions(-) create mode 100644 doc/api/esm.md create mode 100644 lib/internal/loader/Loader.js create mode 100644 lib/internal/loader/ModuleJob.js create mode 100644 lib/internal/loader/ModuleMap.js create mode 100644 lib/internal/loader/ModuleWrap.js create mode 100644 lib/internal/loader/resolveRequestUrl.js create mode 100644 lib/internal/loader/search.js create mode 100644 lib/internal/safe_globals.js create mode 100644 src/module_wrap.cc create mode 100644 src/module_wrap.h create mode 100644 test/es-module/es-module.status create mode 100644 test/es-module/esm-snapshot-mutator.js create mode 100644 test/es-module/esm-snapshot.js create mode 100644 test/es-module/test-esm-basic-imports.mjs create mode 100644 test/es-module/test-esm-encoded-path-native.js create mode 100644 test/es-module/test-esm-encoded-path.mjs create mode 100644 test/es-module/test-esm-forbidden-globals.mjs create mode 100644 test/es-module/test-esm-namespace.mjs create mode 100644 test/es-module/test-esm-ok.mjs create mode 100644 test/es-module/test-esm-pkg-over-ext.mjs create mode 100644 test/es-module/test-esm-preserve-symlinks.js create mode 100644 test/es-module/test-esm-require-cache.mjs create mode 100644 test/es-module/test-esm-shebang.mjs create mode 100644 test/es-module/test-esm-snapshot.mjs create mode 100644 test/es-module/test-esm-symlink.js create mode 100644 test/es-module/testcfg.py create mode 100644 test/fixtures/es-module-require-cache/counter.js create mode 100644 test/fixtures/es-module-require-cache/preload.js create mode 100644 test/fixtures/es-module-url/empty.js create mode 100644 test/fixtures/es-module-url/native.mjs diff --git a/.eslintrc.yaml b/.eslintrc.yaml index 36677b7cea..b68d265746 100644 --- a/.eslintrc.yaml +++ b/.eslintrc.yaml @@ -10,6 +10,11 @@ env: parserOptions: ecmaVersion: 2017 +overrides: + - files: ["doc/api/esm.md", "*.mjs"] + parserOptions: + sourceType: module + rules: # Possible Errors # http://eslint.org/docs/rules/#possible-errors diff --git a/Makefile b/Makefile index 67a568cfa5..547d93dc15 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ coverage-build: all "$(CURDIR)/testing/coverage/gcovr-patches.diff"); fi if [ -d lib_ ]; then $(RM) -r lib; mv lib_ lib; fi mv lib lib_ - $(NODE) ./node_modules/.bin/nyc instrument lib_/ lib/ + $(NODE) ./node_modules/.bin/nyc instrument --extension .js --extension .mjs lib_/ lib/ $(MAKE) coverage-test: coverage-build @@ -886,7 +886,7 @@ JSLINT_TARGETS = benchmark doc lib test tools jslint: @echo "Running JS linter..." - $(NODE) tools/eslint/bin/eslint.js --cache --rulesdir=tools/eslint-rules --ext=.js,.md \ + $(NODE) tools/eslint/bin/eslint.js --cache --rulesdir=tools/eslint-rules --ext=.js,.mjs,.md \ $(JSLINT_TARGETS) jslint-ci: diff --git a/doc/api/esm.md b/doc/api/esm.md new file mode 100644 index 0000000000..108fd76336 --- /dev/null +++ b/doc/api/esm.md @@ -0,0 +1,88 @@ +# ECMAScript Modules + + + +> Stability: 1 - Experimental + + + +Node contains support for ES Modules based upon the [the Node EP for ES Modules][]. + +Not all features of the EP are complete and will be landing as both VM support and implementation is ready. Error messages are still being polished. + +## Enabling + + + +The `--experimental-modules` flag can be used to enable features for loading ESM modules. + +Once this has been set, files ending with `.mjs` will be able to be loaded as ES Modules. + +```sh +node --experimental-modules my-app.mjs +``` + +## Features + + + +### Supported + +Only the CLI argument for the main entry point to the program can be an entry point into an ESM graph. In the future `import()` can be used to create entry points into ESM graphs at run time. + +### Unsupported + +| Feature | Reason | +| --- | --- | +| `require('./foo.mjs')` | ES Modules have differing resolution and timing, use language standard `import()` | +| `import()` | pending newer V8 release used in Node.js | +| `import.meta` | pending V8 implementation | +| Loader Hooks | pending Node.js EP creation/consensus | + +## Notable differences between `import` and `require` + +### No NODE_PATH + +`NODE_PATH` is not part of resolving `import` specifiers. Please use symlinks if this behavior is desired. + +### No `require.extensions` + +`require.extensions` is not used by `import`. The expectation is that loader hooks can provide this workflow in the future. + +### No `require.cache` + +`require.cache` is not used by `import`. It has a separate cache. + +### URL based paths + +ESM are resolved and cached based upon [URL](url.spec.whatwg.org) semantics. This means that files containing special characters such as `#` and `?` need to be escaped. + +Modules will be loaded multiple times if the `import` specifier used to resolve them have a different query or fragment. + +```js +import './foo?query=1'; // loads ./foo with query of "?query=1" +import './foo?query=2'; // loads ./foo with query of "?query=2" +``` + +For now, only modules using the `file:` protocol can be loaded. + +## Interop with existing modules + +All CommonJS, JSON, and C++ modules can be used with `import`. + +Modules loaded this way will only be loaded once, even if their query or fragment string differs between `import` statements. + +When loaded via `import` these modules will provide a single `default` export representing the value of `module.exports` at the time they finished evaluating. + +```js +import fs from 'fs'; +fs.readFile('./foo.txt', (err, body) => { + if (err) { + console.error(err); + } else { + console.log(body); + } +}); +``` + +[the Node EP for ES Modules]: https://github.com/nodejs/node-eps/blob/master/002-es-modules.md diff --git a/lib/internal/bootstrap_node.js b/lib/internal/bootstrap_node.js index 9d776674d2..b43a262682 100644 --- a/lib/internal/bootstrap_node.js +++ b/lib/internal/bootstrap_node.js @@ -109,6 +109,13 @@ 'DeprecationWarning', 'DEP0062', startup, true); } + if (process.binding('config').experimentalModules) { + process.emitWarning( + 'The ESM module loader is experimental.', + 'ExperimentalWarning', undefined); + } + + // There are various modes that Node can run in. The most common two // are running from a script and running the REPL - but there are a few // others like the debugger or running --eval arguments. Here we decide diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 623e57d875..10e5be8a44 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -229,6 +229,9 @@ E('ERR_IPC_ONE_PIPE', 'Child process can have only one IPC pipe'); E('ERR_IPC_SYNC_FORK', 'IPC cannot be used with synchronous forks'); E('ERR_METHOD_NOT_IMPLEMENTED', 'The %s method is not implemented'); E('ERR_MISSING_ARGS', missingArgs); +E('ERR_MISSING_MODULE', 'Cannot find module %s'); +E('ERR_MODULE_RESOLUTION_LEGACY', '%s not found by import in %s.' + + 'Legacy behavior in require would have found it at %s'); E('ERR_MULTIPLE_CALLBACK', 'Callback called multiple times'); E('ERR_NAPI_CONS_FUNCTION', 'Constructor must be a function'); E('ERR_NAPI_CONS_PROTOTYPE_OBJECT', 'Constructor.prototype must be an object'); @@ -237,6 +240,7 @@ E('ERR_NO_ICU', '%s is not supported on Node.js compiled without ICU'); E('ERR_NO_LONGER_SUPPORTED', '%s is no longer supported'); E('ERR_OUTOFMEMORY', 'Out of memory'); E('ERR_PARSE_HISTORY_DATA', 'Could not parse history data in %s'); +E('ERR_REQUIRE_ESM', 'Must use import to load ES Module: %s'); E('ERR_SERVER_ALREADY_LISTEN', 'Listen method has been called more than once without closing.'); E('ERR_SOCKET_ALREADY_BOUND', 'Socket is already bound'); diff --git a/lib/internal/loader/Loader.js b/lib/internal/loader/Loader.js new file mode 100644 index 0000000000..a409d397f8 --- /dev/null +++ b/lib/internal/loader/Loader.js @@ -0,0 +1,75 @@ +'use strict'; + +const { URL } = require('url'); +const { getURLFromFilePath } = require('internal/url'); + +const { + getNamespaceOfModuleWrap +} = require('internal/loader/ModuleWrap'); + +const ModuleMap = require('internal/loader/ModuleMap'); +const ModuleJob = require('internal/loader/ModuleJob'); +const resolveRequestUrl = require('internal/loader/resolveRequestUrl'); +const errors = require('internal/errors'); + +function getBase() { + try { + return getURLFromFilePath(`${process.cwd()}/`); + } catch (e) { + e.stack; + // If the current working directory no longer exists. + if (e.code === 'ENOENT') { + return undefined; + } + throw e; + } +} + +class Loader { + constructor(base = getBase()) { + this.moduleMap = new ModuleMap(); + if (typeof base !== 'undefined' && base instanceof URL !== true) { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'URL'); + } + this.base = base; + } + + async resolve(specifier) { + const request = resolveRequestUrl(this.base, specifier); + if (request.url.protocol !== 'file:') { + throw new errors.Error('ERR_INVALID_PROTOCOL', + request.url.protocol, 'file:'); + } + return request.url; + } + + async getModuleJob(dependentJob, specifier) { + if (!this.moduleMap.has(dependentJob.url)) { + throw new errors.Error('ERR_MISSING_MODULE', dependentJob.url); + } + const request = await resolveRequestUrl(dependentJob.url, specifier); + const url = `${request.url}`; + if (this.moduleMap.has(url)) { + return this.moduleMap.get(url); + } + const dependencyJob = new ModuleJob(this, request); + this.moduleMap.set(url, dependencyJob); + return dependencyJob; + } + + async import(specifier) { + const request = await resolveRequestUrl(this.base, specifier); + const url = `${request.url}`; + let job; + if (this.moduleMap.has(url)) { + job = this.moduleMap.get(url); + } else { + job = new ModuleJob(this, request); + this.moduleMap.set(url, job); + } + const module = await job.run(); + return getNamespaceOfModuleWrap(module); + } +} +Object.setPrototypeOf(Loader.prototype, null); +module.exports = Loader; diff --git a/lib/internal/loader/ModuleJob.js b/lib/internal/loader/ModuleJob.js new file mode 100644 index 0000000000..db4cb6ae5c --- /dev/null +++ b/lib/internal/loader/ModuleJob.js @@ -0,0 +1,116 @@ +'use strict'; + +const { SafeSet, SafePromise } = require('internal/safe_globals'); +const resolvedPromise = SafePromise.resolve(); +const resolvedArrayPromise = SafePromise.resolve([]); +const { ModuleWrap } = require('internal/loader/ModuleWrap'); + +const NOOP = () => { /* No-op */ }; +class ModuleJob { + /** + * @param {module: ModuleWrap?, compiled: Promise} moduleProvider + */ + constructor(loader, moduleProvider, url) { + this.url = `${moduleProvider.url}`; + this.moduleProvider = moduleProvider; + this.loader = loader; + this.error = null; + this.hadError = false; + + if (moduleProvider instanceof ModuleWrap !== true) { + // linked == promise for dependency jobs, with module populated, + // module wrapper linked + this.modulePromise = this.moduleProvider.createModule(); + this.module = undefined; + const linked = async () => { + const dependencyJobs = []; + this.module = await this.modulePromise; + this.module.link(async (dependencySpecifier) => { + const dependencyJobPromise = + this.loader.getModuleJob(this, dependencySpecifier); + dependencyJobs.push(dependencyJobPromise); + const dependencyJob = await dependencyJobPromise; + return dependencyJob.modulePromise; + }); + return SafePromise.all(dependencyJobs); + }; + this.linked = linked(); + + // instantiated == deep dependency jobs wrappers instantiated, + //module wrapper instantiated + this.instantiated = undefined; + } else { + const getModuleProvider = async () => moduleProvider; + this.modulePromise = getModuleProvider(); + this.moduleProvider = { finish: NOOP }; + this.module = moduleProvider; + this.linked = resolvedArrayPromise; + this.instantiated = this.modulePromise; + } + } + + instantiate() { + if (this.instantiated) { + return this.instantiated; + } + return this.instantiated = new Promise(async (resolve, reject) => { + const jobsInGraph = new SafeSet(); + let jobsReadyToInstantiate = 0; + // (this must be sync for counter to work) + const queueJob = (moduleJob) => { + if (jobsInGraph.has(moduleJob)) { + return; + } + jobsInGraph.add(moduleJob); + moduleJob.linked.then((dependencyJobs) => { + for (const dependencyJob of dependencyJobs) { + queueJob(dependencyJob); + } + checkComplete(); + }, (e) => { + if (!this.hadError) { + this.error = e; + this.hadError = true; + } + checkComplete(); + }); + }; + const checkComplete = () => { + if (++jobsReadyToInstantiate === jobsInGraph.size) { + // I believe we only throw once the whole tree is finished loading? + // or should the error bail early, leaving entire tree to still load? + if (this.hadError) { + reject(this.error); + } else { + try { + this.module.instantiate(); + for (const dependencyJob of jobsInGraph) { + dependencyJob.instantiated = resolvedPromise; + } + resolve(this.module); + } catch (e) { + e.stack; + reject(e); + } + } + } + }; + queueJob(this); + }); + } + + async run() { + const module = await this.instantiate(); + try { + module.evaluate(); + } catch (e) { + e.stack; + this.hadError = true; + this.error = e; + throw e; + } + return module; + } +} +Object.setPrototypeOf(ModuleJob.prototype, null); +module.exports = ModuleJob; diff --git a/lib/internal/loader/ModuleMap.js b/lib/internal/loader/ModuleMap.js new file mode 100644 index 0000000000..aa238afbae --- /dev/null +++ b/lib/internal/loader/ModuleMap.js @@ -0,0 +1,33 @@ +'use strict'; + +const ModuleJob = require('internal/loader/ModuleJob'); +const { SafeMap } = require('internal/safe_globals'); +const debug = require('util').debuglog('esm'); +const errors = require('internal/errors'); + +// Tracks the state of the loader-level module cache +class ModuleMap extends SafeMap { + get(url) { + if (typeof url !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string'); + } + return super.get(url); + } + set(url, job) { + if (typeof url !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string'); + } + if (job instanceof ModuleJob !== true) { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'job', 'ModuleJob'); + } + debug(`Storing ${url} in ModuleMap`); + return super.set(url, job); + } + has(url) { + if (typeof url !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string'); + } + return super.has(url); + } +} +module.exports = ModuleMap; diff --git a/lib/internal/loader/ModuleWrap.js b/lib/internal/loader/ModuleWrap.js new file mode 100644 index 0000000000..4d35356ec2 --- /dev/null +++ b/lib/internal/loader/ModuleWrap.js @@ -0,0 +1,61 @@ +'use strict'; + +const { ModuleWrap } = process.binding('module_wrap'); +const debug = require('util').debuglog('esm'); +const ArrayJoin = Function.call.bind(Array.prototype.join); +const ArrayMap = Function.call.bind(Array.prototype.map); + +const getNamespaceOfModuleWrap = (m) => { + const tmp = new ModuleWrap('import * as _ from "";_;', ''); + tmp.link(async () => m); + tmp.instantiate(); + return tmp.evaluate(); +}; + +const createDynamicModule = (exports, url = '', evaluate) => { + debug( + `creating ESM facade for ${url} with exports: ${ArrayJoin(exports, ', ')}` + ); + const names = ArrayMap(exports, (name) => `${name}`); + // sanitized ESM for reflection purposes + const src = `export let executor; + ${ArrayJoin(ArrayMap(names, (name) => `export let $${name}`), ';\n')} + ;(() => [ + fn => executor = fn, + { exports: { ${ + ArrayJoin(ArrayMap(names, (name) => `${name}: { + get: () => $${name}, + set: v => $${name} = v + }`), ',\n') +} } } + ]); + `; + const reflectiveModule = new ModuleWrap(src, `cjs-facade:${url}`); + reflectiveModule.instantiate(); + const [setExecutor, reflect] = reflectiveModule.evaluate()(); + // public exposed ESM + const reexports = `import { executor, + ${ArrayMap(names, (name) => `$${name}`)} + } from ""; + export { + ${ArrayJoin(ArrayMap(names, (name) => `$${name} as ${name}`), ', ')} + } + // add await to this later if top level await comes along + typeof executor === "function" ? executor() : void 0;`; + if (typeof evaluate === 'function') { + setExecutor(() => evaluate(reflect)); + } + const runner = new ModuleWrap(reexports, `${url}`); + runner.link(async () => reflectiveModule); + runner.instantiate(); + return { + module: runner, + reflect + }; +}; + +module.exports = { + createDynamicModule, + getNamespaceOfModuleWrap, + ModuleWrap +}; diff --git a/lib/internal/loader/resolveRequestUrl.js b/lib/internal/loader/resolveRequestUrl.js new file mode 100644 index 0000000000..2245064bfe --- /dev/null +++ b/lib/internal/loader/resolveRequestUrl.js @@ -0,0 +1,104 @@ +'use strict'; + +const { URL } = require('url'); +const internalCJSModule = require('internal/module'); +const internalURLModule = require('internal/url'); +const internalFS = require('internal/fs'); +const NativeModule = require('native_module'); +const { extname } = require('path'); +const { realpathSync } = require('fs'); +const preserveSymlinks = !!process.binding('config').preserveSymlinks; +const { + ModuleWrap, + createDynamicModule +} = require('internal/loader/ModuleWrap'); +const errors = require('internal/errors'); + +const search = require('internal/loader/search'); +const asyncReadFile = require('util').promisify(require('fs').readFile); +const debug = require('util').debuglog('esm'); + +const realpathCache = new Map(); + +class ModuleRequest { + constructor(url) { + this.url = url; + } +} +Object.setPrototypeOf(ModuleRequest.prototype, null); + +// Strategy for loading a standard JavaScript module +class StandardModuleRequest extends ModuleRequest { + async createModule() { + const source = `${await asyncReadFile(this.url)}`; + debug(`Loading StandardModule ${this.url}`); + return new ModuleWrap(internalCJSModule.stripShebang(source), + `${this.url}`); + } +} + +// Strategy for loading a node-style CommonJS module +class CJSModuleRequest extends ModuleRequest { + async createModule() { + const ctx = createDynamicModule(['default'], this.url, (reflect) => { + debug(`Loading CJSModule ${this.url.pathname}`); + const CJSModule = require('module'); + const pathname = internalURLModule.getPathFromURL(this.url); + CJSModule._load(pathname); + }); + this.finish = (module) => { + ctx.reflect.exports.default.set(module.exports); + }; + return ctx.module; + } +} + +// Strategy for loading a node builtin CommonJS module that isn't +// through normal resolution +class NativeModuleRequest extends CJSModuleRequest { + async createModule() { + const ctx = createDynamicModule(['default'], this.url, (reflect) => { + debug(`Loading NativeModule ${this.url.pathname}`); + const exports = require(this.url.pathname); + reflect.exports.default.set(exports); + }); + return ctx.module; + } +} + +const normalizeBaseURL = (baseURLOrString) => { + if (baseURLOrString instanceof URL) return baseURLOrString; + if (typeof baseURLOrString === 'string') return new URL(baseURLOrString); + return undefined; +}; + +const resolveRequestUrl = (baseURLOrString, specifier) => { + if (NativeModule.nonInternalExists(specifier)) { + return new NativeModuleRequest(new URL(`node:${specifier}`)); + } + + const baseURL = normalizeBaseURL(baseURLOrString); + let url = search(specifier, baseURL); + + if (url.protocol !== 'file:') { + throw new errors.Error('ERR_INVALID_PROTOCOL', url.protocol, 'file:'); + } + + if (!preserveSymlinks) { + const real = realpathSync(internalURLModule.getPathFromURL(url), { + [internalFS.realpathCacheKey]: realpathCache + }); + const old = url; + url = internalURLModule.getURLFromFilePath(real); + url.search = old.search; + url.hash = old.hash; + } + + const ext = extname(url.pathname); + if (ext === '.mjs') { + return new StandardModuleRequest(url); + } + + return new CJSModuleRequest(url); +}; +module.exports = resolveRequestUrl; diff --git a/lib/internal/loader/search.js b/lib/internal/loader/search.js new file mode 100644 index 0000000000..f0ec34ae4e --- /dev/null +++ b/lib/internal/loader/search.js @@ -0,0 +1,33 @@ +'use strict'; + +const { URL } = require('url'); +const CJSmodule = require('module'); +const errors = require('internal/errors'); +const { resolve } = process.binding('module_wrap'); + +module.exports = (target, base) => { + target = `${target}`; + if (base === undefined) { + // We cannot search without a base. + throw new errors.Error('ERR_MISSING_MODULE', target); + } + base = `${base}`; + try { + return resolve(target, base); + } catch (e) { + e.stack; // cause V8 to generate stack before rethrow + let error = e; + try { + const questionedBase = new URL(base); + const tmpMod = new CJSmodule(questionedBase.pathname, null); + tmpMod.paths = CJSmodule._nodeModulePaths( + new URL('./', questionedBase).pathname); + const found = CJSmodule._resolveFilename(target, tmpMod); + error = new errors.Error('ERR_MODULE_RESOLUTION_LEGACY', target, + base, found); + } catch (problemChecking) { + // ignore + } + throw error; + } +}; diff --git a/lib/internal/safe_globals.js b/lib/internal/safe_globals.js new file mode 100644 index 0000000000..ad58fa662b --- /dev/null +++ b/lib/internal/safe_globals.js @@ -0,0 +1,26 @@ +'use strict'; + +const copyProps = (unsafe, safe) => { + for (const key of [...Object.getOwnPropertyNames(unsafe), + ...Object.getOwnPropertySymbols(unsafe) + ]) { + if (!Object.getOwnPropertyDescriptor(safe, key)) { + Object.defineProperty( + safe, + key, + Object.getOwnPropertyDescriptor(unsafe, key)); + } + } +}; +const makeSafe = (unsafe, safe) => { + copyProps(unsafe.prototype, safe.prototype); + copyProps(unsafe, safe); + Object.setPrototypeOf(safe.prototype, null); + Object.freeze(safe.prototype); + Object.freeze(safe); + return safe; +}; + +exports.SafeMap = makeSafe(Map, class SafeMap extends Map {}); +exports.SafeSet = makeSafe(Set, class SafeSet extends Set {}); +exports.SafePromise = makeSafe(Promise, class SafePromise extends Promise {}); diff --git a/lib/internal/url.js b/lib/internal/url.js index 95b5e9c5fd..cf0271691a 100644 --- a/lib/internal/url.js +++ b/lib/internal/url.js @@ -1377,6 +1377,12 @@ function getPathFromURL(path) { return isWindows ? getPathFromURLWin32(path) : getPathFromURLPosix(path); } +function getURLFromFilePath(filepath) { + const tmp = new URL('file://'); + tmp.pathname = filepath; + return tmp; +} + function NativeURL(ctx) { this[context] = ctx; } @@ -1405,6 +1411,7 @@ setURLConstructor(constructUrl); module.exports = { toUSVString, getPathFromURL, + getURLFromFilePath, URL, URLSearchParams, domainToASCII, diff --git a/lib/module.js b/lib/module.js index 7bb8288f54..0b87cf7480 100644 --- a/lib/module.js +++ b/lib/module.js @@ -24,6 +24,7 @@ const NativeModule = require('native_module'); const util = require('util'); const internalModule = require('internal/module'); +const { getURLFromFilePath } = require('internal/url'); const vm = require('vm'); const assert = require('assert').ok; const fs = require('fs'); @@ -32,6 +33,14 @@ const path = require('path'); const internalModuleReadFile = process.binding('fs').internalModuleReadFile; const internalModuleStat = process.binding('fs').internalModuleStat; const preserveSymlinks = !!process.binding('config').preserveSymlinks; +const experimentalModules = !!process.binding('config').experimentalModules; + +const errors = require('internal/errors'); + +const Loader = require('internal/loader/Loader'); +const ModuleJob = require('internal/loader/ModuleJob'); +const { createDynamicModule } = require('internal/loader/ModuleWrap'); +const ESMLoader = new Loader(); function stat(filename) { filename = path._makeLong(filename); @@ -412,7 +421,36 @@ Module._load = function(request, parent, isMain) { debug('Module._load REQUEST %s parent: %s', request, parent.id); } - var filename = Module._resolveFilename(request, parent, isMain); + var filename = null; + + if (isMain) { + let err; + try { + filename = Module._resolveFilename(request, parent, isMain); + } catch (e) { + // try to keep stack + e.stack; + err = e; + } + if (experimentalModules) { + if (filename === null || /\.mjs$/.test(filename)) { + try { + ESMLoader.import(request).catch((e) => { + console.error(e); + process.exit(1); + }); + return; + } catch (e) { + // well, it isn't ESM + } + } + } + if (err) { + throw err; + } + } else { + filename = Module._resolveFilename(request, parent, isMain); + } var cachedModule = Module._cache[filename]; if (cachedModule) { @@ -482,6 +520,19 @@ Module.prototype.load = function(filename) { if (!Module._extensions[extension]) extension = '.js'; Module._extensions[extension](this, filename); this.loaded = true; + + if (experimentalModules) { + const url = getURLFromFilePath(filename); + if (ESMLoader.moduleMap.has(`${url}`) !== true) { + const ctx = createDynamicModule(['default'], url); + ctx.reflect.exports.default.set(this.exports); + ESMLoader.moduleMap.set(`${url}`, + new ModuleJob(ESMLoader, ctx.module)); + } else { + ESMLoader.moduleMap.get(`${url}`).moduleProvider.finish( + Module._cache[filename]); + } + } }; @@ -578,6 +629,11 @@ Module._extensions['.node'] = function(module, filename) { return process.dlopen(module, path._makeLong(filename)); }; +if (experimentalModules) { + Module._extensions['.mjs'] = function(module, filename) { + throw new errors.Error('ERR_REQUIRE_ESM', filename); + }; +} // bootstrap main module. Module.runMain = function() { diff --git a/node.gyp b/node.gyp index 14acf375e1..79d9e0a68d 100644 --- a/node.gyp +++ b/node.gyp @@ -91,6 +91,13 @@ 'lib/internal/http.js', 'lib/internal/inspector_async_hook.js', 'lib/internal/linkedlist.js', + 'lib/internal/loader/Loader.js', + 'lib/internal/loader/ModuleMap.js', + 'lib/internal/loader/ModuleJob.js', + 'lib/internal/loader/ModuleWrap.js', + 'lib/internal/loader/resolveRequestUrl.js', + 'lib/internal/loader/search.js', + 'lib/internal/safe_globals.js', 'lib/internal/net.js', 'lib/internal/module.js', 'lib/internal/os.js', @@ -177,6 +184,7 @@ 'src/fs_event_wrap.cc', 'src/handle_wrap.cc', 'src/js_stream.cc', + 'src/module_wrap.cc', 'src/node.cc', 'src/node_api.cc', 'src/node_api.h', @@ -230,6 +238,7 @@ 'src/env-inl.h', 'src/handle_wrap.h', 'src/js_stream.h', + 'src/module_wrap.h', 'src/node.h', 'src/node_http2_core.h', 'src/node_http2_core-inl.h', diff --git a/src/module_wrap.cc b/src/module_wrap.cc new file mode 100644 index 0000000000..05bbe04ef2 --- /dev/null +++ b/src/module_wrap.cc @@ -0,0 +1,531 @@ +#include +#include // PATH_MAX +#include // S_IFDIR +#include "module_wrap.h" + +#include "env.h" +#include "node_url.h" +#include "util.h" +#include "util-inl.h" + +namespace node { +namespace loader { + +using node::url::URL; +using node::url::URL_FLAGS_FAILED; +using v8::Context; +using v8::EscapableHandleScope; +using v8::Function; +using v8::FunctionCallbackInfo; +using v8::FunctionTemplate; +using v8::Integer; +using v8::IntegrityLevel; +using v8::Isolate; +using v8::JSON; +using v8::Local; +using v8::MaybeLocal; +using v8::Module; +using v8::Object; +using v8::Persistent; +using v8::Promise; +using v8::ScriptCompiler; +using v8::ScriptOrigin; +using v8::String; +using v8::Value; + +static const char* EXTENSIONS[] = {".mjs", ".js", ".json", ".node"}; +std::map*> ModuleWrap::module_map_; + +ModuleWrap::ModuleWrap(Environment* env, + Local object, + Local module, + Local url) : BaseObject(env, object) { + Isolate* iso = Isolate::GetCurrent(); + module_.Reset(iso, module); + url_.Reset(iso, url); +} + +ModuleWrap::~ModuleWrap() { + Local module = module_.Get(Isolate::GetCurrent()); + std::vector* same_hash = module_map_[module->GetIdentityHash()]; + auto it = std::find(same_hash->begin(), same_hash->end(), this); + + if (it != same_hash->end()) { + same_hash->erase(it); + } + + module_.Reset(); +} + +void ModuleWrap::New(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + + Isolate* iso = args.GetIsolate(); + + if (!args.IsConstructCall()) { + env->ThrowError("constructor must be called using new"); + return; + } + + if (args.Length() != 2) { + env->ThrowError("constructor must have exactly 2 arguments " + "(string, string)"); + return; + } + + if (!args[0]->IsString()) { + env->ThrowError("first argument is not a string"); + return; + } + + auto source_text = args[0].As(); + + if (!args[1]->IsString()) { + env->ThrowError("second argument is not a string"); + return; + } + + Local url = args[1].As(); + + Local mod; + + // compile + { + ScriptOrigin origin(url, + Integer::New(iso, 0), + Integer::New(iso, 0), + False(iso), + Integer::New(iso, 0), + FIXED_ONE_BYTE_STRING(iso, ""), + False(iso), + False(iso), + True(iso)); + ScriptCompiler::Source source(source_text, origin); + auto maybe_mod = ScriptCompiler::CompileModule(iso, &source); + if (maybe_mod.IsEmpty()) { + return; + } + mod = maybe_mod.ToLocalChecked(); + } + + auto that = args.This(); + auto ctx = that->CreationContext(); + auto url_str = FIXED_ONE_BYTE_STRING(iso, "url"); + + if (!that->Set(ctx, url_str, url).FromMaybe(false)) { + return; + } + + ModuleWrap* obj = + new ModuleWrap(Environment::GetCurrent(ctx), that, mod, url); + + if (ModuleWrap::module_map_.count(mod->GetIdentityHash()) == 0) { + ModuleWrap::module_map_[mod->GetIdentityHash()] = + new std::vector(); + } + + ModuleWrap::module_map_[mod->GetIdentityHash()]->push_back(obj); + Wrap(that, obj); + + that->SetIntegrityLevel(ctx, IntegrityLevel::kFrozen); + args.GetReturnValue().Set(that); +} + +void ModuleWrap::Link(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + Isolate* iso = args.GetIsolate(); + EscapableHandleScope handle_scope(iso); + if (!args[0]->IsFunction()) { + env->ThrowError("first argument is not a function"); + return; + } + + Local resolver_arg = args[0].As(); + + auto that = args.This(); + ModuleWrap* obj = Unwrap(that); + auto mod_context = that->CreationContext(); + if (obj->linked_) return; + obj->linked_ = true; + Local mod(obj->module_.Get(iso)); + + // call the dependency resolve callbacks + for (int i = 0; i < mod->GetModuleRequestsLength(); i++) { + Local specifier = mod->GetModuleRequest(i); + Utf8Value specifier_utf(env->isolate(), specifier); + std::string specifier_std(*specifier_utf, specifier_utf.length()); + + Local argv[] = { + specifier + }; + + MaybeLocal maybe_resolve_return_value = + resolver_arg->Call(mod_context, that, 1, argv); + if (maybe_resolve_return_value.IsEmpty()) { + return; + } + Local resolve_return_value = + maybe_resolve_return_value.ToLocalChecked(); + if (!resolve_return_value->IsPromise()) { + env->ThrowError("linking error, expected resolver to return a promise"); + } + Local resolve_promise = resolve_return_value.As(); + obj->resolve_cache_[specifier_std] = new Persistent(); + obj->resolve_cache_[specifier_std]->Reset(iso, resolve_promise); + } + + args.GetReturnValue().Set(handle_scope.Escape(that)); +} + +void ModuleWrap::Instantiate(const FunctionCallbackInfo& args) { + auto iso = args.GetIsolate(); + auto that = args.This(); + auto ctx = that->CreationContext(); + + ModuleWrap* obj = Unwrap(that); + Local mod = obj->module_.Get(iso); + bool ok = mod->Instantiate(ctx, ModuleWrap::ResolveCallback); + + // clear resolve cache on instantiate + obj->resolve_cache_.clear(); + + if (!ok) { + return; + } +} + +void ModuleWrap::Evaluate(const FunctionCallbackInfo& args) { + auto iso = args.GetIsolate(); + auto that = args.This(); + auto ctx = that->CreationContext(); + ModuleWrap* obj = Unwrap(that); + auto result = obj->module_.Get(iso)->Evaluate(ctx); + + if (result.IsEmpty()) { + return; + } + + auto ret = result.ToLocalChecked(); + args.GetReturnValue().Set(ret); +} + +MaybeLocal ModuleWrap::ResolveCallback(Local context, + Local specifier, + Local referrer) { + Environment* env = Environment::GetCurrent(context); + Isolate* iso = Isolate::GetCurrent(); + if (ModuleWrap::module_map_.count(referrer->GetIdentityHash()) == 0) { + env->ThrowError("linking error, unknown module"); + return MaybeLocal(); + } + + std::vector* possible_deps = + ModuleWrap::module_map_[referrer->GetIdentityHash()]; + ModuleWrap* dependent = nullptr; + + for (auto possible_dep : *possible_deps) { + if (possible_dep->module_ == referrer) { + dependent = possible_dep; + } + } + + if (dependent == nullptr) { + env->ThrowError("linking error, null dep"); + return MaybeLocal(); + } + + Utf8Value specifier_utf(env->isolate(), specifier); + std::string specifier_std(*specifier_utf, specifier_utf.length()); + + if (dependent->resolve_cache_.count(specifier_std) != 1) { + env->ThrowError("linking error, not in local cache"); + return MaybeLocal(); + } + + Local resolve_promise = + dependent->resolve_cache_[specifier_std]->Get(iso); + + if (resolve_promise->State() != Promise::kFulfilled) { + env->ThrowError("linking error, dependency promises must be resolved on " + "instantiate"); + return MaybeLocal(); + } + + auto module_object = resolve_promise->Result().As(); + if (module_object.IsEmpty() || !module_object->IsObject()) { + env->ThrowError("linking error, expected a valid module object from " + "resolver"); + return MaybeLocal(); + } + + ModuleWrap* mod; + ASSIGN_OR_RETURN_UNWRAP(&mod, module_object, MaybeLocal()); + return mod->module_.Get(env->isolate()); +} + +namespace { + +URL __init_cwd() { + std::string specifier = "file://"; +#ifdef _WIN32 + // MAX_PATH is in characters, not bytes. Make sure we have enough headroom. + char buf[MAX_PATH * 4]; +#else + char buf[PATH_MAX]; +#endif + + size_t cwd_len = sizeof(buf); + int err = uv_cwd(buf, &cwd_len); + if (err) { + return URL(""); + } + specifier += buf; + specifier += "/"; + return URL(specifier); +} +static URL INITIAL_CWD(__init_cwd()); +inline bool is_relative_or_absolute_path(std::string specifier) { + auto len = specifier.length(); + if (len <= 0) { + return false; + } else if (specifier[0] == '/') { + return true; + } else if (specifier[0] == '.') { + if (len == 1 || specifier[1] == '/') { + return true; + } else if (specifier[1] == '.') { + if (len == 2 || specifier[2] == '/') { + return true; + } + } + } + return false; +} +struct read_result { + bool had_error = false; + std::string source; +} read_result; +inline const struct read_result read_file(uv_file file) { + struct read_result ret; + std::string src; + uv_fs_t req; + void* base = malloc(4096); + if (base == nullptr) { + ret.had_error = true; + return ret; + } + uv_buf_t buf = uv_buf_init(static_cast(base), 4096); + uv_fs_read(uv_default_loop(), &req, file, &buf, 1, 0, nullptr); + while (req.result > 0) { + src += std::string(static_cast(buf.base), req.result); + uv_fs_read(uv_default_loop(), &req, file, &buf, 1, src.length(), nullptr); + } + ret.source = src; + return ret; +} +struct file_check { + bool failed = true; + uv_file file; +} file_check; +inline const struct file_check check_file(URL search, + bool close = false, + bool allow_dir = false) { + struct file_check ret; + uv_fs_t fs_req; + std::string path = search.ToFilePath(); + if (path.empty()) { + return ret; + } + uv_fs_open(nullptr, &fs_req, path.c_str(), O_RDONLY, 0, nullptr); + auto fd = fs_req.result; + if (fd < 0) { + return ret; + } + if (!allow_dir) { + uv_fs_fstat(nullptr, &fs_req, fd, nullptr); + if (fs_req.statbuf.st_mode & S_IFDIR) { + uv_fs_close(nullptr, &fs_req, fd, nullptr); + return ret; + } + } + ret.failed = false; + ret.file = fd; + if (close) uv_fs_close(nullptr, &fs_req, fd, nullptr); + return ret; +} +URL resolve_extensions(URL search, bool check_exact = true) { + if (check_exact) { + auto check = check_file(search, true); + if (!check.failed) { + return search; + } + } + for (auto extension : EXTENSIONS) { + URL guess(search.path() + extension, &search); + auto check = check_file(guess, true); + if (!check.failed) { + return guess; + } + } + return URL(""); +} +inline URL resolve_index(URL search) { + return resolve_extensions(URL("index", &search), false); +} +URL resolve_main(URL search) { + URL pkg("package.json", &search); + auto check = check_file(pkg); + if (!check.failed) { + auto iso = Isolate::GetCurrent(); + auto ctx = iso->GetCurrentContext(); + auto read = read_file(check.file); + uv_fs_t fs_req; + // if we fail to close :-/ + uv_fs_close(nullptr, &fs_req, check.file, nullptr); + if (read.had_error) return URL(""); + std::string pkg_src = read.source; + Local src = + String::NewFromUtf8(iso, pkg_src.c_str(), + String::kNormalString, pkg_src.length()); + if (src.IsEmpty()) return URL(""); + auto maybe_pkg_json = JSON::Parse(ctx, src); + if (maybe_pkg_json.IsEmpty()) return URL(""); + auto pkg_json_obj = maybe_pkg_json.ToLocalChecked().As(); + if (!pkg_json_obj->IsObject()) return URL(""); + auto maybe_pkg_main = pkg_json_obj->Get( + ctx, FIXED_ONE_BYTE_STRING(iso, "main")); + if (maybe_pkg_main.IsEmpty()) return URL(""); + auto pkg_main_str = maybe_pkg_main.ToLocalChecked().As(); + if (!pkg_main_str->IsString()) return URL(""); + Utf8Value main_utf8(iso, pkg_main_str); + std::string main_std(*main_utf8, main_utf8.length()); + if (!is_relative_or_absolute_path(main_std)) { + main_std.insert(0, "./"); + } + return Resolve(main_std, &search); + } + return URL(""); +} +URL resolve_module(std::string specifier, URL* base) { + URL parent(".", base); + URL dir(""); + do { + dir = parent; + auto check = Resolve("./node_modules/" + specifier, &dir, true); + if (!(check.flags() & URL_FLAGS_FAILED)) { + const auto limit = specifier.find('/'); + const auto spec_len = limit == std::string::npos ? + specifier.length() : + limit + 1; + std::string chroot = + dir.path() + "node_modules/" + specifier.substr(0, spec_len); + if (check.path().substr(0, chroot.length()) != chroot) { + return URL(""); + } + return check; + } else { + // TODO(bmeck) PREVENT FALLTHROUGH + } + parent = URL("..", &dir); + } while (parent.path() != dir.path()); + return URL(""); +} + +URL resolve_directory(URL search, bool read_pkg_json) { + if (read_pkg_json) { + auto main = resolve_main(search); + if (!(main.flags() & URL_FLAGS_FAILED)) return main; + } + return resolve_index(search); +} + +} // anonymous namespace + + +URL Resolve(std::string specifier, URL* base, bool read_pkg_json) { + URL pure_url(specifier); + if (!(pure_url.flags() & URL_FLAGS_FAILED)) { + return pure_url; + } + if (specifier.length() == 0) { + return URL(""); + } + if (is_relative_or_absolute_path(specifier)) { + URL resolved(specifier, base); + auto file = resolve_extensions(resolved); + if (!(file.flags() & URL_FLAGS_FAILED)) return file; + if (specifier.back() != '/') { + resolved = URL(specifier + "/", base); + } + return resolve_directory(resolved, read_pkg_json); + } else { + return resolve_module(specifier, base); + } + return URL(""); +} + +void ModuleWrap::Resolve(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + + if (args.IsConstructCall()) { + env->ThrowError("resolve() must not be called as a constructor"); + return; + } + if (args.Length() != 2) { + env->ThrowError("resolve must have exactly 2 arguments (string, string)"); + return; + } + + if (!args[0]->IsString()) { + env->ThrowError("first argument is not a string"); + return; + } + Utf8Value specifier_utf(env->isolate(), args[0]); + + if (!args[1]->IsString()) { + env->ThrowError("second argument is not a string"); + return; + } + Utf8Value url_utf(env->isolate(), args[1]); + URL url(*url_utf, url_utf.length()); + + if (url.flags() & URL_FLAGS_FAILED) { + env->ThrowError("second argument is not a URL string"); + return; + } + + URL result = node::loader::Resolve(*specifier_utf, &url, true); + if (result.flags() & URL_FLAGS_FAILED) { + std::string msg = "module "; + msg += *specifier_utf; + msg += " not found"; + env->ThrowError(msg.c_str()); + return; + } + + args.GetReturnValue().Set(result.ToObject(env)); +} + +void ModuleWrap::Initialize(Local target, + Local unused, + Local context) { + Environment* env = Environment::GetCurrent(context); + Isolate* isolate = env->isolate(); + + Local tpl = env->NewFunctionTemplate(New); + tpl->SetClassName(FIXED_ONE_BYTE_STRING(isolate, "ModuleWrap")); + tpl->InstanceTemplate()->SetInternalFieldCount(1); + + env->SetProtoMethod(tpl, "link", Link); + env->SetProtoMethod(tpl, "instantiate", Instantiate); + env->SetProtoMethod(tpl, "evaluate", Evaluate); + + target->Set(FIXED_ONE_BYTE_STRING(isolate, "ModuleWrap"), tpl->GetFunction()); + env->SetMethod(target, "resolve", node::loader::ModuleWrap::Resolve); +} + +} // namespace loader +} // namespace node + +NODE_MODULE_CONTEXT_AWARE_BUILTIN(module_wrap, + node::loader::ModuleWrap::Initialize) diff --git a/src/module_wrap.h b/src/module_wrap.h new file mode 100644 index 0000000000..c669834c6f --- /dev/null +++ b/src/module_wrap.h @@ -0,0 +1,58 @@ +#ifndef SRC_MODULE_WRAP_H_ +#define SRC_MODULE_WRAP_H_ + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#include +#include +#include +#include "node_url.h" +#include "base-object.h" +#include "base-object-inl.h" + +namespace node { +namespace loader { + +node::url::URL Resolve(std::string specifier, node::url::URL* base, + bool read_pkg_json = false); + +class ModuleWrap : public BaseObject { + public: + static const std::string EXTENSIONS[]; + static void Initialize(v8::Local target, + v8::Local unused, + v8::Local context); + + private: + ModuleWrap(node::Environment* env, + v8::Local object, + v8::Local module, + v8::Local url); + ~ModuleWrap(); + + static void New(const v8::FunctionCallbackInfo& args); + static void Link(const v8::FunctionCallbackInfo& args); + static void Instantiate(const v8::FunctionCallbackInfo& args); + static void Evaluate(const v8::FunctionCallbackInfo& args); + static void GetUrl(v8::Local property, + const v8::PropertyCallbackInfo& info); + static void Resolve(const v8::FunctionCallbackInfo& args); + static v8::MaybeLocal ResolveCallback( + v8::Local context, + v8::Local specifier, + v8::Local referrer); + + v8::Persistent module_; + v8::Persistent url_; + bool linked_ = false; + std::map*> resolve_cache_; + + static std::map*> module_map_; +}; + +} // namespace loader +} // namespace node + +#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#endif // SRC_MODULE_WRAP_H_ diff --git a/src/node.cc b/src/node.cc index bdff4527c6..5cd0ffc29d 100644 --- a/src/node.cc +++ b/src/node.cc @@ -225,6 +225,11 @@ bool trace_warnings = false; // that is used by lib/module.js bool config_preserve_symlinks = false; +// Set in node.cc by ParseArgs when --experimental-modules is used. +// Used in node_config.cc to set a constant on process.binding('config') +// that is used by lib/module.js +bool config_experimental_modules = false; + // Set by ParseArgs when --pending-deprecation or NODE_PENDING_DEPRECATION // is used. bool config_pending_deprecation = false; @@ -3711,6 +3716,7 @@ static void PrintHelp() { " note: linked-in ICU data is present\n" #endif " --preserve-symlinks preserve symbolic links when resolving\n" + " --experimental-modules experimental ES Module support\n" " and caching modules\n" #endif "\n" @@ -3947,6 +3953,8 @@ static void ParseArgs(int* argc, Revert(cve); } else if (strcmp(arg, "--preserve-symlinks") == 0) { config_preserve_symlinks = true; + } else if (strcmp(arg, "--experimental-modules") == 0) { + config_experimental_modules = true; } else if (strcmp(arg, "--prof-process") == 0) { prof_process = true; short_circuit = true; diff --git a/src/node_config.cc b/src/node_config.cc index 87110dd8c6..2f45a5e971 100644 --- a/src/node_config.cc +++ b/src/node_config.cc @@ -65,6 +65,9 @@ static void InitConfig(Local target, if (config_preserve_symlinks) READONLY_BOOLEAN_PROPERTY("preserveSymlinks"); + if (config_experimental_modules) + READONLY_BOOLEAN_PROPERTY("experimentalModules"); + if (config_pending_deprecation) READONLY_BOOLEAN_PROPERTY("pendingDeprecation"); diff --git a/src/node_internals.h b/src/node_internals.h index 1e099325a3..a241e671ed 100644 --- a/src/node_internals.h +++ b/src/node_internals.h @@ -86,6 +86,10 @@ extern bool config_preserve_symlinks; // Set in node.cc by ParseArgs when --expose-http2 is used. extern bool config_expose_http2; +// Set in node.cc by ParseArgs when --experimental-modules is used. +// Used in node_config.cc to set a constant on process.binding('config') +// that is used by lib/module.js +extern bool config_experimental_modules; // Set in node.cc by ParseArgs when --expose-internals or --expose_internals is // used. diff --git a/src/node_url.cc b/src/node_url.cc index dd3da1133e..f8adc7d7af 100644 --- a/src/node_url.cc +++ b/src/node_url.cc @@ -2080,6 +2080,69 @@ static void DomainToUnicode(const FunctionCallbackInfo& args) { v8::NewStringType::kNormal).ToLocalChecked()); } +std::string URL::ToFilePath() { + if (context_.scheme != "file:") { + return ""; + } + +#ifdef _WIN32 + const char* slash = "\\"; + auto is_slash = [] (char ch) { + return ch == '/' || ch == '\\'; + }; +#else + const char* slash = "/"; + auto is_slash = [] (char ch) { + return ch == '/'; + }; + if ((context_.flags & URL_FLAGS_HAS_HOST) && + context_.host.length() > 0) { + return ""; + } +#endif + std::string decoded_path; + for (std::string& part : context_.path) { + std::string decoded; + PercentDecode(part.c_str(), part.length(), &decoded); + for (char& ch : decoded) { + if (is_slash(ch)) { + return ""; + } + } + decoded_path += slash + decoded; + } + +#ifdef _WIN32 + // TODO(TimothyGu): Use "\\?\" long paths on Windows. + + // If hostname is set, then we have a UNC path. Pass the hostname through + // ToUnicode just in case it is an IDN using punycode encoding. We do not + // need to worry about percent encoding because the URL parser will have + // already taken care of that for us. Note that this only causes IDNs with an + // appropriate `xn--` prefix to be decoded. + if ((context_.flags & URL_FLAGS_HAS_HOST) && + context_.host.length() > 0) { + std::string unicode_host; + if (!ToUnicode(&context_.host, &unicode_host)) { + return ""; + } + return "\\\\" + unicode_host + decoded_path; + } + // Otherwise, it's a local path that requires a drive letter. + if (decoded_path.length() < 3) { + return ""; + } + if (decoded_path[2] != ':' || + !IsASCIIAlpha(decoded_path[1])) { + return ""; + } + // Strip out the leading '\'. + return decoded_path.substr(1); +#else + return decoded_path; +#endif +} + // This function works by calling out to a JS function that creates and // returns the JS URL object. Be mindful of the JS<->Native boundary // crossing that is required. diff --git a/src/node_url.h b/src/node_url.h index 72ac366ec1..cb7bdca7f2 100644 --- a/src/node_url.h +++ b/src/node_url.h @@ -163,6 +163,10 @@ class URL { return ret; } + // Get the path of the file: URL in a format consumable by native file system + // APIs. Returns an empty string if something went wrong. + std::string ToFilePath(); + const Local ToObject(Environment* env) const; private: diff --git a/test/cctest/test_url.cc b/test/cctest/test_url.cc index 2cede1a8a3..0b80d44caa 100644 --- a/test/cctest/test_url.cc +++ b/test/cctest/test_url.cc @@ -79,3 +79,28 @@ TEST_F(URLTest, Base3) { EXPECT_EQ(simple.host(), "example.org"); EXPECT_EQ(simple.path(), "/baz"); } + +TEST_F(URLTest, ToFilePath) { +#define T(url, path) EXPECT_EQ(path, URL(url).ToFilePath()) + T("http://example.org/foo/bar", ""); + +#ifdef _WIN32 + T("file:///C:/Program%20Files/", "C:\\Program Files\\"); + T("file:///C:/a/b/c?query#fragment", "C:\\a\\b\\c"); + T("file://host/path/a/b/c?query#fragment", "\\\\host\\path\\a\\b\\c"); + T("file://xn--weird-prdj8vva.com/host/a", "\\\\wͪ͊eiͬ͋rd.com\\host\\a"); + T("file:///C:/a%2Fb", ""); + T("file:///", ""); + T("file:///home", ""); +#else + T("file:///", "/"); + T("file:///home/user?query#fragment", "/home/user"); + T("file:///home/user/?query#fragment", "/home/user/"); + T("file:///home/user/%20space", "/home/user/ space"); + T("file:///home/us%5Cer", "/home/us\\er"); + T("file:///home/us%2Fer", ""); + T("file://host/path", ""); +#endif + +#undef T +} diff --git a/test/es-module/es-module.status b/test/es-module/es-module.status new file mode 100644 index 0000000000..343e622ca0 --- /dev/null +++ b/test/es-module/es-module.status @@ -0,0 +1,7 @@ +prefix parallel + +# To mark a test as flaky, list the test name in the appropriate section +# below, without ".js", followed by ": PASS,FLAKY". Example: +# sample-test : PASS,FLAKY + +[true] # This section applies to all platforms diff --git a/test/es-module/esm-snapshot-mutator.js b/test/es-module/esm-snapshot-mutator.js new file mode 100644 index 0000000000..a0dfa0c28a --- /dev/null +++ b/test/es-module/esm-snapshot-mutator.js @@ -0,0 +1,5 @@ +/* eslint-disable required-modules */ +'use strict'; +const shouldSnapshotFilePath = require.resolve('./esm-snapshot.js'); +require('./esm-snapshot.js'); +require.cache[shouldSnapshotFilePath].exports++; diff --git a/test/es-module/esm-snapshot.js b/test/es-module/esm-snapshot.js new file mode 100644 index 0000000000..2c3c3a459a --- /dev/null +++ b/test/es-module/esm-snapshot.js @@ -0,0 +1,3 @@ +/* eslint-disable required-modules */ +'use strict'; +module.exports = 1; diff --git a/test/es-module/test-esm-basic-imports.mjs b/test/es-module/test-esm-basic-imports.mjs new file mode 100644 index 0000000000..23989bddd5 --- /dev/null +++ b/test/es-module/test-esm-basic-imports.mjs @@ -0,0 +1,8 @@ +// Flags: --experimental-modules +import '../common'; +import assert from 'assert'; +import ok from './test-esm-ok.mjs'; +import okShebang from './test-esm-shebang.mjs'; + +assert(ok); +assert(okShebang); diff --git a/test/es-module/test-esm-encoded-path-native.js b/test/es-module/test-esm-encoded-path-native.js new file mode 100644 index 0000000000..f32297efdb --- /dev/null +++ b/test/es-module/test-esm-encoded-path-native.js @@ -0,0 +1,10 @@ +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const { spawn } = require('child_process'); + +const native = `${common.fixturesDir}/es-module-url/native.mjs`; +const child = spawn(process.execPath, ['--experimental-modules', native]); +child.on('exit', (code) => { + assert.strictEqual(code, 1); +}); diff --git a/test/es-module/test-esm-encoded-path.mjs b/test/es-module/test-esm-encoded-path.mjs new file mode 100644 index 0000000000..2c6e145927 --- /dev/null +++ b/test/es-module/test-esm-encoded-path.mjs @@ -0,0 +1,7 @@ +// Flags: --experimental-modules +import '../common'; +import assert from 'assert'; +// ./test-esm-ok.mjs +import ok from './test-%65%73%6d-ok.mjs'; + +assert(ok); diff --git a/test/es-module/test-esm-forbidden-globals.mjs b/test/es-module/test-esm-forbidden-globals.mjs new file mode 100644 index 0000000000..d3e92b9238 --- /dev/null +++ b/test/es-module/test-esm-forbidden-globals.mjs @@ -0,0 +1,24 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +if (typeof arguments !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof this !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof exports !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof require !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof module !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof __filename !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof __dirname !== 'undefined') { + throw new Error('not an ESM'); +} diff --git a/test/es-module/test-esm-namespace.mjs b/test/es-module/test-esm-namespace.mjs new file mode 100644 index 0000000000..72b7fed4b3 --- /dev/null +++ b/test/es-module/test-esm-namespace.mjs @@ -0,0 +1,7 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +import * as fs from 'fs'; +import assert from 'assert'; + +assert.deepStrictEqual(Object.keys(fs), ['default']); diff --git a/test/es-module/test-esm-ok.mjs b/test/es-module/test-esm-ok.mjs new file mode 100644 index 0000000000..6712e1ab7d --- /dev/null +++ b/test/es-module/test-esm-ok.mjs @@ -0,0 +1,5 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +const isJs = true; +export default isJs; diff --git a/test/es-module/test-esm-pkg-over-ext.mjs b/test/es-module/test-esm-pkg-over-ext.mjs new file mode 100644 index 0000000000..7e47c4c326 --- /dev/null +++ b/test/es-module/test-esm-pkg-over-ext.mjs @@ -0,0 +1,8 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +import resolved from '../fixtures/module-pkg-over-ext/inner'; +import expected from '../fixtures/module-pkg-over-ext/inner/package.json'; +import assert from 'assert'; + +assert.strictEqual(resolved, expected); diff --git a/test/es-module/test-esm-preserve-symlinks.js b/test/es-module/test-esm-preserve-symlinks.js new file mode 100644 index 0000000000..eea5bf061b --- /dev/null +++ b/test/es-module/test-esm-preserve-symlinks.js @@ -0,0 +1,38 @@ +// Flags: --experimental-modules +'use strict'; + +const common = require('../common'); +const { spawn } = require('child_process'); +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); + +common.refreshTmpDir(); +const tmpDir = common.tmpDir; + +const entry = path.join(tmpDir, 'entry.js'); +const real = path.join(tmpDir, 'real.js'); +const link_absolute_path = path.join(tmpDir, 'link.js'); + +fs.writeFileSync(entry, ` +const assert = require('assert'); +global.x = 0; +require('./real.js'); +assert.strictEqual(x, 1); +require('./link.js'); +assert.strictEqual(x, 2); +`); +fs.writeFileSync(real, 'x++;'); + +try { + fs.symlinkSync(real, link_absolute_path); +} catch (err) { + if (err.code !== 'EPERM') throw err; + common.skip('insufficient privileges for symlinks'); +} + +spawn(process.execPath, + ['--experimental-modules', '--preserve-symlinks', entry], + { stdio: 'inherit' }).on('exit', (code) => { + assert.strictEqual(code, 0); +}); diff --git a/test/es-module/test-esm-require-cache.mjs b/test/es-module/test-esm-require-cache.mjs new file mode 100644 index 0000000000..ff32cde36f --- /dev/null +++ b/test/es-module/test-esm-require-cache.mjs @@ -0,0 +1,7 @@ +// Flags: --experimental-modules +import '../common'; +import '../fixtures/es-module-require-cache/preload.js'; +import '../fixtures/es-module-require-cache/counter.js'; +import assert from 'assert'; +assert.strictEqual(global.counter, 1); +delete global.counter; diff --git a/test/es-module/test-esm-shebang.mjs b/test/es-module/test-esm-shebang.mjs new file mode 100644 index 0000000000..43cc0f8367 --- /dev/null +++ b/test/es-module/test-esm-shebang.mjs @@ -0,0 +1,6 @@ +#! }]) // isn't js +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +const isJs = true; +export default isJs; diff --git a/test/es-module/test-esm-snapshot.mjs b/test/es-module/test-esm-snapshot.mjs new file mode 100644 index 0000000000..89034f5668 --- /dev/null +++ b/test/es-module/test-esm-snapshot.mjs @@ -0,0 +1,7 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ +import './esm-snapshot-mutator'; +import one from './esm-snapshot'; +import assert from 'assert'; + +assert.strictEqual(one, 1); diff --git a/test/es-module/test-esm-symlink.js b/test/es-module/test-esm-symlink.js new file mode 100644 index 0000000000..3b7d689bf8 --- /dev/null +++ b/test/es-module/test-esm-symlink.js @@ -0,0 +1,48 @@ +'use strict'; + +const common = require('../common'); +const { spawn } = require('child_process'); +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); + +common.refreshTmpDir(); +const tmpDir = common.tmpDir; + +const entry = path.join(tmpDir, 'entry.mjs'); +const real = path.join(tmpDir, 'index.mjs'); +const link_absolute_path = path.join(tmpDir, 'absolute'); +const link_relative_path = path.join(tmpDir, 'relative'); +const link_ignore_extension = path.join(tmpDir, + 'ignore_extension.json'); +const link_directory = path.join(tmpDir, 'directory'); + +fs.writeFileSync(real, 'export default [];'); +fs.writeFileSync(entry, ` +import assert from 'assert'; +import real from './index.mjs'; +import absolute from './absolute'; +import relative from './relative'; +import ignoreExtension from './ignore_extension.json'; +import directory from './directory'; + +assert.strictEqual(absolute, real); +assert.strictEqual(relative, real); +assert.strictEqual(ignoreExtension, real); +assert.strictEqual(directory, real); +`); + +try { + fs.symlinkSync(real, link_absolute_path); + fs.symlinkSync(path.basename(real), link_relative_path); + fs.symlinkSync(real, link_ignore_extension); + fs.symlinkSync(path.dirname(real), link_directory); +} catch (err) { + if (err.code !== 'EPERM') throw err; + common.skip('insufficient privileges for symlinks'); +} + +spawn(process.execPath, ['--experimental-modules', entry], + { stdio: 'inherit' }).on('exit', (code) => { + assert.strictEqual(code, 0); +}); diff --git a/test/es-module/testcfg.py b/test/es-module/testcfg.py new file mode 100644 index 0000000000..0d8dfeed46 --- /dev/null +++ b/test/es-module/testcfg.py @@ -0,0 +1,6 @@ +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy + +def GetConfiguration(context, root): + return testpy.SimpleTestConfiguration(context, root, 'es-module') diff --git a/test/fixtures/es-module-require-cache/counter.js b/test/fixtures/es-module-require-cache/counter.js new file mode 100644 index 0000000000..2640d3e372 --- /dev/null +++ b/test/fixtures/es-module-require-cache/counter.js @@ -0,0 +1,2 @@ +global.counter = global.counter || 0; +global.counter++; \ No newline at end of file diff --git a/test/fixtures/es-module-require-cache/preload.js b/test/fixtures/es-module-require-cache/preload.js new file mode 100644 index 0000000000..6090dc0d58 --- /dev/null +++ b/test/fixtures/es-module-require-cache/preload.js @@ -0,0 +1 @@ +require('./counter'); \ No newline at end of file diff --git a/test/fixtures/es-module-url/empty.js b/test/fixtures/es-module-url/empty.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/fixtures/es-module-url/native.mjs b/test/fixtures/es-module-url/native.mjs new file mode 100644 index 0000000000..c8831f9bfe --- /dev/null +++ b/test/fixtures/es-module-url/native.mjs @@ -0,0 +1,2 @@ +// path +import 'p%61th'; diff --git a/test/testpy/__init__.py b/test/testpy/__init__.py index 37e5ac710b..f113c1253a 100644 --- a/test/testpy/__init__.py +++ b/test/testpy/__init__.py @@ -27,7 +27,7 @@ import test import os -from os.path import join, dirname, exists +from os.path import join, dirname, exists, splitext import re import ast @@ -109,18 +109,17 @@ class SimpleTestConfiguration(test.TestConfiguration): self.additional_flags = [] def Ls(self, path): - def SelectTest(name): - return name.startswith('test-') and name.endswith('.js') - return [f[:-3] for f in os.listdir(path) if SelectTest(f)] + return [f for f in os.listdir(path) if re.match('^test-.*\.m?js$', f)] def ListTests(self, current_path, path, arch, mode): all_tests = [current_path + [t] for t in self.Ls(join(self.root))] result = [] for test in all_tests: if self.Contains(path, test): - file_path = join(self.root, reduce(join, test[1:], "") + ".js") - result.append(SimpleTestCase(test, file_path, arch, mode, self.context, - self, self.additional_flags)) + file_path = join(self.root, reduce(join, test[1:], "")) + test_name = test[:-1] + [splitext(test[-1])[0]] + result.append(SimpleTestCase(test_name, file_path, arch, mode, + self.context, self, self.additional_flags)) return result def GetBuildRequirements(self): diff --git a/tools/eslint-rules/required-modules.js b/tools/eslint-rules/required-modules.js index 47ade5cd9f..948c46c036 100644 --- a/tools/eslint-rules/required-modules.js +++ b/tools/eslint-rules/required-modules.js @@ -13,6 +13,7 @@ const path = require('path'); module.exports = function(context) { // trim required module names var requiredModules = context.options; + const isESM = context.parserOptions.sourceType === 'module'; const foundModules = []; @@ -39,39 +40,35 @@ module.exports = function(context) { return node.callee.type === 'Identifier' && node.callee.name === 'require'; } + /** + * Function to check if the path is a required module and return its name. + * @param {String} str The path to check + * @returns {undefined|String} required module name or undefined + */ + function getRequiredModuleName(str) { + var value = path.basename(str); + + // check if value is in required modules array + return requiredModules.indexOf(value) !== -1 ? value : undefined; + } + /** * Function to check if a node has an argument that is a required module and * return its name. * @param {ASTNode} node The node to check * @returns {undefined|String} required module name or undefined */ - function getRequiredModuleName(node) { - var moduleName; - + function getRequiredModuleNameFromCall(node) { // node has arguments and first argument is string if (node.arguments.length && isString(node.arguments[0])) { - var argValue = path.basename(node.arguments[0].value.trim()); - - // check if value is in required modules array - if (requiredModules.indexOf(argValue) !== -1) { - moduleName = argValue; - } + return getRequiredModuleName(node.arguments[0].value.trim()); } - return moduleName; + return undefined; } - return { - 'CallExpression': function(node) { - if (isRequireCall(node)) { - var requiredModuleName = getRequiredModuleName(node); - - if (requiredModuleName) { - foundModules.push(requiredModuleName); - } - } - }, - 'Program:exit': function(node) { + const rules = { + 'Program:exit'(node) { if (foundModules.length < requiredModules.length) { var missingModules = requiredModules.filter( function(module) { @@ -88,6 +85,27 @@ module.exports = function(context) { } } }; + + if (isESM) { + rules.ImportDeclaration = (node) => { + var requiredModuleName = getRequiredModuleName(node.source.value); + if (requiredModuleName) { + foundModules.push(requiredModuleName); + } + }; + } else { + rules.CallExpression = (node) => { + if (isRequireCall(node)) { + var requiredModuleName = getRequiredModuleNameFromCall(node); + + if (requiredModuleName) { + foundModules.push(requiredModuleName); + } + } + }; + } + + return rules; }; module.exports.schema = { diff --git a/tools/test.py b/tools/test.py index 5a50c7f2e6..6839f4e1b2 100755 --- a/tools/test.py +++ b/tools/test.py @@ -279,9 +279,7 @@ class TapProgressIndicator(SimpleProgressIndicator): # hard to decipher what test is running when only the filename is printed. prefix = abspath(join(dirname(__file__), '../test')) + os.sep command = output.command[-1] - if command.endswith('.js'): command = command[:-3] - if command.startswith(prefix): command = command[len(prefix):] - command = command.replace('\\', '/') + command = NormalizePath(command, prefix) if output.UnexpectedOutput(): status_line = 'not ok %i %s' % (self._done, command) @@ -352,9 +350,7 @@ class DeoptsCheckProgressIndicator(SimpleProgressIndicator): # hard to decipher what test is running when only the filename is printed. prefix = abspath(join(dirname(__file__), '../test')) + os.sep command = output.command[-1] - if command.endswith('.js'): command = command[:-3] - if command.startswith(prefix): command = command[len(prefix):] - command = command.replace('\\', '/') + command = NormalizePath(command, prefix) stdout = output.output.stdout.strip() printed_file = False @@ -1509,12 +1505,16 @@ def SplitPath(s): stripped = [ c.strip() for c in s.split('/') ] return [ Pattern(s) for s in stripped if len(s) > 0 ] -def NormalizePath(path): +def NormalizePath(path, prefix='test/'): # strip the extra path information of the specified test - if path.startswith('test/'): - path = path[5:] + prefix = prefix.replace('\\', '/') + path = path.replace('\\', '/') + if path.startswith(prefix): + path = path[len(prefix):] if path.endswith('.js'): path = path[:-3] + elif path.endswith('.mjs'): + path = path[:-4] return path def GetSpecialCommandProcessor(value):