From 4ccdc501d48a0f198a040ee46f4dab27a611a489 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 15 Mar 2010 08:00:19 -0700 Subject: [PATCH] Include lib/ directory in node executable. Compile on demand. Instead of installing the files in /usr/lib/node/libraries and loading them from the file system, the files are built-in to the node executable. However, they are only compiled on demand. The reasoning is: 1. Allow for more complex internal javascript. In particular, process.stdout and process.stdin can be js implemented streams. 2. Ease system installs. Loading from disk each time is unnecessary overhead. Note that there is no "system" path for modules anymore. Only $HOME/.node_libraries. --- doc/api.txt | 27 ++++----------- lib/assert.js | 6 ++-- lib/fs.js | 2 +- lib/http.js | 2 +- src/node.cc | 21 +++++++++++- src/node.js | 92 +++++++++++++++++++++++++++------------------------ tools/js2c.py | 19 +++++++++-- wscript | 8 ++--- 8 files changed, 100 insertions(+), 77 deletions(-) diff --git a/doc/api.txt b/doc/api.txt index 2bdd10bcb6..def050430b 100644 --- a/doc/api.txt +++ b/doc/api.txt @@ -337,33 +337,20 @@ A module prefixed with +"./"+ is relative to the file calling +require()+. That is, +circle.js+ must be in the same directory as +foo.js+ for +require("./circle")+ to find it. -Without the leading +"./"+, like +require("mjsunit")+ the module is searched +Without the leading +"./"+, like +require("assert")+ the module is searched for in the +require.paths+ array. +require.paths+ on my system looks like this: ---------------------------------------- -[ "/home/ryan/.node_libraries" -, "/usr/local/lib/node/libraries" -] +[ "/home/ryan/.node_libraries" ] ---------------------------------------- -That is, when +require("mjsunit")+ is called Node looks for +That is, when +require("assert")+ is called Node looks for - 1. +"/home/ryan/.node_libraries/mjsunit.js"+ - - 2. +"/home/ryan/.node_libraries/mjsunit.node"+ - - 3. +"/home/ryan/.node_libraries/mjsunit/index.js"+ - - 4. +"/home/ryan/.node_libraries/mjsunit/index.node"+ - - 5. +"/usr/local/lib/node/libraries/mjsunit.js"+ - - 6. +"/usr/local/lib/node/libraries/mjsunit.node"+ - - 7. +"/usr/local/lib/node/libraries/mjsunit/index.js"+ - - 8. +"/usr/local/lib/node/libraries/mjsunit/index.node"+ + 1. +"/home/ryan/.node_libraries/assert.js"+ + 2. +"/home/ryan/.node_libraries/assert.node"+ + 3. +"/home/ryan/.node_libraries/assert/index.js"+ + 4. +"/home/ryan/.node_libraries/assert/index.node"+ interrupting once a file is found. Files ending in +".node"+ are binary Addon Modules; see the section below about addons. +"index.js"+ allows one to diff --git a/lib/assert.js b/lib/assert.js index 0a8c19715e..6dc06213cc 100644 --- a/lib/assert.js +++ b/lib/assert.js @@ -6,7 +6,7 @@ // Copyright (c) 2009 Thomas Robinson <280north.com> // // Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the “Software”), to +// of this software and associated documentation files (the 'Software'), to // deal in the Software without restriction, including without limitation the // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or // sell copies of the Software, and to permit persons to whom the Software is @@ -15,7 +15,7 @@ // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // -// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN @@ -23,7 +23,7 @@ // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // UTILITY -var inherits = require('./sys').inherits; +var inherits = require('sys').inherits; var pSlice = Array.prototype.slice; // 1. The assert module provides functions that throw diff --git a/lib/fs.js b/lib/fs.js index f9eae9bd39..63fb166501 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -1,4 +1,4 @@ -var sys = require('./sys'), +var sys = require('sys'), events = require('events'); var fs = exports; diff --git a/lib/http.js b/lib/http.js index 6b7d1f691c..624e4573df 100644 --- a/lib/http.js +++ b/lib/http.js @@ -1,4 +1,4 @@ -var sys = require('./sys'); +var sys = require('sys'); var events = require('events'); var CRLF = "\r\n"; diff --git a/src/node.cc b/src/node.cc index 8cb5e64ed8..b5736c6803 100644 --- a/src/node.cc +++ b/src/node.cc @@ -1156,6 +1156,26 @@ static void Load(int argc, char *argv[]) { HTTPConnection::Initialize(http); // http.cc + Local natives = Object::New(); + process->Set(String::New("natives"), natives); + // Explicitly define native sources. + natives->Set(String::New("assert"), String::New(native_assert)); + natives->Set(String::New("dns"), String::New(native_dns)); + natives->Set(String::New("file"), String::New(native_file)); + natives->Set(String::New("fs"), String::New(native_fs)); + natives->Set(String::New("http"), String::New(native_http)); + natives->Set(String::New("ini"), String::New(native_ini)); + natives->Set(String::New("mjsunit"), String::New(native_mjsunit)); + natives->Set(String::New("multipart"), String::New(native_multipart)); + natives->Set(String::New("posix"), String::New(native_posix)); + natives->Set(String::New("querystring"), String::New(native_querystring)); + natives->Set(String::New("repl"), String::New(native_repl)); + natives->Set(String::New("sys"), String::New(native_sys)); + natives->Set(String::New("tcp"), String::New(native_tcp)); + natives->Set(String::New("uri"), String::New(native_uri)); + natives->Set(String::New("url"), String::New(native_url)); + natives->Set(String::New("utils"), String::New(native_utils)); + // Compile, execute the src/node.js file. (Which was included as static C // string in node_natives.h. 'natve_node' is the string containing that @@ -1262,7 +1282,6 @@ static void ParseArgs(int *argc, char **argv) { exit(0); } else if (strcmp(arg, "--vars") == 0) { printf("NODE_PREFIX: %s\n", NODE_PREFIX); - printf("NODE_LIBRARIES_PREFIX: %s/%s\n", NODE_PREFIX, "lib/node/libraries"); printf("NODE_CFLAGS: %s\n", NODE_CFLAGS); exit(0); } else if (strcmp(arg, "--help") == 0 || strcmp(arg, "-h") == 0) { diff --git a/src/node.js b/src/node.js index 0d08a70009..858cd91a94 100644 --- a/src/node.js +++ b/src/node.js @@ -461,8 +461,7 @@ function existsSync (path) { -process.paths = [ path.join(process.installPrefix, "lib/node/libraries") - ]; +process.paths = []; if (process.env["HOME"]) { process.paths.unshift(path.join(process.env["HOME"], ".node_libraries")); @@ -553,6 +552,8 @@ function resolveModulePath(request, parent) { var id, paths; if (request.charAt(0) == "." && (request.charAt(1) == "/" || request.charAt(1) == ".")) { // Relative request + debug("RELATIVE: requested:" + request + " set ID to: "+id+" from "+parent.id); + var exts = ['js', 'node'], ext; for (ext in extensionCache) { exts.push(ext.slice(1)); @@ -561,7 +562,6 @@ function resolveModulePath(request, parent) { var parentIdPath = path.dirname(parent.id + (path.basename(parent.filename).match(new RegExp('^index\\.(' + exts.join('|') + ')$')) ? "/" : "")); id = path.join(parentIdPath, request); - // debug("RELATIVE: requested:"+request+" set ID to: "+id+" from "+parent.id+"("+parentIdPath+")"); paths = [path.dirname(parent.filename)]; } else { id = request; @@ -573,56 +573,61 @@ function resolveModulePath(request, parent) { } -function loadModuleSync (request, parent) { - var resolvedModule = resolveModulePath(request, parent); - var id = resolvedModule[0]; - var paths = resolvedModule[1]; +function loadModule (request, parent, callback) { + var resolvedModule = resolveModulePath(request, parent), + id = resolvedModule[0], + paths = resolvedModule[1]; - debug("loadModuleSync REQUEST " + (request) + " parent: " + parent.id); + debug("loadModule REQUEST " + (request) + " parent: " + parent.id); var cachedModule = internalModuleCache[id] || parent.moduleCache[id]; - if (cachedModule) { - debug("found " + JSON.stringify(id) + " in cache"); - return cachedModule.exports; - } else { - debug("looking for " + JSON.stringify(id) + " in " + JSON.stringify(paths)); - var filename = findModulePath(request, paths); - if (!filename) { - throw new Error("Cannot find module '" + request + "'"); - } else { - var module = new Module(id, parent); - module.loadSync(filename); - return module.exports; + if (!cachedModule) { + // Try to compile from native modules + if (process.natives[id]) { + debug('load native module ' + id); + cachedModule = new Module(id); + var e = cachedModule._compile(process.natives[id], id); + if (e) throw e; + internalModuleCache[id] = cachedModule; } } -} - - -function loadModule (request, parent, callback) { - var - resolvedModule = resolveModulePath(request, parent), - id = resolvedModule[0], - paths = resolvedModule[1]; - debug("loadModule REQUEST " + (request) + " parent: " + parent.id); - - var cachedModule = internalModuleCache[id] || parent.moduleCache[id]; if (cachedModule) { debug("found " + JSON.stringify(id) + " in cache"); - if (callback) callback(null, cachedModule.exports); - } else { - debug("looking for " + JSON.stringify(id) + " in " + JSON.stringify(paths)); + if (callback) { + callback(null, cachedModule.exports); + } else { + return cachedModule.exports; + } + + } else { // Not in cache - findModulePath(request, paths, function (filename) { + debug("looking for " + JSON.stringify(id) + " in " + JSON.stringify(paths)); + + if (!callback) { + // sync + var filename = findModulePath(request, paths); if (!filename) { - var err = new Error("Cannot find module '" + request + "'"); - if (callback) callback(err); + throw new Error("Cannot find module '" + request + "'"); } else { var module = new Module(id, parent); - module.load(filename, callback); + module.loadSync(filename); + return module.exports; } - }); + + } else { + // async + findModulePath(request, paths, function (filename) { + if (!filename) { + var err = new Error("Cannot find module '" + request + "'"); + callback(err); + } else { + var module = new Module(id, parent); + module.load(filename, callback); + } + }); + } } }; @@ -713,7 +718,8 @@ function cat (id, callback) { } -Module.prototype._loadContent = function (content, filename) { +// Returns exception if any +Module.prototype._compile = function (content, filename) { var self = this; // remove shebang content = content.replace(/^\#\!.*/, ''); @@ -729,7 +735,7 @@ Module.prototype._loadContent = function (content, filename) { } function require (path) { - return loadModuleSync(path, self); + return loadModule(path, self); } require.paths = process.paths; @@ -765,7 +771,7 @@ Module.prototype._loadScriptSync = function (filename) { // remove shebang content = content.replace(/^\#\!.*/, ''); - var e = this._loadContent(content, filename); + var e = this._compile(content, filename); if (e) { throw e; } else { @@ -781,7 +787,7 @@ Module.prototype._loadScript = function (filename, callback) { if (err) { if (callback) callback(err); } else { - var e = self._loadContent(content, filename); + var e = self._compile(content, filename); if (e) { if (callback) callback(e); } else { diff --git a/tools/js2c.py b/tools/js2c.py index fb38ece99b..64e5c2b625 100755 --- a/tools/js2c.py +++ b/tools/js2c.py @@ -35,11 +35,22 @@ import os, re, sys, string import jsmin -def ToCArray(lines): +def ToCArray(filename, lines): result = [] + row = 1 + col = 0 for chr in lines: + col += 1 + if chr == "\n" or chr == "\r": + row += 1 + col = 0 + value = ord(chr) - assert value < 128 + + if value > 128: + print 'non-ascii value ' + filename + ':' + str(row) + ':' + str(col) + sys.exit(1); + result.append(str(value)) result.append("0") return ", ".join(result) @@ -231,6 +242,7 @@ def JS2C(source, target): # Locate the macros file name. consts = {} macros = {} + for s in source: if 'macros.py' == (os.path.split(str(s))[1]): (consts, macros) = ReadMacros(ReadLines(str(s))) @@ -244,10 +256,11 @@ def JS2C(source, target): delay = str(s).endswith('-delay.js') lines = ReadFile(str(s)) do_jsmin = lines.find('// jsminify this file, js2c: jsmin') != -1 + lines = ExpandConstants(lines, consts) lines = ExpandMacros(lines, macros) lines = CompressScript(lines, do_jsmin) - data = ToCArray(lines) + data = ToCArray(s, lines) id = (os.path.split(str(s))[1])[:-3] if delay: id = id[:-6] if delay: diff --git a/wscript b/wscript index 6310451ec9..6a8392b512 100644 --- a/wscript +++ b/wscript @@ -344,11 +344,11 @@ def build(bld): js2c.JS2C(source, targets) native_cc = bld.new_task_gen( - source='src/node.js', + source='src/node.js ' + bld.path.ant_glob('lib/*.js'), target="src/node_natives.h", - before="cxx" + before="cxx", + install_path=None ) - native_cc.install_path = None # Add the rule /after/ cloning the debug # This is a work around for an error had in python 2.4.3 (I'll paste the @@ -458,8 +458,6 @@ def build(bld): bld.install_files('${PREFIX}/lib/node/wafadmin', 'tools/wafadmin/*.py') bld.install_files('${PREFIX}/lib/node/wafadmin/Tools', 'tools/wafadmin/Tools/*.py') - bld.install_files('${PREFIX}/lib/node/libraries/', 'lib/*.js') - def shutdown(): Options.options.debug # HACK to get binding.node out of build directory.