Include lib/ directory in node executable. Compile on demand.

Instead of installing the files in /usr/lib/node/libraries and loading them
from the file system, the files are built-in to the node executable.
However, they are only compiled on demand.

The reasoning is:
  1. Allow for more complex internal javascript. In particular,
  process.stdout and process.stdin can be js implemented streams.

  2. Ease system installs. Loading from disk each time is unnecessary
  overhead. Note that there is no "system" path for modules anymore. Only
  $HOME/.node_libraries.

doc/api.txt

@@ -337,33 +337,20 @@ A module prefixed with +"./"+ is relative to the file calling +require()+.
 That is, +circle.js+ must be in the same directory as +foo.js+ for
 +require("./circle")+ to find it.
 
-Without the leading +"./"+, like +require("mjsunit")+ the module is searched
+Without the leading +"./"+, like +require("assert")+ the module is searched
 for in the +require.paths+ array. +require.paths+ on my system looks like
 this: 
 
 ----------------------------------------
-[ "/home/ryan/.node_libraries"
-, "/usr/local/lib/node/libraries" 
-]
+[ "/home/ryan/.node_libraries" ]
 ----------------------------------------
 
-That is, when +require("mjsunit")+ is called Node looks for 
+That is, when +require("assert")+ is called Node looks for 
 
-  1. +"/home/ryan/.node_libraries/mjsunit.js"+ 
-
-  2. +"/home/ryan/.node_libraries/mjsunit.node"+ 
-
-  3. +"/home/ryan/.node_libraries/mjsunit/index.js"+ 
-
-  4. +"/home/ryan/.node_libraries/mjsunit/index.node"+ 
-
-  5. +"/usr/local/lib/node/libraries/mjsunit.js"+ 
-
-  6. +"/usr/local/lib/node/libraries/mjsunit.node"+ 
-
-  7. +"/usr/local/lib/node/libraries/mjsunit/index.js"+ 
-
-  8. +"/usr/local/lib/node/libraries/mjsunit/index.node"+ 
+  1. +"/home/ryan/.node_libraries/assert.js"+
+  2. +"/home/ryan/.node_libraries/assert.node"+
+  3. +"/home/ryan/.node_libraries/assert/index.js"+
+  4. +"/home/ryan/.node_libraries/assert/index.node"+
 
 interrupting once a file is found. Files ending in +".node"+ are binary Addon
 Modules; see the section below about addons. +"index.js"+ allows one to

lib/assert.js

@@ -6,7 +6,7 @@
 // Copyright (c) 2009 Thomas Robinson <280north.com>
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the “Software”), to
+// of this software and associated documentation files (the 'Software'), to
 // deal in the Software without restriction, including without limitation the
 // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 // sell copies of the Software, and to permit persons to whom the Software is
@@ -15,7 +15,7 @@
 // The above copyright notice and this permission notice shall be included in
 // all copies or substantial portions of the Software.
 //
-// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 // AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
@@ -23,7 +23,7 @@
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 // UTILITY
-var inherits = require('./sys').inherits;
+var inherits = require('sys').inherits;
 var pSlice = Array.prototype.slice;
 
 // 1. The assert module provides functions that throw

lib/fs.js

@@ -1,4 +1,4 @@
-var sys = require('./sys'),
+var sys = require('sys'),
     events = require('events');
 
 var fs = exports;

lib/http.js

@@ -1,4 +1,4 @@
-var sys = require('./sys');
+var sys = require('sys');
 var events = require('events');
 
 var CRLF = "\r\n";

src/node.cc

@@ -1156,6 +1156,26 @@ static void Load(int argc, char *argv[]) {
   HTTPConnection::Initialize(http);             // http.cc
 
 
+  Local<Object> natives = Object::New();
+  process->Set(String::New("natives"), natives);
+  // Explicitly define native sources.
+  natives->Set(String::New("assert"),       String::New(native_assert));
+  natives->Set(String::New("dns"),          String::New(native_dns));
+  natives->Set(String::New("file"),         String::New(native_file));
+  natives->Set(String::New("fs"),           String::New(native_fs));
+  natives->Set(String::New("http"),         String::New(native_http));
+  natives->Set(String::New("ini"),          String::New(native_ini));
+  natives->Set(String::New("mjsunit"),      String::New(native_mjsunit));
+  natives->Set(String::New("multipart"),    String::New(native_multipart));
+  natives->Set(String::New("posix"),        String::New(native_posix));
+  natives->Set(String::New("querystring"),  String::New(native_querystring));
+  natives->Set(String::New("repl"),         String::New(native_repl));
+  natives->Set(String::New("sys"),          String::New(native_sys));
+  natives->Set(String::New("tcp"),          String::New(native_tcp));
+  natives->Set(String::New("uri"),          String::New(native_uri));
+  natives->Set(String::New("url"),          String::New(native_url));
+  natives->Set(String::New("utils"),        String::New(native_utils));
+
 
   // Compile, execute the src/node.js file. (Which was included as static C
   // string in node_natives.h. 'natve_node' is the string containing that
@@ -1262,7 +1282,6 @@ static void ParseArgs(int *argc, char **argv) {
       exit(0);
     } else if (strcmp(arg, "--vars") == 0) {
       printf("NODE_PREFIX: %s\n", NODE_PREFIX);
-      printf("NODE_LIBRARIES_PREFIX: %s/%s\n", NODE_PREFIX, "lib/node/libraries");
       printf("NODE_CFLAGS: %s\n", NODE_CFLAGS);
       exit(0);
     } else if (strcmp(arg, "--help") == 0 || strcmp(arg, "-h") == 0) {

src/node.js

@@ -461,8 +461,7 @@ function existsSync (path) {
 
 
 
-process.paths = [ path.join(process.installPrefix, "lib/node/libraries")
-               ];
+process.paths = [];
 
 if (process.env["HOME"]) {
   process.paths.unshift(path.join(process.env["HOME"], ".node_libraries"));
@@ -553,6 +552,8 @@ function resolveModulePath(request, parent) {
   var id, paths;
   if (request.charAt(0) == "." && (request.charAt(1) == "/" || request.charAt(1) == ".")) {
     // Relative request
+    debug("RELATIVE: requested:" + request + " set ID to: "+id+" from "+parent.id);
+
     var exts = ['js', 'node'], ext;
     for (ext in extensionCache) {
       exts.push(ext.slice(1));
@@ -561,7 +562,6 @@ function resolveModulePath(request, parent) {
     var parentIdPath = path.dirname(parent.id +
       (path.basename(parent.filename).match(new RegExp('^index\\.(' + exts.join('|') + ')$')) ? "/" : ""));
     id = path.join(parentIdPath, request);
-    // debug("RELATIVE: requested:"+request+" set ID to: "+id+" from "+parent.id+"("+parentIdPath+")");
     paths = [path.dirname(parent.filename)];
   } else {
     id = request;
@@ -573,56 +573,61 @@ function resolveModulePath(request, parent) {
 }
 
 
-function loadModuleSync (request, parent) {
-  var resolvedModule = resolveModulePath(request, parent);
-  var id = resolvedModule[0];
-  var paths = resolvedModule[1];
+function loadModule (request, parent, callback) {
+  var resolvedModule = resolveModulePath(request, parent),
+      id = resolvedModule[0],
+      paths = resolvedModule[1];
 
-  debug("loadModuleSync REQUEST  " + (request) + " parent: " + parent.id);
+  debug("loadModule REQUEST  " + (request) + " parent: " + parent.id);
 
   var cachedModule = internalModuleCache[id] || parent.moduleCache[id];
 
-  if (cachedModule) {
-    debug("found  " + JSON.stringify(id) + " in cache");
-    return cachedModule.exports;
-  } else {
-    debug("looking for " + JSON.stringify(id) + " in " + JSON.stringify(paths));
-    var filename = findModulePath(request, paths);
-    if (!filename) {
-      throw new Error("Cannot find module '" + request + "'");
-    } else {
-      var module = new Module(id, parent);
-      module.loadSync(filename);
-      return module.exports;
+  if (!cachedModule) {
+    // Try to compile from native modules
+    if (process.natives[id]) {
+      debug('load native module ' + id);
+      cachedModule = new Module(id);
+      var e = cachedModule._compile(process.natives[id], id);
+      if (e) throw e;
+      internalModuleCache[id] = cachedModule;
     }
   }
-}
-
-
-function loadModule (request, parent, callback) {
-  var
-    resolvedModule = resolveModulePath(request, parent),
-    id = resolvedModule[0],
-    paths = resolvedModule[1];
 
-  debug("loadModule REQUEST  " + (request) + " parent: " + parent.id);
-
-  var cachedModule = internalModuleCache[id] || parent.moduleCache[id];
   if (cachedModule) {
     debug("found  " + JSON.stringify(id) + " in cache");
-    if (callback) callback(null, cachedModule.exports);
-   } else {
-    debug("looking for " + JSON.stringify(id) + " in " + JSON.stringify(paths));
+    if (callback) {
+      callback(null, cachedModule.exports);
+    } else {
+      return cachedModule.exports;
+    }
+
+  } else {
     // Not in cache
-    findModulePath(request, paths, function (filename) {
+    debug("looking for " + JSON.stringify(id) + " in " + JSON.stringify(paths));
+
+    if (!callback) {
+      // sync
+      var filename = findModulePath(request, paths);
       if (!filename) {
-        var err = new Error("Cannot find module '" + request + "'");
-        if (callback) callback(err);
+        throw new Error("Cannot find module '" + request + "'");
       } else {
         var module = new Module(id, parent);
-        module.load(filename, callback);
+        module.loadSync(filename);
+        return module.exports;
       }
-    });
+
+    } else {
+      // async
+      findModulePath(request, paths, function (filename) {
+        if (!filename) {
+          var err = new Error("Cannot find module '" + request + "'");
+          callback(err);
+        } else {
+          var module = new Module(id, parent);
+          module.load(filename, callback);
+        }
+      });
+    }
   }
 };
 
@@ -713,7 +718,8 @@ function cat (id, callback) {
 }
 
 
-Module.prototype._loadContent = function (content, filename) {
+// Returns exception if any
+Module.prototype._compile = function (content, filename) {
   var self = this;
   // remove shebang
   content = content.replace(/^\#\!.*/, '');
@@ -729,7 +735,7 @@ Module.prototype._loadContent = function (content, filename) {
   }
 
   function require (path) {
-    return loadModuleSync(path, self);
+    return loadModule(path, self);
   }
 
   require.paths = process.paths;
@@ -765,7 +771,7 @@ Module.prototype._loadScriptSync = function (filename) {
   // remove shebang
   content = content.replace(/^\#\!.*/, '');
 
-  var e = this._loadContent(content, filename);
+  var e = this._compile(content, filename);
   if (e) {
     throw e;
   } else {
@@ -781,7 +787,7 @@ Module.prototype._loadScript = function (filename, callback) {
     if (err) {
       if (callback) callback(err);
     } else {
-      var e = self._loadContent(content, filename);
+      var e = self._compile(content, filename);
       if (e) {
         if (callback) callback(e);
       } else {

tools/js2c.py

@@ -35,11 +35,22 @@ import os, re, sys, string
 import jsmin
 
 
-def ToCArray(lines):
+def ToCArray(filename, lines):
   result = []
+  row = 1
+  col = 0
   for chr in lines:
+    col += 1
+    if chr == "\n" or chr == "\r":
+      row += 1
+      col = 0
+
     value = ord(chr)
-    assert value < 128
+
+    if value > 128:
+      print 'non-ascii value ' + filename + ':' + str(row) + ':' + str(col)
+      sys.exit(1);
+
     result.append(str(value))
   result.append("0")
   return ", ".join(result)
@@ -231,6 +242,7 @@ def JS2C(source, target):
   # Locate the macros file name.
   consts = {}
   macros = {}
+
   for s in source:
     if 'macros.py' == (os.path.split(str(s))[1]):
       (consts, macros) = ReadMacros(ReadLines(str(s)))
@@ -244,10 +256,11 @@ def JS2C(source, target):
     delay = str(s).endswith('-delay.js')
     lines = ReadFile(str(s))
     do_jsmin = lines.find('// jsminify this file, js2c: jsmin') != -1
+
     lines = ExpandConstants(lines, consts)
     lines = ExpandMacros(lines, macros)
     lines = CompressScript(lines, do_jsmin)
-    data = ToCArray(lines)
+    data = ToCArray(s, lines)
     id = (os.path.split(str(s))[1])[:-3]
     if delay: id = id[:-6]
     if delay:

wscript

@@ -344,11 +344,11 @@ def build(bld):
     js2c.JS2C(source, targets)
 
   native_cc = bld.new_task_gen(
-    source='src/node.js',
+    source='src/node.js ' + bld.path.ant_glob('lib/*.js'),
     target="src/node_natives.h",
-    before="cxx"
+    before="cxx",
+    install_path=None
   )
-  native_cc.install_path = None
 
   # Add the rule /after/ cloning the debug
   # This is a work around for an error had in python 2.4.3 (I'll paste the
@@ -458,8 +458,6 @@ def build(bld):
   bld.install_files('${PREFIX}/lib/node/wafadmin', 'tools/wafadmin/*.py')
   bld.install_files('${PREFIX}/lib/node/wafadmin/Tools', 'tools/wafadmin/Tools/*.py')
 
-  bld.install_files('${PREFIX}/lib/node/libraries/', 'lib/*.js')
-
 def shutdown():
   Options.options.debug
   # HACK to get binding.node out of build directory.