debugger, cluster: each worker has new debug port

Implement support for debugging cluster workers. Each worker process
is assigned a new debug port in an increasing sequence.

I.e. when master process uses port 5858, then worker 1 uses port 5859,
worker 2 uses port 5860, and so on.

Introduce new command-line parameter '--debug-port=' which sets debug_port
but does not start debugger. This option works for all node processes, it
is not specific to cluster workers.

Fixes joyent/node#5318.

lib/cluster.js

@@ -59,6 +59,25 @@ else
   workerInit();
 
 
+function createWorkerExecArgv(masterExecArgv, worker) {
+  var args = masterExecArgv.slice();
+  var debugPort = process.debugPort + worker.id;
+  var hasDebugArg = false;
+
+  for (var i = 0; i < args.length; i++) {
+    var match = args[i].match(/^(--debug|--debug-brk)(=\d+)?$/);
+    if (!match) continue;
+    args[i] = match[1] + '=' + debugPort;
+    hasDebugArg = true;
+  }
+
+  if (!hasDebugArg)
+    args = ['--debug-port=' + debugPort].concat(args);
+
+  return args;
+}
+
+
 function masterInit() {
   cluster.workers = {};
 
@@ -93,6 +112,13 @@ function masterInit() {
       settings.execArgv = settings.execArgv.concat(['--logfile=v8-%p.log']);
     }
     cluster.settings = settings;
+
+    process.on('internalMessage', function(message) {
+      if (message.cmd !== 'NODE_DEBUG_ENABLED') return;
+      for (key in cluster.workers)
+        process._debugProcess(cluster.workers[key].process.pid);
+    });
+
     cluster.emit('setup');
   };
 
@@ -107,7 +133,7 @@ function masterInit() {
     worker.process = fork(settings.exec, settings.args, {
       env: workerEnv,
       silent: settings.silent,
-      execArgv: settings.execArgv
+      execArgv: createWorkerExecArgv(settings.execArgv, worker)
     });
     worker.process.once('exit', function(exitCode, signalCode) {
       worker.suicide = !!worker.suicide;

src/node.cc

@@ -171,6 +171,7 @@ static double prog_start_time;
 
 static volatile bool debugger_running = false;
 static uv_async_t dispatch_debug_messages_async;
+static uv_async_t emit_debug_enabled_async;
 
 // Declared in node_internals.h
 Isolate* node_isolate = NULL;
@@ -2590,6 +2591,10 @@ static void PrintHelp();
 static void ParseDebugOpt(const char* arg) {
   const char *p = 0;
 
+  if (strstr(arg, "--debug-port=") == arg) {
+    p = 1 + strchr(arg, '=');
+    debug_port = atoi(p);
+  } else {
     use_debug_agent = true;
     if (!strcmp (arg, "--debug-brk")) {
       debug_wait_connect = true;
@@ -2604,6 +2609,7 @@ static void ParseDebugOpt(const char* arg) {
       p = 1 + strchr(arg, '=');
       debug_port = atoi(p);
     }
+  }
   if (p && debug_port > 1024 && debug_port <  65536)
       return;
 
@@ -2643,6 +2649,7 @@ static void PrintHelp() {
          "Documentation can be found at http://nodejs.org/\n");
 }
 
+
 // Parse node command line arguments.
 static void ParseArgs(int argc, char **argv) {
   int i;
@@ -2729,6 +2736,26 @@ static void DispatchMessagesDebugAgentCallback() {
 }
 
 
+// Called from the main thread
+static void EmitDebugEnabledAsyncCallback(uv_async_t* handle, int status) {
+  HandleScope handle_scope(node_isolate);
+  Local<Object> obj = Object::New();
+  obj->Set(String::New("cmd"), String::New("NODE_DEBUG_ENABLED"));
+  Local<Value> args[] = { String::New("internalMessage"), obj };
+  MakeCallback(process, "emit", ARRAY_SIZE(args), args);
+}
+
+
+// Called from the signal handler (unix) or off-thread (windows)
+static void EmitDebugEnabled() {
+  uv_async_init(uv_default_loop(),
+                &emit_debug_enabled_async,
+                EmitDebugEnabledAsyncCallback);
+  uv_unref((uv_handle_t*) &emit_debug_enabled_async);
+  uv_async_send(&emit_debug_enabled_async);
+}
+
+
 static void EnableDebug(bool wait_connect) {
   // If we're called from another thread, make sure to enter the right
   // v8 isolate.
@@ -2756,6 +2783,11 @@ static void EnableDebug(bool wait_connect) {
 
   debugger_running = true;
 
+  // Do not emit _debug_enabled when debugger is enabled before starting
+  // the main process (i.e. when called via `node --debug`)
+  if (!process.IsEmpty())
+    EmitDebugEnabled();
+
   node_isolate->Exit();
 }
 
@@ -3088,15 +3120,8 @@ void AtExit(void (*cb)(void* arg), void* arg) {
 void EmitExit(v8::Handle<v8::Object> process_l) {
   // process.emit('exit')
   process_l->Set(String::NewSymbol("_exiting"), True(node_isolate));
-  Local<Value> emit_v = process_l->Get(String::New("emit"));
-  assert(emit_v->IsFunction());
-  Local<Function> emit = Local<Function>::Cast(emit_v);
   Local<Value> args[] = { String::New("exit"), Integer::New(0, node_isolate) };
-  TryCatch try_catch;
-  emit->Call(process_l, 2, args);
-  if (try_catch.HasCaught()) {
-    FatalException(try_catch);
-  }
+  MakeCallback(process, "emit", ARRAY_SIZE(args), args);
 }
 
 static char **copy_argv(int argc, char **argv) {

test/fixtures/clustered-server/app.js

@@ -0,0 +1,25 @@
+var http = require('http');
+var cluster = require('cluster');
+var common = require('../../common.js');
+
+function handleRequest(request, response) {
+  response.end('hello world\n');
+}
+
+var NUMBER_OF_WORKERS = 2;
+var workersOnline = 0;
+
+if (cluster.isMaster) {
+  cluster.on('online', function() {
+      workersOnline++;
+      if (workersOnline == NUMBER_OF_WORKERS)
+        console.error('all workers are running');
+    });
+
+  for (var i = 0; i < NUMBER_OF_WORKERS; i++) {
+    cluster.fork();
+  }
+} else {
+  var server = http.createServer(handleRequest);
+  server.listen(common.PORT+1000);
+}

test/simple/test-debug-cluster.js

@@ -0,0 +1,68 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+var common = require('../common');
+var assert = require('assert');
+var spawn = require('child_process').spawn;
+
+var args = ['--debug', common.fixturesDir + '/clustered-server/app.js' ];
+var child = spawn(process.execPath, args);
+var outputLines = [];
+
+
+child.stderr.on('data', function(data) {
+  var lines = data.toString().replace(/\r/g, '').trim().split('\n');
+  var line = lines[0];
+
+  lines.forEach(function(ln) { console.log('> ' + ln) } );
+
+  if (line === 'all workers are running') {
+    assertOutputLines();
+    process.exit();
+  } else {
+    outputLines = outputLines.concat(lines);
+  }
+});
+
+setTimeout(function testTimedOut() {
+  assert(false, 'test timed out.');
+}, 3000);
+
+process.on('exit', function() {
+    child.kill();
+});
+
+function assertOutputLines() {
+  var expectedLines = [
+    'debugger listening on port ' + 5858,
+    'debugger listening on port ' + 5859,
+    'debugger listening on port ' + 5860,
+  ];
+
+  // Do not assume any particular order of output messages,
+  // since workers can take different amout of time to
+  // start up
+  outputLines.sort();
+
+  assert.equal(outputLines.length, expectedLines.length)
+  for (var i = 0; i < expectedLines.length; i++)
+    assert.equal(outputLines[i], expectedLines[i]);
+}

test/simple/test-debug-port-cluster.js

@@ -0,0 +1,73 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+var common = require('../common');
+var assert = require('assert');
+var spawn = require('child_process').spawn;
+
+var port = common.PORT + 1337;
+
+var args = [
+  '--debug=' + port,
+  common.fixturesDir + '/clustered-server/app.js'
+];
+
+var child = spawn(process.execPath, args);
+var outputLines = [];
+
+child.stderr.on('data', function(data) {
+  var lines = data.toString().replace(/\r/g, '').trim().split('\n');
+  var line = lines[0];
+
+  lines.forEach(function(ln) { console.log('> ' + ln) } );
+
+  if (line === 'all workers are running') {
+    assertOutputLines();
+    process.exit();
+  } else {
+    outputLines = outputLines.concat(lines);
+  }
+});
+
+setTimeout(function testTimedOut() {
+  assert(false, 'test timed out.');
+}, 3000);
+
+process.on('exit', function() {
+    child.kill();
+});
+
+function assertOutputLines() {
+  var expectedLines = [
+    'debugger listening on port ' + port,
+    'debugger listening on port ' + (port+1),
+    'debugger listening on port ' + (port+2),
+  ];
+
+  // Do not assume any particular order of output messages,
+  // since workers can take different amout of time to
+  // start up
+  outputLines.sort();
+
+  assert.equal(outputLines.length, expectedLines.length)
+  for (var i = 0; i < expectedLines.length; i++)
+    assert.equal(outputLines[i], expectedLines[i]);
+}

test/simple/test-debug-port-from-cmdline.js

@@ -0,0 +1,74 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+var common = require('../common');
+var assert = require('assert');
+var spawn = require('child_process').spawn;
+
+var debugPort = common.PORT;
+var args = ['--debug-port=' + debugPort];
+var child = spawn(process.execPath, args);
+
+child.stderr.on('data', function(data) {
+  var lines = data.toString().replace(/\r/g, '').trim().split('\n');
+  lines.forEach(processStderrLine);
+});
+
+setTimeout(testTimedOut, 3000);
+function testTimedOut() {
+  assert(false, 'test timed out.');
+}
+
+// Give the child process small amout of time to start
+setTimeout(function() {
+  process._debugProcess(child.pid);
+}, 100);
+
+process.on('exit', function() {
+  child.kill();
+});
+
+var outputLines = [];
+function processStderrLine(line) {
+  console.log('> ' + line);
+  outputLines.push(line);
+
+  if (/debugger listening/.test(line)) {
+    assertOutputLines();
+    process.exit();
+  }
+}
+
+function assertOutputLines() {
+  var startLog = process.platform === 'win32'
+                 ? 'Starting debugger agent.'
+                 : 'Hit SIGUSR1 - starting debugger agent.';
+
+  var expectedLines = [
+    startLog,
+    'debugger listening on port ' + debugPort
+  ];
+
+  assert.equal(outputLines.length, expectedLines.length);
+  for (var i = 0; i < expectedLines.length; i++)
+    assert.equal(outputLines[i], expectedLines[i]);
+
+}

test/simple/test-debug-signal-cluster.js

@@ -0,0 +1,86 @@
+// Copyright Joyent, Inc. and other Node contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a
+// copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to permit
+// persons to whom the Software is furnished to do so, subject to the
+// following conditions:
+//
+// The above copyright notice and this permission notice shall be included
+// in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+// USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+var common = require('../common');
+var assert = require('assert');
+var spawn = require('child_process').spawn;
+
+var args = [ common.fixturesDir + '/clustered-server/app.js' ];
+var child = spawn(process.execPath, args);
+var outputLines = [];
+var outputTimerId;
+var waitingForDebuggers = false;
+
+child.stderr.on('data', function(data) {
+  var lines = data.toString().replace(/\r/g, '').trim().split('\n');
+  var line = lines[0];
+
+  lines.forEach(function(ln) { console.log('> ' + ln) } );
+
+  if (outputTimerId !== undefined)
+    clearTimeout(outputTimerId);
+
+  if (waitingForDebuggers) {
+    outputLines = outputLines.concat(lines);
+    outputTimerId = setTimeout(onNoMoreLines, 200);
+  } else if (line === 'all workers are running') {
+    waitingForDebuggers = true;
+    process._debugProcess(child.pid);
+  }
+});
+
+function onNoMoreLines() {
+  assertOutputLines();
+  process.exit();
+}
+
+setTimeout(function testTimedOut() {
+  assert(false, 'test timed out.');
+}, 3000);
+
+process.on('exit', function onExit() {
+    child.kill();
+});
+
+function assertOutputLines() {
+  var startLog = process.platform === 'win32'
+                 ? 'Starting debugger agent.'
+                 : 'Hit SIGUSR1 - starting debugger agent.';
+
+  var expectedLines = [
+    startLog,
+    'debugger listening on port ' + 5858,
+    startLog,
+    'debugger listening on port ' + 5859,
+    startLog,
+    'debugger listening on port ' + 5860,
+  ];
+
+  // Do not assume any particular order of output messages,
+  // since workers can take different amout of time to
+  // start up
+  outputLines.sort();
+  expectedLines.sort();
+
+  assert.equal(outputLines.length, expectedLines.length);
+  for (var i = 0; i < expectedLines.length; i++)
+    assert.equal(outputLines[i], expectedLines[i]);
+}