lukechilds
/
node
mirror of https://github.com/lukechilds/node.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

benchmark: refactor to use process.send 

This removes the need for parsing stdout from the benchmarks. If the
process wasn't executed by fork, it will just print like it used to.

This also fixes the parsing of CLI arguments, by inferring the type
from the options object instead of the value content.

Only two benchmarks had to be changed:

* http/http_server_for_chunky_client.js this previously used a spawn
now it uses a fork and relays the messages using common.sendResult.

* misc/v8-bench.js this utilized that v8/benchmark/run.js called
global.print and reformatted the input. It now interfaces directly
with the benchmark runner global.BenchmarkSuite.

PR-URL: https://github.com/nodejs/node/pull/7094
Reviewed-By: Trevor Norris <trev.norris@gmail.com>
Reviewed-By: Jeremiah Senkpiel <fishrock123@rocketmail.com>
Reviewed-By: Brian White <mscdex@mscdex.net>
Reviewed-By: Anna Henningsen <anna@addaleax.net>
v7.x
Andreas Madsen 9 years ago
parent
0f9bfaa7c5
commit
f99471b2ae
5 changed files with 358 additions and 232 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 99
      benchmark/_cli.js
  2. 318
      benchmark/common.js
  3. 31
      benchmark/http/http_server_for_chunky_client.js
  4. 54
      benchmark/misc/v8-bench.js
  5. 88
      benchmark/run.js

99
benchmark/_cli.js
View File

@ -0,0 +1,99 @@
'use strict';
const fs = require('fs');
const path = require('path');
// Create an object of all benchmark scripts
const benchmarks = {};
fs.readdirSync(__dirname)
.filter(function(name) {
return fs.statSync(path.resolve(__dirname, name)).isDirectory();
})
.forEach(function(category) {
benchmarks[category] = fs.readdirSync(path.resolve(__dirname, category))
.filter((filename) => filename[0] !== '.' && filename[0] !== '_');
});
function CLI(usage, settings) {
if (!(this instanceof CLI)) return new CLI(usage, settings);
if (process.argv.length < 3) {
this.abort(usage); // abort will exit the process
}
this.usage = usage;
this.optional = {};
this.items = [];
for (const argName of settings.arrayArgs) {
this.optional[argName] = [];
}
let currentOptional = null;
let mode = 'both'; // possible states are: [both, option, item]
for (const arg of process.argv.slice(2)) {
if (arg === '--') {
// Only items can follow --
mode = 'item';
} else if (['both', 'option'].includes(mode) && arg[0] === '-') {
// Optional arguments declaration
if (arg[1] === '-') {
currentOptional = arg.slice(2);
} else {
currentOptional = arg.slice(1);
}
// Default the value to true
if (!settings.arrayArgs.includes(currentOptional)) {
this.optional[currentOptional] = true;
}
// expect the next value to be option related (either -- or the value)
mode = 'option';
} else if (mode === 'option') {
// Optional arguments value
if (settings.arrayArgs.includes(currentOptional)) {
this.optional[currentOptional].push(arg);
} else {
this.optional[currentOptional] = arg;
}
// the next value can be either an option or an item
mode = 'both';
} else if (['both', 'item'].includes(mode)) {
// item arguments
this.items.push(arg);
// the next value must be an item
mode = 'item';
} else {
// Bad case, abort
this.abort(usage);
return;
}
}
}
module.exports = CLI;
CLI.prototype.abort = function(msg) {
console.error(msg);
process.exit(1);
};
CLI.prototype.benchmarks = function() {
const paths = [];
const filter = this.optional.filter || false;
for (const category of this.items) {
for (const scripts of benchmarks[category]) {
if (filter && scripts.lastIndexOf(filter) === -1) continue;
paths.push(path.join(category, scripts));
}
}
return paths;
};

318
benchmark/common.js
View File

@ -1,201 +1,227 @@
'use strict'; 'use strict';
var assert = require('assert');
var fs = require('fs');
var path = require('path');
var child_process = require('child_process');
var outputFormat = process.env.OUTPUT_FORMAT ||
(+process.env.NODE_BENCH_SILENT ? 'silent' : false) ||
'default';
// verify outputFormat
if (['default', 'csv', 'silent'].indexOf(outputFormat) == -1) {
throw new Error('OUTPUT_FORMAT set to invalid value');
}
exports.PORT = process.env.PORT || 12346; const child_process = require('child_process');
function hasWrk() { // The port used by servers and wrk
var result = child_process.spawnSync('wrk', ['-h']); exports.PORT = process.env.PORT || 12346;
if (result.error && result.error.code === 'ENOENT') {
console.error('Couldn\'t locate `wrk` which is needed for running ' +
'benchmarks. Check benchmark/README.md for further instructions.');
process.exit(-1);
}
}
exports.createBenchmark = function(fn, options) { exports.createBenchmark = function(fn, options) {
return new Benchmark(fn, options); return new Benchmark(fn, options);
}; };
function Benchmark(fn, options) { function Benchmark(fn, options) {
this.fn = fn; this.name = require.main.filename.slice(__dirname.length + 1);
this.options = options; this.options = this._parseArgs(process.argv.slice(2), options);
this.config = parseOpts(options); this.queue = this._queue(this.options);
this._name = require.main.filename.split(/benchmark[\/\\]/).pop(); this.config = this.queue[0];
this._start = [0, 0];
this._time = [0, 0]; // holds process.hrtime value
this._started = false; this._started = false;
var self = this; // this._run will use fork() to create a new process for each configuration
// combination.
if (process.env.hasOwnProperty('NODE_RUN_BENCHMARK_FN')) {
process.nextTick(() => fn(this.config));
} else {
process.nextTick(() => this._run());
}
}
process.nextTick(function() { Benchmark.prototype._parseArgs = function(argv, options) {
self._run(); const cliOptions = Object.assign({}, options);
});
// Parse configuarion arguments
for (const arg of argv) {
const match = arg.match(/^(.+?)=([\s\S]*)$/);
if (!match || !match[1]) {
console.error('bad argument: ' + arg);
process.exit(1);
}
// Infer the type from the options object and parse accordingly
const isNumber = typeof options[match[1]][0] === 'number';
const value = isNumber ? +match[2] : match[2];
cliOptions[match[1]] = [value];
}
return cliOptions;
};
Benchmark.prototype._queue = function(options) {
const queue = [];
const keys = Object.keys(options);
// Perform a depth-first walk though all options to genereate a
// configuration list that contains all combinations.
function recursive(keyIndex, prevConfig) {
const key = keys[keyIndex];
const values = options[key];
const type = typeof values[0];
for (const value of values) {
if (typeof value !== 'number' && typeof value !== 'string') {
throw new TypeError(`configuration "${key}" had type ${typeof value}`);
}
if (typeof value !== type) {
// This is a requirement for being able to consistently and predictably
// parse CLI provided configuration values.
throw new TypeError(`configuration "${key}" has mixed types`);
}
const currConfig = Object.assign({ [key]: value }, prevConfig);
if (keyIndex + 1 < keys.length) {
recursive(keyIndex + 1, currConfig);
} else {
queue.push(currConfig);
}
}
}
if (keys.length > 0) {
recursive(0, {});
} else {
queue.push({});
}
return queue;
};
function hasWrk() {
const result = child_process.spawnSync('wrk', ['-h']);
if (result.error && result.error.code === 'ENOENT') {
console.error('Couldn\'t locate `wrk` which is needed for running ' +
'benchmarks. Check benchmark/README.md for further instructions.');
process.exit(1);
}
} }
// benchmark an http server. // benchmark an http server.
Benchmark.prototype.http = function(p, args, cb) { const WRK_REGEXP = /Requests\/sec:[ \t]+([0-9\.]+)/;
Benchmark.prototype.http = function(urlPath, args, cb) {
hasWrk(); hasWrk();
var self = this; const self = this;
var regexp = /Requests\/sec:[ \t]+([0-9\.]+)/;
var url = 'http://127.0.0.1:' + exports.PORT + p;
args = args.concat(url);
var out = ''; const urlFull = 'http://127.0.0.1:' + exports.PORT + urlPath;
var child = child_process.spawn('wrk', args); args = args.concat(urlFull);
child.stdout.setEncoding('utf8'); const childStart = process.hrtime();
const child = child_process.spawn('wrk', args);
child.stderr.pipe(process.stderr);
child.stdout.on('data', function(chunk) { // Collect stdout
out += chunk; let stdout = '';
}); child.stdout.on('data', (chunk) => stdout += chunk.toString());
child.on('close', function(code) { child.once('close', function(code) {
if (cb) const elapsed = process.hrtime(childStart);
cb(code); if (cb) cb(code);
if (code) { if (code) {
console.error('wrk failed with ' + code); console.error('wrk failed with ' + code);
process.exit(code); process.exit(code);
} }
var match = out.match(regexp);
var qps = match && +match[1]; // Extract requests pr second and check for odd results
if (!qps) { const match = stdout.match(WRK_REGEXP);
console.error('%j', out); if (!match || match.length <= 1) {
console.error('wrk produced strange output'); console.error('wrk produced strange output:');
console.error(stdout);
process.exit(1); process.exit(1);
} }
self.report(+qps);
// Report rate
self.report(+match[1], elapsed);
}); });
}; };
Benchmark.prototype._run = function() { Benchmark.prototype._run = function() {
if (this.config) const self = this;
return this.fn(this.config);
(function recursive(queueIndex) {
// some options weren't set. const config = self.queue[queueIndex];
// run with all combinations
var main = require.main.filename;
var options = this.options;
var queue = Object.keys(options).reduce(function(set, key) {
var vals = options[key];
assert(Array.isArray(vals));
// match each item in the set with each item in the list
var newSet = new Array(set.length * vals.length);
var j = 0;
set.forEach(function(s) {
vals.forEach(function(val) {
if (typeof val !== 'number' && typeof val !== 'string') {
throw new TypeError(`configuration "${key}" had type ${typeof val}`);
}
newSet[j++] = s.concat(key + '=' + val);
});
});
return newSet;
}, [[main]]);
// output csv heading
if (outputFormat == 'csv')
console.log('filename,' + Object.keys(options).join(',') + ',result');
var node = process.execPath;
var i = 0;
function run() {
var argv = queue[i++];
if (!argv)
return;
argv = process.execArgv.concat(argv);
var child = child_process.spawn(node, argv, { stdio: 'inherit' });
child.on('close', function(code, signal) {
if (code)
console.error('child process exited with code ' + code);
else
run();
});
}
run();
};
function parseOpts(options) { // set NODE_RUN_BENCHMARK_FN to indicate that the child shouldn't construct
// verify that there's an option provided for each of the options // a configuration queue, but just execute the benchmark function.
// if they're not *all* specified, then we return null. const childEnv = Object.assign({}, process.env);
var keys = Object.keys(options); childEnv.NODE_RUN_BENCHMARK_FN = '';
var num = keys.length;
var conf = {}; // Create configuration arguments
for (var i = 2; i < process.argv.length; i++) { const childArgs = [];
var match = process.argv[i].match(/^(.+?)=([\s\S]*)$/); for (const key of Object.keys(config)) {
if (!match || !match[1] || !options[match[1]]) { childArgs.push(`${key}=${config[key]}`);
return null;
} else {
conf[match[1]] = match[2];
num--;
} }
}
// still go ahead and set whatever WAS set, if it was. const child = child_process.fork(require.main.filename, childArgs, {
if (num !== 0) { env: childEnv
Object.keys(conf).forEach(function(k) {
options[k] = [conf[k]];
}); });
} child.on('message', sendResult);
return num === 0 ? conf : null; child.on('close', function(code) {
} if (code) {
process.exit(code);
return;
}
if (queueIndex + 1 < self.queue.length) {
recursive(queueIndex + 1);
}
});
})(0);
};
Benchmark.prototype.start = function() { Benchmark.prototype.start = function() {
if (this._started) if (this._started)
throw new Error('Called start more than once in a single benchmark'); throw new Error('Called start more than once in a single benchmark');
this._started = true; this._started = true;
this._start = process.hrtime(); this._time = process.hrtime();
}; };
Benchmark.prototype.end = function(operations) { Benchmark.prototype.end = function(operations) {
var elapsed = process.hrtime(this._start); // get elapsed time now and do error checking later for accuracy.
const elapsed = process.hrtime(this._time);
if (!this._started) if (!this._started) {
throw new Error('called end without start'); throw new Error('called end without start');
if (typeof operations !== 'number') }
if (typeof operations !== 'number') {
throw new Error('called end() without specifying operation count'); throw new Error('called end() without specifying operation count');
}
var time = elapsed[0] + elapsed[1] / 1e9; const time = elapsed[0] + elapsed[1] / 1e9;
var rate = operations / time; const rate = operations / time;
this.report(rate); this.report(rate, elapsed);
}; };
Benchmark.prototype.report = function(value) { function formatResult(data) {
var heading = this.getHeading(); // Construct confiuration string, " A=a, B=b, ..."
let conf = '';
for (const key of Object.keys(data.conf)) {
conf += ' ' + key + '=' + JSON.stringify(data.conf[key]);
}
if (outputFormat == 'default') return `${data.name}${conf}: ${data.rate}`;
console.log('%s: %s', heading, value.toFixed(5)); }
else if (outputFormat == 'csv')
console.log('%s,%s', heading, value.toFixed(5));
};
Benchmark.prototype.getHeading = function() { function sendResult(data) {
var conf = this.config; if (process.send) {
// If forked, report by process send
if (outputFormat == 'default') { process.send(data);
return this._name + ' ' + Object.keys(conf).map(function(key) { } else {
return key + '=' + JSON.stringify('' + conf[key]); // Otherwise report by stdout
}).join(' '); console.log(formatResult(data));
} else if (outputFormat == 'csv') {
return this._name + ',' + Object.keys(conf).map(function(key) {
return JSON.stringify('' + conf[key]);
}).join(',');
} }
}
exports.sendResult = sendResult;
Benchmark.prototype.report = function(rate, elapsed) {
sendResult({
name: this.name,
conf: this.config,
rate: rate,
time: elapsed[0] + elapsed[1] / 1e9
});
}; };
exports.v8ForceOptimization = function(method, ...args) { exports.v8ForceOptimization = function(method, ...args) {

31
benchmark/http/http_server_for_chunky_client.js
View File

@ -3,8 +3,8 @@
var path = require('path'); var path = require('path');
var http = require('http'); var http = require('http');
var fs = require('fs'); var fs = require('fs');
var spawn = require('child_process').spawn; var fork = require('child_process').fork;
require('../common.js'); var common = require('../common.js');
var test = require('../../test/common.js'); var test = require('../../test/common.js');
var pep = path.dirname(process.argv[1]) + '/_chunky_http_client.js'; var pep = path.dirname(process.argv[1]) + '/_chunky_http_client.js';
var PIPE = test.PIPE; var PIPE = test.PIPE;
@ -30,25 +30,10 @@ server.on('error', function(err) {
throw new Error('server error: ' + err); throw new Error('server error: ' + err);
}); });
try { server.listen(PIPE);
var child;
server.listen(PIPE);
child = spawn(process.execPath, [pep], { });
child.on('error', function(err) {
throw new Error('spawn error: ' + err);
});
child.stdout.pipe(process.stdout);
child.stderr.pipe(process.stderr);
child.on('close', function(exitCode) {
server.close();
});
} catch (e) {
throw new Error('error: ' + e);
}
var child = fork(pep, process.argv.slice(2));
child.on('message', common.sendResult);
child.on('close', function() {
server.close();
});

54
benchmark/misc/v8-bench.js
View File

@ -3,21 +3,49 @@
var fs = require('fs'); var fs = require('fs');
var path = require('path'); var path = require('path');
var vm = require('vm'); var vm = require('vm');
var common = require('../common.js');
var dir = path.join(__dirname, '..', '..', 'deps', 'v8', 'benchmarks'); var dir = path.join(__dirname, '..', '..', 'deps', 'v8', 'benchmarks');
global.print = function(s) { function load(filename, inGlobal) {
if (s === '----') return;
console.log('misc/v8_bench.js %s', s);
};
global.load = function(filename) {
var source = fs.readFileSync(path.join(dir, filename), 'utf8'); var source = fs.readFileSync(path.join(dir, filename), 'utf8');
// deps/v8/benchmarks/regexp.js breaks console.log() because it clobbers if (!inGlobal) source = '(function () {' + source + '\n})()';
// the RegExp global, Restore the original when the script is done. vm.runInThisContext(source, { filename: 'v8/bechmark/' + filename });
var $RegExp = global.RegExp; }
vm.runInThisContext(source, { filename: filename });
global.RegExp = $RegExp; load('base.js', true);
}; load('richards.js');
load('deltablue.js');
load('crypto.js');
load('raytrace.js');
load('earley-boyer.js');
load('regexp.js');
load('splay.js');
load('navier-stokes.js');
global.load('run.js'); const times = {};
global.BenchmarkSuite.RunSuites({
NotifyStart: function(name) {
times[name] = process.hrtime();
},
NotifyResult: function(name, result) {
const elapsed = process.hrtime(times[name]);
common.sendResult({
name: name,
conf: {},
rate: result,
time: elapsed[0] + elapsed[1] / 1e9
});
},
NotifyError: function(name, error) {
console.error(name + ': ' + error);
},
NotifyScore: function(score) {
common.sendResult({
name: 'Score (version ' + global.BenchmarkSuite.version + ')',
conf: {},
rate: score,
time: 0
});
}
});

88
benchmark/run.js
View File

@ -1,63 +1,51 @@
'use strict'; 'use strict';
const fs = require('fs');
const path = require('path'); const path = require('path');
const child_process = require('child_process'); const fork = require('child_process').fork;
const CLI = require('./_cli.js');
var outputFormat = process.env.OUTPUT_FORMAT ||
(+process.env.NODE_BENCH_SILENT ? 'silent' : false) || const cli = CLI(`usage: ./node run.js [options] [--] <category> ...
'default'; Run each benchmark in the <category> directory a single time, more than one
<categoty> directory can be specified.
// If this is the main module, then run the benchmarks
if (module === require.main) { --filter pattern string to filter benchmark scripts
var type = process.argv[2]; --set variable=value set benchmark variable (can be repeated)
var testFilter = process.argv[3]; `, {
if (!type) { arrayArgs: ['set']
console.error('usage:\n ./node benchmark/run.js <type> [testFilter]'); });
process.exit(1); const benchmarks = cli.benchmarks();
}
if (benchmarks.length === 0) {
var dir = path.join(__dirname, type); console.error('no benchmarks found');
var tests = fs.readdirSync(dir); process.exit(1);
if (testFilter) {
var filteredTests = tests.filter(function(item) {
if (item.lastIndexOf(testFilter) >= 0) {
return item;
}
});
if (filteredTests.length === 0) {
console.error('%s is not found in \n %j', testFilter, tests);
return;
}
tests = filteredTests;
}
runBenchmarks();
} }
function runBenchmarks() { (function recursive(i) {
var test = tests.shift(); const filename = benchmarks[i];
if (!test) const child = fork(path.resolve(__dirname, filename), cli.optional.set);
return;
if (test.match(/^[\._]/)) console.log();
return process.nextTick(runBenchmarks); console.log(filename);
if (outputFormat == 'default') child.on('message', function(data) {
console.error(type + '/' + test); // Construct configuration string, " A=a, B=b, ..."
let conf = '';
for (const key of Object.keys(data.conf)) {
conf += ' ' + key + '=' + JSON.stringify(data.conf[key]);
}
test = path.resolve(dir, test); console.log(`${data.name}${conf}: ${data.rate}`);
});
var a = (process.execArgv || []).concat(test); child.once('close', function(code) {
var child = child_process.spawn(process.execPath, a, { stdio: 'inherit' });
child.on('close', function(code) {
if (code) { if (code) {
process.exit(code); process.exit(code);
} else { return;
console.log(''); }
runBenchmarks();
// If there are more benchmarks execute the next
if (i + 1 < benchmarks.length) {
recursive(i + 1);
} }
}); });
} })(0);

Write Preview
Loading…
Cancel
Save