diff --git a/Makefile b/Makefile index 529cc27f48..b46afaebfd 100644 --- a/Makefile +++ b/Makefile @@ -627,13 +627,6 @@ ifeq ($(XZ), 0) ssh $(STAGINGSERVER) "touch nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/node-$(FULLVERSION)-$(OSTYPE)-$(ARCH).tar.xz.done" endif -haswrk=$(shell which wrk > /dev/null 2>&1; echo $$?) -wrk: -ifneq ($(haswrk), 0) - @echo "please install wrk before proceeding. More information can be found in benchmark/README.md." >&2 - @exit 1 -endif - bench-net: all @$(NODE) benchmark/run.js net @@ -643,7 +636,7 @@ bench-crypto: all bench-tls: all @$(NODE) benchmark/run.js tls -bench-http: wrk all +bench-http: all @$(NODE) benchmark/run.js http bench-fs: all diff --git a/benchmark/README.md b/benchmark/README.md index 225236cc10..770df01837 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -14,9 +14,25 @@ This folder contains benchmarks to measure the performance of the Node.js APIs. ## Prerequisites -Most of the http benchmarks require [`wrk`][wrk] to be installed. It may be -available through your preferred package manager. If not, `wrk` can be built -[from source][wrk] via `make`. +Most of the HTTP benchmarks require a benchmarker to be installed, this can be +either [`wrk`][wrk] or [`autocannon`][autocannon]. + +`Autocannon` is a Node script that can be installed using +`npm install -g autocannon`. It will use the Node executable that is in the +path, hence if you want to compare two HTTP benchmark runs make sure that the +Node version in the path is not altered. + +`wrk` may be available through your preferred package manger. If not, you can +easily build it [from source][wrk] via `make`. + +By default `wrk` will be used as benchmarker. If it is not available +`autocannon` will be used in it its place. When creating a HTTP benchmark you +can specify which benchmarker should be used. You can force a specific +benchmarker to be used by providing it as an argument, e. g.: + +`node benchmark/run.js --set benchmarker=autocannon http` + +`node benchmark/http/simple.js benchmarker=autocannon` To analyze the results `R` should be installed. Check you package manager or download it from https://www.r-project.org/. @@ -287,5 +303,48 @@ function main(conf) { } ``` +## Creating HTTP benchmark + +The `bench` object returned by `createBenchmark` implements +`http(options, callback)` method. It can be used to run external tool to +benchmark HTTP servers. + +```js +'use strict'; + +const common = require('../common.js'); + +const bench = common.createBenchmark(main, { + kb: [64, 128, 256, 1024], + connections: [100, 500] +}); + +function main(conf) { + const http = require('http'); + const len = conf.kb * 1024; + const chunk = Buffer.alloc(len, 'x'); + const server = http.createServer(function(req, res) { + res.end(chunk); + }); + + server.listen(common.PORT, function() { + bench.http({ + connections: conf.connections, + }, function() { + server.close(); + }); + }); +} +``` + +Supported options keys are: +* `port` - defaults to `common.PORT` +* `path` - defaults to `/` +* `connections` - number of concurrent connections to use, defaults to 100 +* `duration` - duration of the benchmark in seconds, defaults to 10 +* `benchmarker` - benchmarker to use, defaults to +`common.default_http_benchmarker` + +[autocannon]: https://github.com/mcollina/autocannon [wrk]: https://github.com/wg/wrk [t-test]: https://en.wikipedia.org/wiki/Student%27s_t-test#Equal_or_unequal_sample_sizes.2C_unequal_variances diff --git a/benchmark/_http-benchmarkers.js b/benchmark/_http-benchmarkers.js new file mode 100644 index 0000000000..ca8b462506 --- /dev/null +++ b/benchmark/_http-benchmarkers.js @@ -0,0 +1,130 @@ +'use strict'; + +const child_process = require('child_process'); + +// The port used by servers and wrk +exports.PORT = process.env.PORT || 12346; + +function AutocannonBenchmarker() { + this.name = 'autocannon'; + this.autocannon_exe = process.platform === 'win32' + ? 'autocannon.cmd' + : 'autocannon'; + const result = child_process.spawnSync(this.autocannon_exe, ['-h']); + this.present = !(result.error && result.error.code === 'ENOENT'); +} + +AutocannonBenchmarker.prototype.create = function(options) { + const args = ['-d', options.duration, '-c', options.connections, '-j', '-n', + `http://127.0.0.1:${options.port}${options.path}` ]; + const child = child_process.spawn(this.autocannon_exe, args); + return child; +}; + +AutocannonBenchmarker.prototype.processResults = function(output) { + let result; + try { + result = JSON.parse(output); + } catch (err) { + // Do nothing, let next line handle this + } + if (!result || !result.requests || !result.requests.average) { + return undefined; + } else { + return result.requests.average; + } +}; + +function WrkBenchmarker() { + this.name = 'wrk'; + this.regexp = /Requests\/sec:[ \t]+([0-9\.]+)/; + const result = child_process.spawnSync('wrk', ['-h']); + this.present = !(result.error && result.error.code === 'ENOENT'); +} + +WrkBenchmarker.prototype.create = function(options) { + const args = ['-d', options.duration, '-c', options.connections, '-t', 8, + `http://127.0.0.1:${options.port}${options.path}` ]; + const child = child_process.spawn('wrk', args); + return child; +}; + +WrkBenchmarker.prototype.processResults = function(output) { + const match = output.match(this.regexp); + const result = match && +match[1]; + if (!result) { + return undefined; + } else { + return result; + } +}; + +const http_benchmarkers = [ new WrkBenchmarker(), + new AutocannonBenchmarker() ]; + +const benchmarkers = {}; + +http_benchmarkers.forEach((benchmarker) => { + benchmarkers[benchmarker.name] = benchmarker; + if (!exports.default_http_benchmarker && benchmarker.present) { + exports.default_http_benchmarker = benchmarker.name; + } +}); + +exports.run = function(options, callback) { + options = Object.assign({ + port: exports.PORT, + path: '/', + connections: 100, + duration: 10, + benchmarker: exports.default_http_benchmarker + }, options); + if (!options.benchmarker) { + callback(new Error('Could not locate any of the required http ' + + 'benchmarkers. Check benchmark/README.md for further ' + + 'instructions.')); + return; + } + const benchmarker = benchmarkers[options.benchmarker]; + if (!benchmarker) { + callback(new Error(`Requested benchmarker '${options.benchmarker}' is ` + + 'not supported')); + return; + } + if (!benchmarker.present) { + callback(new Error(`Requested benchmarker '${options.benchmarker}' is ` + + 'not installed')); + return; + } + + const benchmarker_start = process.hrtime(); + + const child = benchmarker.create(options); + + child.stderr.pipe(process.stderr); + + let stdout = ''; + child.stdout.on('data', (chunk) => stdout += chunk.toString()); + + child.once('close', function(code) { + const elapsed = process.hrtime(benchmarker_start); + if (code) { + let error_message = `${options.benchmarker} failed with ${code}.`; + if (stdout !== '') { + error_message += ` Output: ${stdout}`; + } + callback(new Error(error_message), code); + return; + } + + const result = benchmarker.processResults(stdout); + if (!result) { + callback(new Error(`${options.benchmarker} produced strange output: ` + + stdout, code)); + return; + } + + callback(null, code, options.benchmarker, result, elapsed); + }); + +}; diff --git a/benchmark/common.js b/benchmark/common.js index 3807fea795..adc04a0b80 100644 --- a/benchmark/common.js +++ b/benchmark/common.js @@ -1,9 +1,7 @@ 'use strict'; const child_process = require('child_process'); - -// The port used by servers and wrk -exports.PORT = process.env.PORT || 12346; +const http_benchmarkers = require('./_http-benchmarkers.js'); exports.createBenchmark = function(fn, options) { return new Benchmark(fn, options); @@ -11,7 +9,9 @@ exports.createBenchmark = function(fn, options) { function Benchmark(fn, options) { this.name = require.main.filename.slice(__dirname.length + 1); - this.options = this._parseArgs(process.argv.slice(2), options); + const parsed_args = this._parseArgs(process.argv.slice(2), options); + this.options = parsed_args.cli; + this.extra_options = parsed_args.extra; this.queue = this._queue(this.options); this.config = this.queue[0]; @@ -29,7 +29,7 @@ function Benchmark(fn, options) { Benchmark.prototype._parseArgs = function(argv, options) { const cliOptions = Object.assign({}, options); - + const extraOptions = {}; // Parse configuration arguments for (const arg of argv) { const match = arg.match(/^(.+?)=([\s\S]*)$/); @@ -38,14 +38,16 @@ Benchmark.prototype._parseArgs = function(argv, options) { process.exit(1); } - // Infer the type from the options object and parse accordingly - const isNumber = typeof options[match[1]][0] === 'number'; - const value = isNumber ? +match[2] : match[2]; - - cliOptions[match[1]] = [value]; + if (options[match[1]]) { + // Infer the type from the options object and parse accordingly + const isNumber = typeof options[match[1]][0] === 'number'; + const value = isNumber ? +match[2] : match[2]; + cliOptions[match[1]] = [value]; + } else { + extraOptions[match[1]] = match[2]; + } } - - return cliOptions; + return { cli: cliOptions, extra: extraOptions }; }; Benchmark.prototype._queue = function(options) { @@ -88,51 +90,29 @@ Benchmark.prototype._queue = function(options) { return queue; }; -function hasWrk() { - const result = child_process.spawnSync('wrk', ['-h']); - if (result.error && result.error.code === 'ENOENT') { - console.error('Couldn\'t locate `wrk` which is needed for running ' + - 'benchmarks. Check benchmark/README.md for further instructions.'); - process.exit(1); - } -} +// Benchmark an http server. +exports.default_http_benchmarker = + http_benchmarkers.default_http_benchmarker; +exports.PORT = http_benchmarkers.PORT; -// benchmark an http server. -const WRK_REGEXP = /Requests\/sec:[ \t]+([0-9\.]+)/; -Benchmark.prototype.http = function(urlPath, args, cb) { - hasWrk(); +Benchmark.prototype.http = function(options, cb) { const self = this; - - const urlFull = 'http://127.0.0.1:' + exports.PORT + urlPath; - args = args.concat(urlFull); - - const childStart = process.hrtime(); - const child = child_process.spawn('wrk', args); - child.stderr.pipe(process.stderr); - - // Collect stdout - let stdout = ''; - child.stdout.on('data', (chunk) => stdout += chunk.toString()); - - child.once('close', function(code) { - const elapsed = process.hrtime(childStart); - if (cb) cb(code); - - if (code) { - console.error('wrk failed with ' + code); - process.exit(code); + const http_options = Object.assign({ }, options); + http_options.benchmarker = http_options.benchmarker || + self.config.benchmarker || + self.extra_options.benchmarker || + exports.default_http_benchmarker; + http_benchmarkers.run(http_options, function(error, code, used_benchmarker, + result, elapsed) { + if (cb) { + cb(code); } - - // Extract requests pr second and check for odd results - const match = stdout.match(WRK_REGEXP); - if (!match || match.length <= 1) { - console.error('wrk produced strange output:'); - console.error(stdout); - process.exit(1); + if (error) { + console.error(error); + process.exit(code || 1); } - - // Report rate - self.report(+match[1], elapsed); + self.config.benchmarker = used_benchmarker; + self.report(result, elapsed); }); }; @@ -152,6 +132,9 @@ Benchmark.prototype._run = function() { for (const key of Object.keys(config)) { childArgs.push(`${key}=${config[key]}`); } + for (const key of Object.keys(self.extra_options)) { + childArgs.push(`${key}=${self.extra_options[key]}`); + } const child = child_process.fork(require.main.filename, childArgs, { env: childEnv diff --git a/benchmark/http/chunked.js b/benchmark/http/chunked.js index a61978c732..46d6ab2e26 100644 --- a/benchmark/http/chunked.js +++ b/benchmark/http/chunked.js @@ -20,8 +20,6 @@ function main(conf) { const http = require('http'); var chunk = Buffer.alloc(conf.size, '8'); - var args = ['-d', '10s', '-t', 8, '-c', conf.c]; - var server = http.createServer(function(req, res) { function send(left) { if (left === 0) return res.end(); @@ -34,7 +32,9 @@ function main(conf) { }); server.listen(common.PORT, function() { - bench.http('/', args, function() { + bench.http({ + connections: conf.c + }, function() { server.close(); }); }); diff --git a/benchmark/http/cluster.js b/benchmark/http/cluster.js index 95e76e69cc..732a5fad66 100644 --- a/benchmark/http/cluster.js +++ b/benchmark/http/cluster.js @@ -27,9 +27,11 @@ function main(conf) { setTimeout(function() { var path = '/' + conf.type + '/' + conf.length; - var args = ['-d', '10s', '-t', 8, '-c', conf.c]; - bench.http(path, args, function() { + bench.http({ + path: path, + connections: conf.c + }, function() { w1.destroy(); w2.destroy(); }); diff --git a/benchmark/http/end-vs-write-end.js b/benchmark/http/end-vs-write-end.js index 0cdc88111d..62b1a6a097 100644 --- a/benchmark/http/end-vs-write-end.js +++ b/benchmark/http/end-vs-write-end.js @@ -43,14 +43,15 @@ function main(conf) { } var method = conf.method === 'write' ? write : end; - var args = ['-d', '10s', '-t', 8, '-c', conf.c]; var server = http.createServer(function(req, res) { method(res); }); server.listen(common.PORT, function() { - bench.http('/', args, function() { + bench.http({ + connections: conf.c + }, function() { server.close(); }); }); diff --git a/benchmark/http/simple.js b/benchmark/http/simple.js index eedda8e98f..66113ed375 100644 --- a/benchmark/http/simple.js +++ b/benchmark/http/simple.js @@ -15,9 +15,11 @@ function main(conf) { var server = require('./_http_simple.js'); setTimeout(function() { var path = '/' + conf.type + '/' + conf.length + '/' + conf.chunks; - var args = ['-d', '10s', '-t', 8, '-c', conf.c]; - bench.http(path, args, function() { + bench.http({ + path: path, + connections: conf.c + }, function() { server.close(); }); }, 2000);