From 85cab944c74e3c211da55f69925e90dc93668bce Mon Sep 17 00:00:00 2001 From: James Talmage Date: Mon, 1 Feb 2016 14:16:45 +0700 Subject: [PATCH] Close #496 PR: Add a benchmark suite. --- .gitignore | 1 + bench/compare.js | 105 ++++++++++++++++++ bench/concurrent/alternating-sync-async.js | 9 ++ bench/concurrent/async-immediate.js | 5 + bench/concurrent/async-timeout.js | 5 + bench/concurrent/sync.js | 5 + bench/other/failures.js | 7 ++ bench/run.js | 117 +++++++++++++++++++++ bench/serial/alternating-sync-async.js | 9 ++ bench/serial/async-immediate.js | 5 + bench/serial/async-timeout.js | 5 + bench/serial/sync.js | 5 + package.json | 6 +- 13 files changed, 283 insertions(+), 1 deletion(-) create mode 100644 bench/compare.js create mode 100644 bench/concurrent/alternating-sync-async.js create mode 100644 bench/concurrent/async-immediate.js create mode 100644 bench/concurrent/async-timeout.js create mode 100644 bench/concurrent/sync.js create mode 100644 bench/other/failures.js create mode 100644 bench/run.js create mode 100644 bench/serial/alternating-sync-async.js create mode 100644 bench/serial/async-immediate.js create mode 100644 bench/serial/async-timeout.js create mode 100644 bench/serial/sync.js diff --git a/.gitignore b/.gitignore index 1e59e82..0fe5f8e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules .nyc_output coverage +bench/.results diff --git a/bench/compare.js b/bench/compare.js new file mode 100644 index 0000000..15004da --- /dev/null +++ b/bench/compare.js @@ -0,0 +1,105 @@ +'use strict'; + +var path = require('path'); +var fs = require('fs'); +var Table = require('cli-table2'); +var chalk = require('chalk'); + +var files = fs.readdirSync(path.join(__dirname, '.results')) + .map(function (file) { + var result = JSON.parse(fs.readFileSync(path.join(__dirname, '.results', file), 'utf8')); + result['.file'] = path.basename(file, '.json'); + return result; + }) + // find the most recent benchmark runs + .sort(function (fileA, fileB) { + return fileB['.time'] - fileA['.time']; + }); + +// Only the 3 most recent runs +files = files.slice(0, 3); + +function prepStats(times) { + times = times + .map(function (time) { + return time.time; + }) + .sort(function (timeA, timeB) { + return timeA - timeB; + }); + + // remove fastest and slowest + times = times.slice(1, times.length - 1); + + var sum = times.reduce(function (a, b) { + return a + b; + }, 0); + + return { + mean: Math.round((sum / times.length) * 1000) / 1000, + median: times[Math.floor(times.length / 2)], + min: times[0], + max: times[times.length - 1] + }; +} + +var results = {}; +var fileNames = files.map(function (file) { + return file['.file']; +}); +var stats = ['mean', 'median', 'min', 'max']; + +files.forEach(function (file) { + Object.keys(file) + .filter(function (key) { + return !/^\./.test(key); + }) + .forEach(function (key) { + results[key] = results[key] || {}; + results[key][file['.file']] = prepStats(file[key]); + }); +}); + +var table = new Table(); +table.push( + [''].concat(stats.map(function (stat) { + return { + content: stat, + colSpan: fileNames.length, + hAlign: 'center' + }; + })), + stats.reduce(function (arr) { + return arr.concat(fileNames); + }, ['args']) +); + +Object.keys(results) + .forEach(function (key) { + table.push(stats.reduce(function (arr, stat) { + var min = Infinity; + var max = -Infinity; + + var statGroup = fileNames.map(function (fileName) { + var result = results[key][fileName]; + result = result && result[stat]; + if (result) { + min = Math.min(min, result); + max = Math.max(max, result); + return result; + } + return ''; + }); + return arr.concat(statGroup.map(function (stat) { + if (stat === min) { + return chalk.green(stat); + } + if (stat === max) { + return chalk.red(stat); + } + return stat; + })); + }, [key])); + }); + +console.log(table.toString()); diff --git a/bench/concurrent/alternating-sync-async.js b/bench/concurrent/alternating-sync-async.js new file mode 100644 index 0000000..f01e7f2 --- /dev/null +++ b/bench/concurrent/alternating-sync-async.js @@ -0,0 +1,9 @@ +import test from '../../'; + +for (var i = 0; i < 10000; i++) { + if (i % 2) { + test('test' + i, () => new Promise(resolve => setImmediate(resolve))); + } else { + test('test' + i, () => {}); + } +} diff --git a/bench/concurrent/async-immediate.js b/bench/concurrent/async-immediate.js new file mode 100644 index 0000000..aabed8f --- /dev/null +++ b/bench/concurrent/async-immediate.js @@ -0,0 +1,5 @@ +import test from '../../'; + +for (var i = 0; i < 10000; i++) { + test('test' + i, () => new Promise(resolve => setImmediate(resolve))); +} diff --git a/bench/concurrent/async-timeout.js b/bench/concurrent/async-timeout.js new file mode 100644 index 0000000..cbb4c3d --- /dev/null +++ b/bench/concurrent/async-timeout.js @@ -0,0 +1,5 @@ +import test from '../../'; + +for (var i = 0; i < 10000; i++) { + test('test' + i, () => new Promise(resolve => setTimeout(resolve, 0))); +} diff --git a/bench/concurrent/sync.js b/bench/concurrent/sync.js new file mode 100644 index 0000000..a37c70c --- /dev/null +++ b/bench/concurrent/sync.js @@ -0,0 +1,5 @@ +import test from '../../'; + +for (var i = 0; i < 10000; i++) { + test('test' + i, () => {}); +} diff --git a/bench/other/failures.js b/bench/other/failures.js new file mode 100644 index 0000000..1ef892a --- /dev/null +++ b/bench/other/failures.js @@ -0,0 +1,7 @@ +import test from '../../'; + +for (var i = 0; i < 1000; i++) { + test.serial('test' + i, t => { + t.is(Math.random(), Math.random()); + }); +} diff --git a/bench/run.js b/bench/run.js new file mode 100644 index 0000000..603f83c --- /dev/null +++ b/bench/run.js @@ -0,0 +1,117 @@ +'use strict'; + +var childProcess = require('child_process'); +var path = require('path'); +var fs = require('fs'); +var arrify = require('arrify'); +var Promise = require('bluebird'); +var mkdirp = require('mkdirp'); +var branch = require('git-branch').sync(path.join(__dirname, '..')); +var cliPath = require.resolve('../cli'); + +function runTests(_args) { + return new Promise(function (resolve) { + var args = [cliPath] + .concat(arrify(_args)); + var start = Date.now(); + childProcess.execFile(process.execPath, args, { + cwd: __dirname, + maxBuffer: 100000 * 200 + }, function (err, stdout, stderr) { + var end = Date.now(); + resolve({ + args: arrify(_args), + time: end - start, + err: err, + stdout: stdout, + stderr: stderr + }); + }); + }); +} + +var list; + +if (process.argv.length === 2) { + list = [ + {args: 'other/failures.js', shouldFail: true}, + 'serial/alternating-sync-async.js', + 'serial/async-immediate.js', + 'serial/async-timeout.js', + 'serial/sync.js', + 'concurrent/alternating-sync-async.js', + 'concurrent/async-immediate.js', + 'concurrent/async-timeout.js', + 'concurrent/sync.js', + ['concurrent/*.js', 'serial/*.js'] + ].map(function (definition) { + if (Array.isArray(definition) || typeof definition === 'string') { + definition = { + shouldFail: false, + args: definition + }; + } + return definition; + }); +} else { + list = []; + var currentArgs = []; + var shouldFail = false; + process.argv.slice(2).forEach(function (arg) { + if (arg === '--') { + list.push({ + args: currentArgs, + shouldFail: shouldFail + }); + currentArgs = []; + shouldFail = false; + return; + } + if (arg === '--should-fail') { + shouldFail = true; + return; + } + currentArgs.push(arg); + }); + if (currentArgs.length) { + list.push({ + args: currentArgs, + shouldFail: shouldFail + }); + } +} + +list.forEach(function (definition) { + definition.args = ['--verbose'].concat(definition.args); +}); + +var combined = []; +for (var i = 0; i < 11; i++) { + combined = combined.concat(list); +} + +var results = {}; + +Promise.each(combined, function (definition) { + var args = definition.args; + return runTests(args).then(function (result) { + var key = result.args.join(' '); + var passedOrFaild = result.err ? 'failed' : 'passed'; + var seconds = result.time / 1000; + console.log('%s %s in %d seconds', key, passedOrFaild, seconds); + if (result.err && !definition.shouldFail) { + console.log(result.stdout); + console.log(result.stderr); + throw (result.err); + } + results[key] = results[key] || []; + results[key].push({passed: !results.err, shouldFail: definition.shouldFail, time: seconds}); + }); +}).then(function () { + mkdirp.sync(path.join(__dirname, '.results')); + results['.time'] = Date.now(); + fs.writeFileSync( + path.join(__dirname, '.results', branch + '.json'), + JSON.stringify(results, null, 4) + ); +}); diff --git a/bench/serial/alternating-sync-async.js b/bench/serial/alternating-sync-async.js new file mode 100644 index 0000000..50e3e1c --- /dev/null +++ b/bench/serial/alternating-sync-async.js @@ -0,0 +1,9 @@ +import test from '../../'; + +for (var i = 0; i < 10000; i++) { + if (i % 2) { + test.serial('test' + i, () => new Promise(resolve => setImmediate(resolve))); + } else { + test.serial('test' + i, () => {}); + } +} diff --git a/bench/serial/async-immediate.js b/bench/serial/async-immediate.js new file mode 100644 index 0000000..a26c3a1 --- /dev/null +++ b/bench/serial/async-immediate.js @@ -0,0 +1,5 @@ +import test from '../../'; + +for (var i = 0; i < 10000; i++) { + test.serial('test' + i, () => new Promise(resolve => setImmediate(resolve))); +} diff --git a/bench/serial/async-timeout.js b/bench/serial/async-timeout.js new file mode 100644 index 0000000..ba3bf20 --- /dev/null +++ b/bench/serial/async-timeout.js @@ -0,0 +1,5 @@ +import test from '../../'; + +for (var i = 0; i < 3000; i++) { + test.serial('test' + i, () => new Promise(resolve => setTimeout(resolve, 0))); +} diff --git a/bench/serial/sync.js b/bench/serial/sync.js new file mode 100644 index 0000000..f41140e --- /dev/null +++ b/bench/serial/sync.js @@ -0,0 +1,5 @@ +import test from '../../'; + +for (var i = 0; i < 10000; i++) { + test.serial('test' + i, () => {}); +} diff --git a/package.json b/package.json index 74b1db8..dcd1ef8 100644 --- a/package.json +++ b/package.json @@ -124,10 +124,13 @@ "update-notifier": "^0.6.0" }, "devDependencies": { + "cli-table2": "^0.1.9", "coveralls": "^2.11.4", "delay": "^1.3.0", "get-stream": "^1.1.0", + "git-branch": "^0.3.0", "inquirer": "^0.11.1", + "mkdirp": "^0.5.1", "nyc": "^5.1.0", "pify": "^2.3.0", "rimraf": "^2.5.0", @@ -140,7 +143,8 @@ }, "xo": { "ignore": [ - "cli.js" + "cli.js", + "bench/*/*.js" ] } }