mirror of https://github.com/lukechilds/node.git
Browse Source
This commit switches from the eslint command-line tool to a custom tool that uses eslint programmatically in order to perform linting in parallel and to display linting results incrementally instead of buffering them until the end. Fixes: https://github.com/nodejs/node/issues/5596 PR-URL: https://github.com/nodejs/node/pull/5638 Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl> Reviewed-By: Johan Bergström <bugs@bergstroem.nu>process-exit-stdio-flushing
Brian White
9 years ago
3 changed files with 279 additions and 5 deletions
@ -0,0 +1,262 @@ |
|||
'use strict'; |
|||
|
|||
const rulesDirs = ['tools/eslint-rules']; |
|||
// This is the maximum number of files to be linted per worker at any given time
|
|||
const maxWorkload = 40; |
|||
|
|||
const cluster = require('cluster'); |
|||
const path = require('path'); |
|||
const fs = require('fs'); |
|||
const totalCPUs = require('os').cpus().length; |
|||
|
|||
const CLIEngine = require('./eslint').CLIEngine; |
|||
const glob = require('./eslint/node_modules/glob'); |
|||
|
|||
const cwd = process.cwd(); |
|||
const cli = new CLIEngine({ |
|||
rulePaths: rulesDirs |
|||
}); |
|||
|
|||
if (cluster.isMaster) { |
|||
var numCPUs = 1; |
|||
const paths = []; |
|||
var files = null; |
|||
var totalPaths = 0; |
|||
var failures = 0; |
|||
var successes = 0; |
|||
var lastLineLen = 0; |
|||
var curPath = 'Starting ...'; |
|||
var showProgress = true; |
|||
const globOptions = { |
|||
nodir: true |
|||
}; |
|||
const workerConfig = {}; |
|||
var startTime; |
|||
var formatter; |
|||
var outFn; |
|||
var fd; |
|||
var i; |
|||
|
|||
// Check if spreading work among all cores/cpus
|
|||
if (process.argv.indexOf('-J') !== -1) |
|||
numCPUs = totalCPUs; |
|||
|
|||
// Check if spreading work among an explicit number of cores/cpus
|
|||
i = process.argv.indexOf('-j'); |
|||
if (i !== -1) { |
|||
if (!process.argv[i + 1]) |
|||
throw new Error('Missing parallel job count'); |
|||
numCPUs = parseInt(process.argv[i + 1], 10); |
|||
if (!isFinite(numCPUs) || numCPUs <= 0) |
|||
throw new Error('Bad parallel job count'); |
|||
} |
|||
|
|||
// Check for custom JSLint report formatter
|
|||
i = process.argv.indexOf('-f'); |
|||
if (i !== -1) { |
|||
if (!process.argv[i + 1]) |
|||
throw new Error('Missing format name'); |
|||
const format = process.argv[i + 1]; |
|||
formatter = cli.getFormatter(format); |
|||
if (!formatter) |
|||
throw new Error('Invalid format name'); |
|||
// Automatically disable progress display
|
|||
showProgress = false; |
|||
// Tell worker to send all results, not just linter errors
|
|||
workerConfig.sendAll = true; |
|||
} else { |
|||
// Use default formatter
|
|||
formatter = cli.getFormatter(); |
|||
} |
|||
|
|||
// Check if outputting JSLint report to a file instead of stdout
|
|||
i = process.argv.indexOf('-o'); |
|||
if (i !== -1) { |
|||
if (!process.argv[i + 1]) |
|||
throw new Error('Missing output filename'); |
|||
var outPath = process.argv[i + 1]; |
|||
if (!path.isAbsolute(outPath)) |
|||
outPath = path.join(cwd, outPath); |
|||
fd = fs.openSync(outPath, 'w'); |
|||
outFn = function(str) { |
|||
fs.writeSync(fd, str, 'utf8'); |
|||
}; |
|||
process.on('exit', function() { |
|||
fs.closeSync(fd); |
|||
}); |
|||
} else { |
|||
outFn = function(str) { |
|||
process.stdout.write(str); |
|||
}; |
|||
} |
|||
|
|||
// Process the rest of the arguments as paths to lint, ignoring any unknown
|
|||
// flags
|
|||
for (i = 2; i < process.argv.length; ++i) { |
|||
if (process.argv[i][0] === '-') { |
|||
switch (process.argv[i]) { |
|||
case '-f': // Skip format name
|
|||
case '-o': // Skip filename
|
|||
case '-j': // Skip parallel job count number
|
|||
++i; |
|||
break; |
|||
} |
|||
continue; |
|||
} |
|||
paths.push(process.argv[i]); |
|||
} |
|||
|
|||
if (paths.length === 0) |
|||
return; |
|||
totalPaths = paths.length; |
|||
|
|||
if (showProgress) { |
|||
// Start the progress display update timer when the first worker is ready
|
|||
cluster.once('online', function(worker) { |
|||
startTime = process.hrtime(); |
|||
setInterval(printProgress, 1000).unref(); |
|||
printProgress(); |
|||
}); |
|||
} |
|||
|
|||
cluster.on('online', function(worker) { |
|||
// Configure worker and give it some initial work to do
|
|||
worker.send(workerConfig); |
|||
sendWork(worker); |
|||
}); |
|||
|
|||
cluster.on('message', function(worker, results) { |
|||
if (typeof results !== 'number') { |
|||
// The worker sent us results that are not all successes
|
|||
if (!workerConfig.sendAll) |
|||
failures += results.length; |
|||
outFn(formatter(results) + '\r\n'); |
|||
printProgress(); |
|||
} else { |
|||
successes += results; |
|||
} |
|||
// Try to give the worker more work to do
|
|||
sendWork(worker); |
|||
}); |
|||
|
|||
process.on('exit', function() { |
|||
if (showProgress) { |
|||
curPath = 'Done'; |
|||
printProgress(); |
|||
outFn('\r\n'); |
|||
} |
|||
process.exit(failures ? 1 : 0); |
|||
}); |
|||
|
|||
for (i = 0; i < numCPUs; ++i) |
|||
cluster.fork(); |
|||
|
|||
function sendWork(worker) { |
|||
if (!files || !files.length) { |
|||
// We either just started or we have no more files to lint for the current
|
|||
// path. Find the next path that has some files to be linted.
|
|||
while (paths.length) { |
|||
var dir = paths.shift(); |
|||
curPath = dir; |
|||
if (dir.indexOf('/') > 0) |
|||
dir = path.join(cwd, dir); |
|||
const patterns = cli.resolveFileGlobPatterns([dir]); |
|||
dir = path.resolve(patterns[0]); |
|||
files = glob.sync(dir, globOptions); |
|||
if (files.length) |
|||
break; |
|||
} |
|||
if ((!files || !files.length) && !paths.length) { |
|||
// We exhausted all input paths and thus have nothing left to do, so end
|
|||
// the worker
|
|||
return worker.disconnect(); |
|||
} |
|||
} |
|||
// Give the worker an equal portion of the work left for the current path,
|
|||
// but not exceeding a maximum file count in order to help keep *all*
|
|||
// workers busy most of the time instead of only a minority doing most of
|
|||
// the work.
|
|||
const sliceLen = Math.min(maxWorkload, Math.ceil(files.length / numCPUs)); |
|||
var slice; |
|||
if (sliceLen === files.length) { |
|||
// Micro-ptimization to avoid splicing to an empty array
|
|||
slice = files; |
|||
files = null; |
|||
} else { |
|||
slice = files.splice(0, sliceLen); |
|||
} |
|||
worker.send(slice); |
|||
} |
|||
|
|||
function printProgress() { |
|||
if (!showProgress) |
|||
return; |
|||
|
|||
// Clear line
|
|||
outFn('\r' + ' '.repeat(lastLineLen) + '\r'); |
|||
|
|||
// Calculate and format the data for displaying
|
|||
const elapsed = process.hrtime(startTime)[0]; |
|||
const mins = padString(Math.floor(elapsed / 60), 2, '0'); |
|||
const secs = padString(elapsed % 60, 2, '0'); |
|||
const passed = padString(successes, 6, ' '); |
|||
const failed = padString(failures, 6, ' '); |
|||
var pct = Math.ceil(((totalPaths - paths.length) / totalPaths) * 100); |
|||
pct = padString(pct, 3, ' '); |
|||
|
|||
var line = `[${mins}:${secs}|%${pct}|+${passed}|-${failed}]: ${curPath}`; |
|||
|
|||
// Truncate line like cpplint does in case it gets too long
|
|||
if (line.length > 75) |
|||
line = line.slice(0, 75) + '...'; |
|||
|
|||
// Store the line length so we know how much to erase the next time around
|
|||
lastLineLen = line.length; |
|||
|
|||
outFn(line); |
|||
} |
|||
|
|||
function padString(str, len, chr) { |
|||
str = '' + str; |
|||
if (str.length >= len) |
|||
return str; |
|||
return chr.repeat(len - str.length) + str; |
|||
} |
|||
} else { |
|||
// Worker
|
|||
|
|||
var config = {}; |
|||
process.on('message', function(files) { |
|||
if (files instanceof Array) { |
|||
// Lint some files
|
|||
const report = cli.executeOnFiles(files); |
|||
if (config.sendAll) { |
|||
// Return both success and error results
|
|||
|
|||
const results = report.results; |
|||
// Silence warnings for files with no errors while keeping the "ok"
|
|||
// status
|
|||
if (report.warningCount > 0) { |
|||
for (var i = 0; i < results.length; ++i) { |
|||
const result = results[i]; |
|||
if (result.errorCount === 0 && result.warningCount > 0) { |
|||
result.warningCount = 0; |
|||
result.messages = []; |
|||
} |
|||
} |
|||
} |
|||
process.send(results); |
|||
} else if (report.errorCount === 0) { |
|||
// No errors, return number of successful lint operations
|
|||
process.send(files.length); |
|||
} else { |
|||
// One or more errors, return the error results only
|
|||
process.send(CLIEngine.getErrorResults(report.results)); |
|||
} |
|||
} else if (typeof files === 'object') { |
|||
// The master process is actually sending us our configuration and not a
|
|||
// list of files to lint
|
|||
config = files; |
|||
} |
|||
}); |
|||
} |
|||
Loading…
Reference in new issue