Add text miner tool for test262 logs
This commit is contained in:
Anthony Van de Gejuchte
parent
b57f6c3031
commit
7450604677
375
tools/ecmaTestProcessor.js
Normal file
375
tools/ecmaTestProcessor.js
Normal file
|
|
@ -0,0 +1,375 @@
|
||||||
|
var fs = require("fs");
|
||||||
|
|
||||||
|
var async = require("async");
|
||||||
|
var yargs = require("yargs");
|
||||||
|
|
||||||
|
var OK = 0;
|
||||||
|
var FAIL = 1;
|
||||||
|
|
||||||
|
var MESSAGE = /^(=== )?[^\s]* (passed|failed|was expected to fail) in (non-)?strict mode( as expected|, but didn't)?( ===)?$/;
|
||||||
|
var SELF_ERROR = /A minified script failed to run after being minified\. Please report node version, test and error to the maintainer of the minifier tool\./;
|
||||||
|
var BOTH_ERROR = /A minified script failed to run both minified and unminified\. This is likely an invalid js file/;
|
||||||
|
var SELF_ERROR_FILTER = function(input) {
|
||||||
|
return SELF_ERROR.test(input)
|
||||||
|
};
|
||||||
|
var BOTH_ERROR_FILTER = function(input) {
|
||||||
|
return BOTH_ERROR.test(input)
|
||||||
|
};
|
||||||
|
|
||||||
|
var roundToPercentage = function(input) {
|
||||||
|
return Math.round(input * 10000) / 100;
|
||||||
|
};
|
||||||
|
|
||||||
|
var isNodeOnlyError = function(summary, output) {
|
||||||
|
// Errors caused when failing both running minified and unminified are very likely node caused
|
||||||
|
if (output.filter(BOTH_ERROR_FILTER).length > 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
var isUglifyOnlyError = function(summary, output) {
|
||||||
|
// Errors caused when being minified, unminified ran fine
|
||||||
|
if (output.filter(SELF_ERROR_FILTER).length > 0) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
var ARGS = yargs.usage(
|
||||||
|
"$0 logs.txt -o output.md\n\n" +
|
||||||
|
"Processes test262 logs and convert it to more human readable markup files.\n"
|
||||||
|
)
|
||||||
|
.describe("h", "Get help")
|
||||||
|
.describe("o", "Output file")
|
||||||
|
|
||||||
|
.describe("debug", "Prints out very large json debug file")
|
||||||
|
.describe("debugFormat", "Prints out structure of json debug file")
|
||||||
|
.describe("es5", "Notifies that tests are run on test262 es5 mode")
|
||||||
|
|
||||||
|
.alias("h", "help")
|
||||||
|
.alias("o", "output")
|
||||||
|
|
||||||
|
.boolean("es5")
|
||||||
|
|
||||||
|
.wrap(80)
|
||||||
|
|
||||||
|
.argv;
|
||||||
|
|
||||||
|
if (ARGS.h || ARGS.help) {
|
||||||
|
console.log(yargs.help());
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ARGS.debugFormat) {
|
||||||
|
console.log("Root:")
|
||||||
|
console.log(" logs: <Object LogObject> See LogObject");
|
||||||
|
console.log(" unitTestUrl: <String> Base url to test suite, used as link template for tests");
|
||||||
|
console.log("");
|
||||||
|
console.log("LogObject:");
|
||||||
|
console.log(" errors: <Int> Total errors in this LogObjects and childs");
|
||||||
|
console.log(" errors_local: <Int> Total errors in this LogObject alone");
|
||||||
|
console.log(" node_error: <Int> Total errors caused by Node in this LogObject and childs");
|
||||||
|
console.log(" node_error_local: <Int> Total errors caused by Node in this LogObject alone");
|
||||||
|
console.log(" self_error: <Int> Total errors caused by Uglify in this LogObject and childs");
|
||||||
|
console.log(" self_error_local: <Int> Total errors caused by Uglify in this LogObject alone");
|
||||||
|
console.log(" testsCount: <Int> Total number of tests");
|
||||||
|
console.log(" testsCount_local: <Int> Total number of local tests");
|
||||||
|
console.log(" dir: <Object LogObject> Contains LogObject childs");
|
||||||
|
console.log(" tests: <Array TestArrayList> See TestObject");
|
||||||
|
console.log("");
|
||||||
|
console.log("TestArrayList:");
|
||||||
|
console.log(" {for every array element}: <Array TestArray>");
|
||||||
|
console.log("");
|
||||||
|
console.log("TestArray:");
|
||||||
|
console.log(" 0: <Int> OK or FAIL to indicate that the test has succeed");
|
||||||
|
console.log(" 1: <String> Summary message");
|
||||||
|
console.log(" 2: <Object Error | undefined> Error Object containing details if any necessary");
|
||||||
|
console.log(" 3: <Int> Line of test in log");
|
||||||
|
console.log("");
|
||||||
|
console.log("Error:");
|
||||||
|
console.log(" logs: <Array String> stdout of the test failure");
|
||||||
|
console.log(" node_error: <Bool> True if likelyhood that node screwed up is very high, Indicated use only.");
|
||||||
|
console.log(" self_error: <Bool> True if likelyhood that Uglify screwed up is very high. Indicated use only.");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ARGS._.length === 0) {
|
||||||
|
console.log("Please provide logs\nYou can add --help to the command to get help");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
var data;
|
||||||
|
|
||||||
|
// Read files
|
||||||
|
async.mapLimit(ARGS._, 1, function(file, callback) {
|
||||||
|
var content = fs.readFileSync(file, {
|
||||||
|
encoding: 'utf8'
|
||||||
|
}).split(/\s*(?:\r(?:\n)?|\n)/);
|
||||||
|
|
||||||
|
var output = [];
|
||||||
|
var errors = 0;
|
||||||
|
|
||||||
|
var tmp;
|
||||||
|
for (var i = 0; i < content.length; i++) {
|
||||||
|
// Make sure the message is done in the expected format
|
||||||
|
if (!MESSAGE.test(content[i])) {
|
||||||
|
if (/^\s*$/.test(content[i])) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Unexpected message at line " + i + ":");
|
||||||
|
console.log(content[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Push test formatted to the output
|
||||||
|
if (content[i].indexOf("=") === -1) {
|
||||||
|
output.push([OK, content[i], undefined, i]);
|
||||||
|
} else if (/^===/.test(content[i])) {
|
||||||
|
tmp = i;
|
||||||
|
// Find first line without errors
|
||||||
|
do {
|
||||||
|
errors++;
|
||||||
|
i++;
|
||||||
|
while (!/^.*===$/.test(content[i])) {
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
} while (!MESSAGE.test(content[i + 1]));
|
||||||
|
|
||||||
|
var error = {
|
||||||
|
logs: content.slice(tmp + 1, i),
|
||||||
|
};
|
||||||
|
var summary = content[tmp].substr(4, content[tmp].length - 8)
|
||||||
|
error.node_error = isNodeOnlyError(summary, error.logs);
|
||||||
|
error.self_error = isUglifyOnlyError(summary, error.logs);
|
||||||
|
|
||||||
|
output.push([
|
||||||
|
FAIL,
|
||||||
|
summary,
|
||||||
|
error,
|
||||||
|
i
|
||||||
|
]);
|
||||||
|
} else {
|
||||||
|
console.log("Unknown message at line " + i + ":");
|
||||||
|
console.log(content[i]);
|
||||||
|
console.log("Error message started at line " + tmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats in console
|
||||||
|
console.log(content.length + " lines of logs");
|
||||||
|
console.log(output.length + " tests found");
|
||||||
|
console.log(errors + " errors found");
|
||||||
|
|
||||||
|
if (errors > 0) {
|
||||||
|
console.log("Failure rate: " + roundToPercentage(errors / output.length) + "%");
|
||||||
|
} else {
|
||||||
|
console.log("Failure rate: 0%");
|
||||||
|
}
|
||||||
|
|
||||||
|
callback(undefined, output);
|
||||||
|
}, function(err, results) {
|
||||||
|
// Check if everything succeed without errors
|
||||||
|
if (err) {
|
||||||
|
console.log(err);
|
||||||
|
process.exit()
|
||||||
|
}
|
||||||
|
|
||||||
|
var getNewLogsObject = function() {
|
||||||
|
return {
|
||||||
|
errors: 0,
|
||||||
|
errors_local: 0,
|
||||||
|
node_error: 0,
|
||||||
|
node_error_local: 0,
|
||||||
|
self_error: 0,
|
||||||
|
self_error_local: 0,
|
||||||
|
testsCount: 0,
|
||||||
|
testsCount_local: 0,
|
||||||
|
dir: {},
|
||||||
|
tests: {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge log results
|
||||||
|
data = {
|
||||||
|
logs: getNewLogsObject()
|
||||||
|
};
|
||||||
|
|
||||||
|
if (ARGS.es5) {
|
||||||
|
data.unitTestUrl = "https://github.com/tc39/test262/blob/es5-tests/test/suite/";
|
||||||
|
} else {
|
||||||
|
data.unitTestUrl = "https://github.com/tc39/test262/blob/master/test/";
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var i = 0; i < results.length; i++) {
|
||||||
|
for (var j = 0; j < results[i].length; j++) {
|
||||||
|
var status = results[i][j][0];
|
||||||
|
var test = results[i][j][1];
|
||||||
|
var error = results[i][j][2];
|
||||||
|
var line = results[i][j][3];
|
||||||
|
var node_error = status === FAIL && error.node_error ? 1 : 0;
|
||||||
|
var self_error = status === FAIL && error.self_error ? 1 : 0;
|
||||||
|
|
||||||
|
if (typeof test !== "string") {
|
||||||
|
console.log("Error!");
|
||||||
|
console.log(test);
|
||||||
|
}
|
||||||
|
var path = test.split(" ")[0].split(/\/|\\/);
|
||||||
|
|
||||||
|
var pos = data.logs;
|
||||||
|
for (var k = 0; k < path.length - 1; k++) {
|
||||||
|
var dir = path[k] + "/";
|
||||||
|
|
||||||
|
pos.errors += status;
|
||||||
|
pos.node_error += node_error;
|
||||||
|
pos.self_error += self_error;
|
||||||
|
pos.testsCount++;
|
||||||
|
|
||||||
|
if (!(dir in pos.dir)) {
|
||||||
|
pos.dir[dir] = getNewLogsObject();
|
||||||
|
}
|
||||||
|
pos = pos.dir[dir];
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pos.errors += status;
|
||||||
|
pos.errors_local += status;
|
||||||
|
pos.node_error += node_error;
|
||||||
|
pos.node_error_local += node_error;
|
||||||
|
pos.self_error += self_error;
|
||||||
|
pos.self_error_local += self_error;
|
||||||
|
pos.testsCount++;
|
||||||
|
pos.testsCount_local++;
|
||||||
|
|
||||||
|
if (path[path.length - 1] + ".js" in pos.tests) {
|
||||||
|
pos.tests[path[path.length - 1] + ".js"].push(results[i][j]);
|
||||||
|
} else {
|
||||||
|
pos.tests[path[path.length - 1] + ".js"] = [results[i][j]];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Produce debug output
|
||||||
|
if (ARGS.debug) {
|
||||||
|
fs.writeFile(ARGS.debug, JSON.stringify(data));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Intro
|
||||||
|
var mdFile = "# Test262 results\n\n";
|
||||||
|
mdFile += "All tests are passed as js code to UglifyJS, and run through the minify method. " +
|
||||||
|
"The minified code will then be run by node. If the minified code errors, " +
|
||||||
|
"the original code will be run through node and the results will be compared. " +
|
||||||
|
"Ultimately, the test runner decides if a test fail. Some tests may be expected to fail.\n\n";
|
||||||
|
mdFile += "Tests only working without UglifyJS are marked in bold.\n\n";
|
||||||
|
mdFile += "Most tests are tested in strict and non-strict mode. " +
|
||||||
|
"While failures may be listed twice, the test will only be shown once.\n\n";
|
||||||
|
mdFile += "Note: there is a bug that prevents links from being visible, " +
|
||||||
|
"see https://github.com/vmg/redcarpet/issues/443\n\n";
|
||||||
|
|
||||||
|
// Show tests, failures, percentage and used tools
|
||||||
|
mdFile += "## Global stats\n\n";
|
||||||
|
mdFile += "- Found " + data.logs.testsCount + " executed tests\n";
|
||||||
|
mdFile += "- " + data.logs.errors + " tests failed (" + roundToPercentage(data.logs.errors / data.logs.testsCount) + "%)\n";
|
||||||
|
mdFile += "- " + data.logs.node_error + " tests failed becuase of Node (" + roundToPercentage(data.logs.node_error / data.logs.testsCount) + "%)\n"
|
||||||
|
mdFile += "- " + data.logs.self_error + " tests failed because of UglifyJS (" + roundToPercentage(data.logs.self_error / data.logs.testsCount) + "%)\n";
|
||||||
|
mdFile += "\n";
|
||||||
|
|
||||||
|
function getSummary(obj) {
|
||||||
|
var subDirectoryCount = Object.keys(obj.dir).length;
|
||||||
|
|
||||||
|
// Templates:
|
||||||
|
// 1 failures whereof 1 caused by UglifyJS (1 failure over 10 local tests with 1 caused by UglifyJS, with 10 subdirectories)
|
||||||
|
// 0 failures (no local tests, 10 subdirectories)
|
||||||
|
return obj.errors + " failures over " + obj.testsCount + " tests" + (
|
||||||
|
obj.self_error > 0 ? " whereof " + obj.self_error + " caused by UglifyJS" : ""
|
||||||
|
) + " (" + (
|
||||||
|
obj.testsCount_local > 0 ? (
|
||||||
|
obj.errors_local > 0 ? obj.errors_local + " failures over " + obj.testsCount_local + " local tests" + (
|
||||||
|
obj.self_error_local > 0 ? " with " + obj.self_error_local + " caused by UglifyJS" : ""
|
||||||
|
) : "No local failures"
|
||||||
|
) : "No local tests"
|
||||||
|
) + ", with " + (subDirectoryCount > 0 ? subDirectoryCount : "no") + " subdirectories)"
|
||||||
|
}
|
||||||
|
|
||||||
|
function printResult(obj, level) {
|
||||||
|
var prefix = "";
|
||||||
|
|
||||||
|
for (var c = 0; c < level; c++) {
|
||||||
|
prefix += " ";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Print current directory
|
||||||
|
var dirs = "";
|
||||||
|
for (var i in obj.dir) {
|
||||||
|
dirs += prefix + "- [" + (obj.dir[i].errors === 0 ? "x" : " ") +"] `" + i + "` "
|
||||||
|
+ getSummary(obj.dir[i]) + "\n";
|
||||||
|
|
||||||
|
if (obj.dir[i].errors > 0) {
|
||||||
|
dirs += printResult(obj.dir[i], level + 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var fails = "";
|
||||||
|
for (var j in obj.tests) {
|
||||||
|
var node_error_count = 0;
|
||||||
|
var self_error_count = 0;
|
||||||
|
var fail_count = 0;
|
||||||
|
var total_count = 0;
|
||||||
|
|
||||||
|
for (var k = 0; k < obj.tests[j].length; k++) {
|
||||||
|
total_count++;
|
||||||
|
|
||||||
|
if (obj.tests[j][k][0] !== FAIL) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
fail_count++;
|
||||||
|
if (obj.tests[j][k][2].node_error) {
|
||||||
|
node_error_count++;
|
||||||
|
}
|
||||||
|
if (obj.tests[j][k][2].self_error) {
|
||||||
|
self_error_count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fail_count > 0) {
|
||||||
|
var counter = " - failed: " + fail_count + "/" + total_count;
|
||||||
|
var error_style = self_error_count > 0 ? "**" : (node_error_count > 0 ? "~~" : "");
|
||||||
|
|
||||||
|
// Add formatted link
|
||||||
|
var test = obj.tests[j][0][1]
|
||||||
|
.replace(/\\/g, "\\\\")
|
||||||
|
.replace(/-/g, "\\-");
|
||||||
|
var links = " [\\[test\\]](" + data.unitTestUrl +
|
||||||
|
obj.tests[j][0][1]
|
||||||
|
.split(" ")[0]
|
||||||
|
.replace(/\\/g, "/") + ".js)";
|
||||||
|
|
||||||
|
var cause = "";
|
||||||
|
if (self_error_count > 0 && node_error_count === 0) {
|
||||||
|
cause = " (Caused by UglifyJS)";
|
||||||
|
} else if (self_error_count === 0 && node_error_count > 0) {
|
||||||
|
cause = " (Caused by Node)";
|
||||||
|
} else if (self_error_count > 0 && node_error_count > 0) {
|
||||||
|
cause = " (Some caused by Node and some by UglifyJS)";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add error summary
|
||||||
|
fails += prefix + "- *" + error_style + test + error_style + "*" +
|
||||||
|
links + cause + counter + "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fails + dirs;
|
||||||
|
}
|
||||||
|
|
||||||
|
mdFile += "## Error tree\n\n";
|
||||||
|
mdFile += getSummary(data.logs) + "\n\n";
|
||||||
|
mdFile += printResult(data.logs, 0);
|
||||||
|
|
||||||
|
if (ARGS.o) {
|
||||||
|
fs.writeFile(ARGS.o,mdFile);
|
||||||
|
} else {
|
||||||
|
console.log(mdFile);
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user