583 lines
15 KiB
JavaScript
583 lines
15 KiB
JavaScript
|
|
"use strict";
|
|
|
|
var INFER = '$';
|
|
var GIVEN = '#';
|
|
|
|
var EXPECTED_MAX_NODES_PER_NONOBFUSACATED_LINE = 25;
|
|
var MAX_RATIO_SHORT_NAMES = 0.45;
|
|
var NUM_NUMBERED_LOCALS = 5;
|
|
|
|
function isMinified(toplevel, code, file){
|
|
var numLines = code.split(/\r\n|\r|\n/).length;
|
|
var numStatements = 0;
|
|
var numNames = 0;
|
|
var numShortNames = 0;
|
|
var numNumberedNames = 0;
|
|
|
|
toplevel.walk(new TreeWalker(function(node, descend){
|
|
numStatements++;
|
|
if (node instanceof AST_Symbol && !(node instanceof AST_This)) {
|
|
numNames++;
|
|
if (node.name.length <= 2 && node.name != "el" && node.name != "$") {
|
|
numShortNames++;
|
|
}
|
|
if (node.name.length >= 2 && node.name[0] == '_') {
|
|
var c2 = node.name[1];
|
|
if (c2 >= '0' && c2 <= '9') ++numNumberedNames;
|
|
}
|
|
}
|
|
}));
|
|
|
|
return (EXPECTED_MAX_NODES_PER_NONOBFUSACATED_LINE * numLines <= numStatements) ||
|
|
(numShortNames > numNames * MAX_RATIO_SHORT_NAMES) ||
|
|
numNumberedNames == numNames ||
|
|
numNumberedNames >= NUM_NUMBERED_LOCALS;
|
|
}
|
|
|
|
function replaceMangled(code, file) {
|
|
var toplevel;
|
|
try {
|
|
toplevel = parseFile(code, file);
|
|
} catch (e) {
|
|
console.warn("Cannot parse file: '%s'", file);
|
|
return null;
|
|
}
|
|
|
|
extendAst(toplevel);
|
|
|
|
var feature_outputter = new FeatureJsonOutputter();
|
|
generateAstFeatures(toplevel, feature_outputter);
|
|
generateFnamesFeatures(toplevel, feature_outputter);
|
|
|
|
//feature_outputter.string_map defines what id is assigment to each node in the final output
|
|
//therefore to assign same ids, we need to first populate by running feature extraction
|
|
var stream = OutputStream({beautify : true, replace_mangled_map : feature_outputter.string_map});
|
|
toplevel.print(stream);
|
|
return stream.toString();
|
|
}
|
|
|
|
function extractFeatures(code, file, print_ast, features, skip_minified) {
|
|
var toplevel;
|
|
try {
|
|
toplevel = parseFile(code, file);
|
|
} catch (e) {
|
|
console.warn("Cannot parse file: '%s'", file);
|
|
return null;
|
|
}
|
|
|
|
extendAst(toplevel);
|
|
|
|
if (print_ast) {
|
|
return printAst(toplevel);
|
|
}
|
|
|
|
if (skip_minified && isMinified(toplevel, code, file)){
|
|
console.warn("Skipping minified file: '%s'", file);
|
|
return null;
|
|
}
|
|
|
|
var feature_outputter = new FeatureJsonOutputter();
|
|
feature_outputter.openElem();
|
|
feature_outputter.openArray("query");
|
|
|
|
if (features.indexOf("ASTREL") != -1) {
|
|
generateAstFeatures(toplevel, feature_outputter);
|
|
}
|
|
|
|
if (features.indexOf("FNAMES") != -1) {
|
|
generateFnamesFeatures(toplevel, feature_outputter);
|
|
}
|
|
|
|
if (features.indexOf("FSCOPE") != -1) {
|
|
generateFscopeConstraints(toplevel, feature_outputter);
|
|
}
|
|
|
|
feature_outputter.closeArray();
|
|
feature_outputter.dumpSymbols();
|
|
feature_outputter.closeElem();
|
|
|
|
return feature_outputter.output;
|
|
}
|
|
|
|
/* -----[ functions ]----- */
|
|
|
|
function nodeToString(node) {
|
|
if (node == null) return null;
|
|
|
|
if (node instanceof AST_Symbol){
|
|
if (node instanceof AST_This ){
|
|
return GIVEN + node.name;
|
|
}
|
|
// AST_Symbol::unmangleable() returns true if this symbol cannot be renamed (it's either global, undeclared, or defined in scope where eval or with are in use.
|
|
if (node.unmangleable()){
|
|
return GIVEN + node.name;
|
|
}
|
|
return INFER + node.definition().id + "-" + node.name;
|
|
} else if (node instanceof AST_Constant){
|
|
return GIVEN + "!" + nodeType(node) + "!" + String(node.value).slice(0,64);
|
|
} else if (node instanceof AST_Sub){
|
|
//x[1], x -> expression, 1 -> property
|
|
return nodeToString(node.expression) + "[]";
|
|
} else if (node instanceof AST_PropAccess){
|
|
return GIVEN + node.property;
|
|
} else if (node instanceof AST_Defun) {
|
|
//function foo(...) { ... }
|
|
return nodeToString(node.name);
|
|
} else if (node instanceof AST_VarDef){
|
|
// var x = function () { ... }
|
|
return nodeToString(node.name);
|
|
} else if (node instanceof AST_Assign){
|
|
//x = function () { ... }
|
|
return nodeToString(node.left);
|
|
} else if (node instanceof AST_ObjectProperty){
|
|
// { "x" : function () { ... } }
|
|
return GIVEN + node.key;
|
|
} else if (node instanceof AST_Call){
|
|
//x.foo( function () { ... } )
|
|
//foo( function () { ... } )
|
|
return nodeToString(node.expression);
|
|
} else if (node instanceof AST_Lambda) {
|
|
if (node.parent instanceof AST_Call){
|
|
//'node.parent.expression != node' as lambda can call itself
|
|
return (node.parent.expression == node) ? null : nodeToString(node.parent.expression) + "(" + node.child_id + ")";
|
|
}
|
|
return nodeToString(node.parent);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function nodeType(node) {
|
|
if (node instanceof AST_Binary || node instanceof AST_Unary) {
|
|
return node.__proto__.TYPE + node.operator;
|
|
} else if (node instanceof AST_Boolean) {
|
|
return "Bool";
|
|
} else if (node instanceof AST_Atom && !(node instanceof AST_Constant)) {
|
|
//atoms are special constant values as Nan, Undefined, Infinity,..
|
|
return "Atom";
|
|
}
|
|
|
|
return node.__proto__.TYPE;
|
|
}
|
|
|
|
function pathToStringFw(path, start){
|
|
var res = "";
|
|
for (var i = start; i < path.length - 1; i++) {
|
|
res += nodeType(path[i]);
|
|
res += "[" + path[i+1].child_id + "]";
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
function pathToStringBw(path, start){
|
|
var res = "";
|
|
for (var i = path.length - 2; i >= start; i--) {
|
|
res += nodeType(path[i]);
|
|
res += "[" + path[i+1].child_id + "]";
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
function printAst(toplevel){
|
|
var output = "";
|
|
|
|
var walker = new TreeWalker(function(node){
|
|
output += string_template(" node{id} [label=\"{label}\"];\n", {
|
|
id: node.id,
|
|
label: nodeType(node)
|
|
});
|
|
|
|
if (walker.parent() != null) {
|
|
output += string_template(" node{id1} -> node{id2} [weight=1];\n", {
|
|
id1: walker.parent().id,
|
|
id2: node.id
|
|
});
|
|
}
|
|
});
|
|
|
|
output += "digraph AST {\n";
|
|
toplevel.walk(walker);
|
|
output += "}\n";
|
|
return output;
|
|
}
|
|
|
|
function generateAstFeatures(toplevel, feature_outputter) {
|
|
var walker = new TreeWalker(function(node){
|
|
// console.log(nodeType(node) + " - " + nodeToString(node));
|
|
var paths = this.node_finder.find(node);
|
|
for (var i = 0; i < paths.length; i++) {
|
|
var path1 = paths[i];
|
|
var node1 = path1[path1.length - 1];
|
|
|
|
for (var j = i + 1; j < paths.length; j++) {
|
|
var common_prefix_len = 0;
|
|
var path2 = paths[j];
|
|
var node2 = path2[path2.length - 1];
|
|
|
|
//determine common prefix to be skipped
|
|
while(common_prefix_len < path1.length && common_prefix_len < path2.length
|
|
&& path1[common_prefix_len] === path2[common_prefix_len]){
|
|
common_prefix_len++;
|
|
}
|
|
|
|
if (common_prefix_len == 0) {
|
|
throw "common prefix not greater than 0!";
|
|
}
|
|
|
|
feature_outputter.addFeature(
|
|
nodeToString(node1),
|
|
nodeToString(node2),
|
|
//pathToStringBw(path1, common_prefix_len) + ":" + nodeType(path1[common_prefix_len - 1]) + ":" + pathToStringFw(path2, common_prefix_len)
|
|
(path2.length != common_prefix_len)
|
|
? pathToStringBw(path1, common_prefix_len) + ":" + pathToStringFw(path2, common_prefix_len - 1)
|
|
: pathToStringBw(path2, common_prefix_len) + ":" + pathToStringFw(path1, common_prefix_len - 1)
|
|
);
|
|
|
|
}
|
|
}
|
|
});
|
|
|
|
walker.node_finder = new NodePathFinder(3, function(node) {
|
|
return (node instanceof AST_Symbol || node instanceof AST_Constant || node instanceof AST_PropAccess);
|
|
});
|
|
|
|
toplevel.walk(walker);
|
|
}
|
|
|
|
function addFeatures(lhss, lhs_label, rhs, rhs_label, feature_outputter){
|
|
var prefix = "";
|
|
for (var i = lhss.length - 1; i >= 0; i--) {
|
|
prefix += lhs_label;
|
|
feature_outputter.addFeature(lhss[i], rhs, prefix + rhs_label);
|
|
}
|
|
}
|
|
|
|
function addScopeConstraints(node, toplevel, feature_outputter){
|
|
feature_outputter.beginScope();
|
|
var name = nodeToString(node);
|
|
if (name != null)
|
|
feature_outputter.addToScope(name);
|
|
|
|
for (var i = 0; i < node.enclosed.length; i++){
|
|
feature_outputter.addToScope(nodeToString(node.enclosed[i].orig[0]));
|
|
}
|
|
|
|
node.variables.each(function(symbol){
|
|
feature_outputter.addToScope(nodeToString(symbol.orig[0]));
|
|
});
|
|
|
|
toplevel.globals.each(function(symbol){
|
|
feature_outputter.addToScope(nodeToString(symbol.orig[0]));
|
|
});
|
|
|
|
feature_outputter.endScope();
|
|
}
|
|
|
|
|
|
function generateFscopeConstraints(toplevel, feature_outputter){
|
|
addScopeConstraints(toplevel, toplevel, feature_outputter);
|
|
toplevel.walk(new TreeWalker(function(node) {
|
|
if (node instanceof AST_Defun || node instanceof AST_Lambda) {
|
|
addScopeConstraints(node, toplevel, feature_outputter);
|
|
}
|
|
}));
|
|
}
|
|
|
|
function generateFnamesFeatures(toplevel, feature_outputter){
|
|
var outer_funcs = [];
|
|
|
|
toplevel.walk(new TreeWalker(function(node, descend){
|
|
|
|
if ((node instanceof AST_Defun || node instanceof AST_Lambda) && nodeToString(node) != null) {
|
|
var name = nodeToString(node);
|
|
|
|
for (var i = 0; i < node.argnames.length; i++) {
|
|
addFeatures([name], "FN", nodeToString(node.argnames[i]), "PAR", feature_outputter);
|
|
}
|
|
|
|
outer_funcs.push(name);
|
|
descend(); //traverse childs
|
|
outer_funcs.pop();
|
|
|
|
return true; //do not traverse childs again
|
|
}
|
|
|
|
if (node instanceof AST_New) {
|
|
addFeatures(outer_funcs, "FN", nodeToString(node), "NEW", feature_outputter);
|
|
} else if (node instanceof AST_Call) {
|
|
addFeatures(outer_funcs, "FN", nodeToString(node), "CALL", feature_outputter);
|
|
} else if (node instanceof AST_Constant){
|
|
addFeatures(outer_funcs, "FN", nodeToString(node), nodeType(node).toUpperCase(), feature_outputter);
|
|
} else if (node instanceof AST_VarDef){
|
|
addFeatures(outer_funcs, "FN", nodeToString(node.name), "DECL", feature_outputter);
|
|
} else if (node instanceof AST_Dot && !(node.parent instanceof AST_Call)) {
|
|
addFeatures(outer_funcs, "FN", nodeToString(node), "PROP", feature_outputter);
|
|
} else if (node instanceof AST_Return && nodeToString(node.value) != null) {
|
|
addFeatures(outer_funcs, "FN", nodeToString(node.value), "RETURN", feature_outputter);
|
|
}
|
|
}));
|
|
}
|
|
|
|
/* -----[ NodePathFinder ]----- */
|
|
|
|
function NodePathFinder(max_depth, filter) {
|
|
this.max_depth = max_depth;
|
|
this.paths = [];
|
|
this.filter = filter;
|
|
}
|
|
|
|
NodePathFinder.prototype = new TreeWalker(function(node, descend){
|
|
if (this.stack.length > this.max_depth || node instanceof AST_Defun){
|
|
return true;
|
|
}
|
|
|
|
//enforce in-order traversal
|
|
//otherwise we get for "x.foo()" feature foo - x instead of x - foo as x is a parent of foo in the AST
|
|
descend();
|
|
|
|
if (this.filter(node)) {
|
|
this.paths.push(this.stack.slice(0));
|
|
}
|
|
|
|
return true;
|
|
});
|
|
|
|
NodePathFinder.prototype.find = function(node) {
|
|
this.root = node;
|
|
this.paths = [];
|
|
node.walk(this);
|
|
return this.paths;
|
|
};
|
|
|
|
/* ---[ JsonOutputter ]--- */
|
|
|
|
function FeatureJsonOutputter() {
|
|
this.string_map = new StringMap(false);
|
|
this.first_element = true;
|
|
this.output = "";
|
|
this.depth = 0;
|
|
this.pairs = {};
|
|
this.cur_scope = {};
|
|
}
|
|
|
|
FeatureJsonOutputter.prototype.indent = function() {
|
|
var res = "";
|
|
for (var i = 0; i < this.depth; i++) {
|
|
res += " ";
|
|
}
|
|
return res;
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.openElem = function() {
|
|
if (!this.first_element) {
|
|
this.output += ",";
|
|
}
|
|
this.output += "\n" + this.indent() + "{";
|
|
this.first_element = true;
|
|
this.depth++;
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.closeElem = function() {
|
|
this.depth--;
|
|
this.output += "}";
|
|
this.first_element = false;
|
|
};
|
|
|
|
|
|
FeatureJsonOutputter.prototype.openArray = function(name){
|
|
if (!this.first_element) {
|
|
this.output += ",";
|
|
}
|
|
this.output += "\n" + this.indent() + "\"" + name + "\":[";
|
|
this.first_element = true;
|
|
this.depth++;
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.closeArray = function(){
|
|
this.depth--;
|
|
this.output += "\n" + this.indent() + "]";
|
|
this.first_element = false;
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.visitFeature = function(a_id, b_id, name){
|
|
if (! (a_id in this.pairs) ) {
|
|
this.pairs[a_id] = [];
|
|
}
|
|
var visited = this.pairs[a_id];
|
|
|
|
if (visited.indexOf(b_id + "-" + name) >= 0) {
|
|
return true;
|
|
}
|
|
visited.push(b_id + "-" + name);
|
|
return false;
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.addFeature = function(a, b, name){
|
|
if (a == null || b == null){
|
|
return;
|
|
}
|
|
|
|
//do not add features between two fixed nodes
|
|
if (a[0] == GIVEN && b[0] == GIVEN) {
|
|
return;
|
|
}
|
|
|
|
var a_id = this.string_map.getId(a);
|
|
var b_id = this.string_map.getId(b);
|
|
|
|
if (a_id == b_id || this.visitFeature(a_id, b_id, name)){
|
|
return;
|
|
}
|
|
|
|
this.openElem();
|
|
|
|
this.output += '"a": ' + a_id + ",";
|
|
this.output += '\t"b": ' + b_id + ",";
|
|
this.output += '\t"f2": "' + name + '"';
|
|
|
|
this.closeElem();
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.addSymbol = function(key){
|
|
this.openElem();
|
|
|
|
this.output += '"v": ' + this.string_map.getId(key) + ",";
|
|
if (key[0] == INFER){
|
|
//${id}-{name}
|
|
this.output += '\t"inf": "' + escapeString(key.split("-")[1]) + '"';
|
|
} else {
|
|
//#{name}
|
|
this.output += '\t"giv": "' + escapeString(key.slice(1)) + '"';
|
|
}
|
|
|
|
this.closeElem();
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.dumpSymbols = function(){
|
|
this.openArray("assign");
|
|
|
|
// var keys = Object.keys( this.string_map.map );
|
|
var keys = this.string_map.keys;
|
|
for( var i = 0,length = keys.length; i < length; i++ ) {
|
|
this.addSymbol(keys[i]);
|
|
}
|
|
|
|
this.closeArray();
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.beginScope = function(){
|
|
this.cur_scope = {};
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.addToScope = function(a){
|
|
var a_id = this.string_map.getId(a);
|
|
this.cur_scope[a_id] = true;
|
|
};
|
|
|
|
FeatureJsonOutputter.prototype.endScope = function(){
|
|
//{"cn":"!=","n":[14,366,370,372,108,40,356]}
|
|
var keys = Object.keys(this.cur_scope);
|
|
if (keys.length <= 1) {
|
|
return;
|
|
}
|
|
|
|
this.openElem();
|
|
this.output += '"cn":"!=", "n":[';
|
|
|
|
this.output += keys[0];
|
|
for(var i = 1,length = keys.length; i < length; i++ ) {
|
|
this.output += ',';
|
|
this.output += keys[i];
|
|
}
|
|
|
|
this.output += "]";
|
|
this.closeElem();
|
|
};
|
|
|
|
/* -----[ StringMap ]----- */
|
|
|
|
function StringMap(nice_names) {
|
|
this.map = {};
|
|
this.current_id = 0;
|
|
this.nice_names = nice_names;
|
|
this.keys = [];
|
|
}
|
|
|
|
StringMap.prototype.getId = function(input){
|
|
if (input == null){
|
|
throw new Error("error null");
|
|
}
|
|
if (this.nice_names) return input;
|
|
|
|
//we add a special character in from to allow for keys such as "toString"
|
|
var escaped_input = "#" + input;
|
|
|
|
if (!(escaped_input in this.map)) {
|
|
this.map[escaped_input] = this.current_id;
|
|
|
|
//keep ordered map of keys for iterating later
|
|
this.keys.push(input);
|
|
this.current_id++;
|
|
}
|
|
|
|
return this.map[escaped_input];
|
|
};
|
|
|
|
/* ------------------------ */
|
|
|
|
function escapeString(input){
|
|
return encodeURIComponent(input);
|
|
}
|
|
|
|
function parseFile(code, file) {
|
|
var toplevel = parse(code, {
|
|
filename : file
|
|
});
|
|
toplevel.figure_out_scope();
|
|
return toplevel;
|
|
}
|
|
|
|
function FakeSymbolDef(name, id) {
|
|
this.name = name;
|
|
this.id = id;
|
|
};
|
|
|
|
function extendAst(root){
|
|
var current_id = 0;
|
|
|
|
var walker = new TreeWalker(function(node){
|
|
|
|
if (!node.hasOwnProperty("id")){
|
|
node.id = current_id;
|
|
current_id += 1;
|
|
}
|
|
if (!node.hasOwnProperty("parent")){
|
|
node.parent = walker.parent();
|
|
}
|
|
node.num_childs = 0;
|
|
node.child_id = 0;
|
|
if (walker.parent() !== undefined){
|
|
node.child_id = walker.parent().num_childs;
|
|
walker.parent().num_childs++;
|
|
}
|
|
|
|
if (node instanceof AST_Symbol) {
|
|
// if (node.definition() == null && node instanceof AST_This){
|
|
// var scope = node;
|
|
// while (!(scope instanceof AST_Lambda) && scope.parent != null){
|
|
// scope = scope.parent;
|
|
// }
|
|
// var name = nodeToString(scope);
|
|
// node.thedef = new FakeSymbolDef(name + "_this", scope.id);
|
|
// } else {
|
|
if (node.definition() != null) {
|
|
node.definition().id = current_id;
|
|
current_id++;
|
|
}
|
|
}
|
|
});
|
|
root.walk(walker);
|
|
} |