Skip to content

Commit

Permalink
improve AST tests & tools (#4873)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexlamsl
alexlamsl committed Apr 27, 2021
1 parent acf951a commit 97bd56b
Show file tree
Hide file tree
Showing 7 changed files with 136 additions and 74 deletions.
6 changes: 5 additions & 1 deletion .github/workflows/build.yml
View file
Expand Up @@ -8,7 +8,11 @@ jobs:
strategy:
fail-fast: false
matrix:
options: [ '-mb braces', '--ie8 -c', '-mc', '--toplevel -mc passes=3,pure_getters,unsafe' ]
options:
- '-mb braces'
- '--ie8 -c'
- '-mc'
- '--toplevel -mc passes=3,pure_getters,unsafe'
script:
- acorn.sh
- bootstrap.sh
Expand Down
26 changes: 20 additions & 6 deletions bin/uglifyjs
View file
Expand Up @@ -235,10 +235,11 @@ if (options.mangle && options.mangle.properties) {
});
}
}
if (output == "ast" || output == "spidermonkey") options.output = {
ast: true,
code: false,
};
if (/^ast|spidermonkey$/.test(output)) {
if (typeof options.output != "object") options.output = {};
options.output.ast = true;
options.output.code = false;
}
if (options.parse && (options.parse.acorn || options.parse.spidermonkey)
&& options.sourceMap && options.sourceMap.content == "inline") {
fatal("inline source map only works with built-in parser");
Expand All @@ -265,7 +266,9 @@ if (specified["self"]) {
paths = UglifyJS.FILES;
}
if (specified["in-situ"]) {
if (output || specified["reduce-test"] || specified["self"]) fatal("incompatible options specified");
if (output && output != "spidermonkey" || specified["reduce-test"] || specified["self"]) {
fatal("incompatible options specified");
}
paths.forEach(function(name) {
print(name);
if (/^ast|spidermonkey$/.test(name)) fatal("invalid file name specified");
Expand Down Expand Up @@ -313,6 +316,7 @@ function run() {
if (options.parse.acorn) {
files = convert_ast(function(toplevel, name) {
return require("acorn").parse(files[name], {
allowHashBang: true,
ecmaVersion: "latest",
locations: true,
program: toplevel,
Expand Down Expand Up @@ -413,7 +417,17 @@ function run() {
} else if (output == "spidermonkey") {
print(JSON.stringify(result.ast.to_mozilla_ast(), null, 2));
} else if (output) {
fs.writeFileSync(output, result.code);
var code;
if (result.ast) {
var opts = {};
for (var name in options.output) {
if (!/^ast|code$/.test(name)) opts[name] = options.output[name];
}
code = UglifyJS.AST_Node.from_mozilla_ast(result.ast.to_mozilla_ast()).print_to_string(opts);
} else {
code = result.code;
}
fs.writeFileSync(output, code);
if (result.map) fs.writeFileSync(output + ".map", result.map);
} else {
print(result.code);
Expand Down
22 changes: 12 additions & 10 deletions lib/minify.js
View file
Expand Up @@ -204,27 +204,29 @@ function minify(files, options) {
if (options.mangle && options.mangle.properties) mangle_properties(toplevel, options.mangle.properties);
if (timings) timings.output = Date.now();
var result = {};
if (options.output.ast) {
result.ast = toplevel;
}
if (!HOP(options.output, "code") || options.output.code) {
var output = defaults(options.output, {
ast: false,
code: true,
});
if (output.ast) result.ast = toplevel;
if (output.code) {
if (options.sourceMap) {
options.output.source_map = SourceMap(options.sourceMap);
output.source_map = SourceMap(options.sourceMap);
if (options.sourceMap.includeSources) {
if (files instanceof AST_Toplevel) {
throw new Error("original source content unavailable");
} else for (var name in files) if (HOP(files, name)) {
options.output.source_map.setSourceContent(name, files[name]);
output.source_map.setSourceContent(name, files[name]);
}
}
}
delete options.output.ast;
delete options.output.code;
var stream = OutputStream(options.output);
delete output.ast;
delete output.code;
var stream = OutputStream(output);
toplevel.print(stream);
result.code = stream.get();
if (options.sourceMap) {
result.map = options.output.source_map.toString();
result.map = output.source_map.toString();
var url = options.sourceMap.url;
if (url) {
result.code = result.code.replace(/\n\/\/# sourceMappingURL=\S+\s*$/, "");
Expand Down
41 changes: 25 additions & 16 deletions lib/mozilla-ast.js
View file
Expand Up @@ -220,7 +220,7 @@
start: my_start_token(M),
end: my_end_token(M),
key: key,
value: from_moz(M[M.shorthand ? "key" : "value"]),
value: from_moz(M.value),
};
if (M.kind == "init") return new (M.method ? AST_ObjectMethod : AST_ObjectKeyVal)(args);
args.value = new AST_Accessor(args.value);
Expand Down Expand Up @@ -395,6 +395,20 @@
path: M.source.value,
});
},
ImportExpression: function(M) {
var start = my_start_token(M);
var arg = from_moz(M.source);
return new AST_Call({
start: start,
end: my_end_token(M),
expression: new AST_SymbolRef({
start: start,
end: arg.start,
name: "import",
}),
args: [ arg ],
});
},
VariableDeclaration: function(M) {
return new ({
const: AST_Const,
Expand Down Expand Up @@ -462,7 +476,7 @@
} while (p.type == "ArrayPattern"
|| p.type == "AssignmentPattern" && p.left === FROM_MOZ_STACK[level + 1]
|| p.type == "ObjectPattern"
|| p.type == "Property" && p[p.shorthand ? "key" : "value"] === FROM_MOZ_STACK[level + 1]
|| p.type == "Property" && p.value === FROM_MOZ_STACK[level + 1]
|| p.type == "VariableDeclarator" && p.id === FROM_MOZ_STACK[level + 1]);
var ctor = AST_SymbolRef;
switch (p.type) {
Expand Down Expand Up @@ -731,18 +745,9 @@
});

def_to_moz(AST_ExportDefault, function To_Moz_ExportDefaultDeclaration(M) {
var decl = to_moz(M.body);
switch (decl.type) {
case "ClassExpression":
decl.type = "ClassDeclaration";
break;
case "FunctionExpression":
decl.type = "FunctionDeclaration";
break;
}
return {
type: "ExportDefaultDeclaration",
declaration: decl,
declaration: to_moz(M.body),
};
});

Expand Down Expand Up @@ -1158,11 +1163,15 @@

var FROM_MOZ_STACK = null;

function from_moz(node) {
FROM_MOZ_STACK.push(node);
var ret = node != null ? MOZ_TO_ME[node.type](node) : null;
function from_moz(moz) {
FROM_MOZ_STACK.push(moz);
var node = null;
if (moz) {
if (!HOP(MOZ_TO_ME, moz.type)) throw new Error("Unsupported type: " + moz.type);
node = MOZ_TO_ME[moz.type](moz);
}
FROM_MOZ_STACK.pop();
return ret;
return node;
}

AST_Node.from_mozilla_ast = function(node) {
Expand Down
111 changes: 72 additions & 39 deletions test/mozilla-ast.js
View file
Expand Up @@ -5,23 +5,23 @@ var acorn = require("acorn");
var ufuzz = require("./ufuzz");
var UglifyJS = require("..");

function try_beautify(code) {
var beautified = UglifyJS.minify(code, {
function beautify(ast) {
var beautified = UglifyJS.minify(ast, {
compress: false,
mangle: false,
output: {
beautify: true,
braces: true,
}
},
});
if (beautified.error) return beautified;
return UglifyJS.minify(beautified.code, {
compress: false,
mangle: false,
output: {
ast: true,
},
});
if (beautified.error) {
console.log("// !!! beautify failed !!!");
console.log(beautified.error.stack);
console.log(code);
} else {
console.log("// (beautified)");
console.log(beautified.code);
}
}

function validate(ast) {
Expand All @@ -35,30 +35,71 @@ function validate(ast) {
return UglifyJS.minify(ast, {
compress: false,
mangle: false,
output: {
ast: true,
},
validate: true,
});
}

function fuzzy(code) {
function patch_import(code) {
return code.replace(/\bimport\s*\{\s*\}\s*from\s*(['"])/g, "import$1")
.replace(/\b(import\b.*?)\s*,\s*\{\s*\}\s*(from\s*['"])/g, "$1 $2");
}

function test(original, estree, description) {
var transformed = validate(UglifyJS.AST_Node.from_mozilla_ast(estree));
if (transformed.error || original !== transformed.code && fuzzy(original) !== fuzzy(transformed.code)) {
function equals(input, transformed) {
if (input.code === transformed.code) return true;
return patch_import(input.code) === patch_import(transformed.code);
}

function test(input, to_moz, description, skip_on_error, beautified) {
try {
var ast = UglifyJS.AST_Node.from_mozilla_ast(to_moz(input));
} catch (e) {
if (skip_on_error) return true;
console.log("//=============================================================");
console.log("//", description, "failed... round", round);
console.log(e);
console.log("// original code");
if (beautified === true) console.log("// (beautified)");
console.log(input.code);
return false;
}
var transformed = validate(ast);
if (transformed.error || !equals(input, transformed)) {
if (!beautified) {
beautified = beautify(input.ast);
if (!beautified.error) {
beautified.raw = beautified.code;
if (!test(beautified, to_moz, description, skip_on_error, true)) return false;
}
}
console.log("//=============================================================");
console.log("// !!!!!! Failed... round", round);
console.log("// original code");
try_beautify(original);
if (beautified.error) {
console.log("// !!! beautify failed !!!");
console.log(beautified.error.stack);
} else if (beautified === true) {
console.log("// (beautified)");
}
console.log(input.raw);
console.log();
console.log();
console.log("//-------------------------------------------------------------");
console.log("//", description);
if (transformed.error) {
console.log(transformed.error.stack);
} else {
try_beautify(transformed.code);
beautified = beautify(transformed.ast);
if (beautified.error) {
console.log("// !!! beautify failed !!!");
console.log(beautified.error.stack);
console.log(transformed.code);
} else {
console.log("// (beautified)");
console.log(beautified.code);
}
}
console.log("!!!!!! Failed... round", round);
return false;
Expand All @@ -73,41 +114,33 @@ for (var round = 1; round <= num_iterations; round++) {
process.stdout.write(round + " of " + num_iterations + "\r");
var code = ufuzz.createTopLevelCode();
minify_options.forEach(function(options) {
var input = options ? UglifyJS.minify(code, JSON.parse(options)).code : code;
var uglified = UglifyJS.minify(input, {
var ok = true;
var input = UglifyJS.minify(options ? UglifyJS.minify(code, JSON.parse(options)).code : code, {
compress: false,
mangle: false,
output: {
ast: true,
},
});
var ok = true;
try {
var estree = uglified.ast.to_mozilla_ast();
} catch (e) {
input.raw = options ? input.code : code;
if (input.error) {
ok = false;
console.log("//=============================================================");
console.log("// AST_Node.to_mozilla_ast() failed... round", round);
console.log(e);
console.log("// minify() failed... round", round);
console.log(input.error);
console.log("// original code");
console.log(input);
console.log(code);
}
if (ok) ok = test(uglified.code, estree, "AST_Node.to_mozilla_ast()");
if (ok) try {
ok = test(uglified.code, acorn.parse(input, {
if (ok) ok = test(input, function(input) {
return input.ast.to_mozilla_ast();
}, "AST_Node.to_mozilla_ast()");
if (ok) ok = test(input, function(input) {
return acorn.parse(input.raw, {
ecmaVersion: "latest",
locations: true,
sourceType: "module",
}), "acorn.parse()");
} catch (e) {
if (ufuzz.verbose) {
console.log("//=============================================================");
console.log("// acorn parser failed... round", round);
console.log(e);
console.log("// original code");
console.log(input);
}
}
});
}, "acorn.parse()", !ufuzz.verbose);
if (!ok) process.exit(1);
});
}
Expand Down
2 changes: 1 addition & 1 deletion test/release/rollup-ts.sh
View file
Expand Up @@ -15,7 +15,7 @@ minify_in_situ() {
for i in `find $DIRS -type f -name '*.ts' | grep -v '\.d\.ts'`
do
echo "$i"
node_modules/.bin/esbuild --loader=ts --target=node14 < "$i" \
node_modules/.bin/esbuild --loader=ts --target=es2019 < "$i" \
| uglify-js $UGLIFY_OPTIONS -o "$i"
done
}
Expand Down
2 changes: 1 addition & 1 deletion test/release/sucrase.sh
View file
Expand Up @@ -19,7 +19,7 @@ minify_in_situ() {
for i in `find $DIRS -type f -name '*.ts' | grep -v '\.d\.ts'`
do
echo "$i"
node_modules/.bin/esbuild --loader=ts --target=node14 < "$i" \
node_modules/.bin/esbuild --loader=ts --target=es2019 < "$i" \
| uglify-js $UGLIFY_OPTIONS -o "$i"
done
}
Expand Down

0 comments on commit 97bd56b

Please sign in to comment.