From 97bd56b7e89c29fe2ba2800f88972e168366ae99 Mon Sep 17 00:00:00 2001 From: "Alex Lam S.L" Date: Tue, 27 Apr 2021 01:53:45 +0100 Subject: [PATCH] improve AST tests & tools (#4873) --- .github/workflows/build.yml | 6 +- bin/uglifyjs | 26 +++++++-- lib/minify.js | 22 +++---- lib/mozilla-ast.js | 41 +++++++------ test/mozilla-ast.js | 111 +++++++++++++++++++++++------------- test/release/rollup-ts.sh | 2 +- test/release/sucrase.sh | 2 +- 7 files changed, 136 insertions(+), 74 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed6c28919c..381b79aeaa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,11 @@ jobs: strategy: fail-fast: false matrix: - options: [ '-mb braces', '--ie8 -c', '-mc', '--toplevel -mc passes=3,pure_getters,unsafe' ] + options: + - '-mb braces' + - '--ie8 -c' + - '-mc' + - '--toplevel -mc passes=3,pure_getters,unsafe' script: - acorn.sh - bootstrap.sh diff --git a/bin/uglifyjs b/bin/uglifyjs index 325dacdde3..69a98a4bd8 100755 --- a/bin/uglifyjs +++ b/bin/uglifyjs @@ -235,10 +235,11 @@ if (options.mangle && options.mangle.properties) { }); } } -if (output == "ast" || output == "spidermonkey") options.output = { - ast: true, - code: false, -}; +if (/^ast|spidermonkey$/.test(output)) { + if (typeof options.output != "object") options.output = {}; + options.output.ast = true; + options.output.code = false; +} if (options.parse && (options.parse.acorn || options.parse.spidermonkey) && options.sourceMap && options.sourceMap.content == "inline") { fatal("inline source map only works with built-in parser"); @@ -265,7 +266,9 @@ if (specified["self"]) { paths = UglifyJS.FILES; } if (specified["in-situ"]) { - if (output || specified["reduce-test"] || specified["self"]) fatal("incompatible options specified"); + if (output && output != "spidermonkey" || specified["reduce-test"] || specified["self"]) { + fatal("incompatible options specified"); + } paths.forEach(function(name) { print(name); if (/^ast|spidermonkey$/.test(name)) fatal("invalid file name specified"); @@ -313,6 +316,7 @@ function run() { if (options.parse.acorn) { files = convert_ast(function(toplevel, name) { return require("acorn").parse(files[name], { + allowHashBang: true, ecmaVersion: "latest", locations: true, program: toplevel, @@ -413,7 +417,17 @@ function run() { } else if (output == "spidermonkey") { print(JSON.stringify(result.ast.to_mozilla_ast(), null, 2)); } else if (output) { - fs.writeFileSync(output, result.code); + var code; + if (result.ast) { + var opts = {}; + for (var name in options.output) { + if (!/^ast|code$/.test(name)) opts[name] = options.output[name]; + } + code = UglifyJS.AST_Node.from_mozilla_ast(result.ast.to_mozilla_ast()).print_to_string(opts); + } else { + code = result.code; + } + fs.writeFileSync(output, code); if (result.map) fs.writeFileSync(output + ".map", result.map); } else { print(result.code); diff --git a/lib/minify.js b/lib/minify.js index b92d4b536c..52246d705c 100644 --- a/lib/minify.js +++ b/lib/minify.js @@ -204,27 +204,29 @@ function minify(files, options) { if (options.mangle && options.mangle.properties) mangle_properties(toplevel, options.mangle.properties); if (timings) timings.output = Date.now(); var result = {}; - if (options.output.ast) { - result.ast = toplevel; - } - if (!HOP(options.output, "code") || options.output.code) { + var output = defaults(options.output, { + ast: false, + code: true, + }); + if (output.ast) result.ast = toplevel; + if (output.code) { if (options.sourceMap) { - options.output.source_map = SourceMap(options.sourceMap); + output.source_map = SourceMap(options.sourceMap); if (options.sourceMap.includeSources) { if (files instanceof AST_Toplevel) { throw new Error("original source content unavailable"); } else for (var name in files) if (HOP(files, name)) { - options.output.source_map.setSourceContent(name, files[name]); + output.source_map.setSourceContent(name, files[name]); } } } - delete options.output.ast; - delete options.output.code; - var stream = OutputStream(options.output); + delete output.ast; + delete output.code; + var stream = OutputStream(output); toplevel.print(stream); result.code = stream.get(); if (options.sourceMap) { - result.map = options.output.source_map.toString(); + result.map = output.source_map.toString(); var url = options.sourceMap.url; if (url) { result.code = result.code.replace(/\n\/\/# sourceMappingURL=\S+\s*$/, ""); diff --git a/lib/mozilla-ast.js b/lib/mozilla-ast.js index 09c5b105c5..028441b760 100644 --- a/lib/mozilla-ast.js +++ b/lib/mozilla-ast.js @@ -220,7 +220,7 @@ start: my_start_token(M), end: my_end_token(M), key: key, - value: from_moz(M[M.shorthand ? "key" : "value"]), + value: from_moz(M.value), }; if (M.kind == "init") return new (M.method ? AST_ObjectMethod : AST_ObjectKeyVal)(args); args.value = new AST_Accessor(args.value); @@ -395,6 +395,20 @@ path: M.source.value, }); }, + ImportExpression: function(M) { + var start = my_start_token(M); + var arg = from_moz(M.source); + return new AST_Call({ + start: start, + end: my_end_token(M), + expression: new AST_SymbolRef({ + start: start, + end: arg.start, + name: "import", + }), + args: [ arg ], + }); + }, VariableDeclaration: function(M) { return new ({ const: AST_Const, @@ -462,7 +476,7 @@ } while (p.type == "ArrayPattern" || p.type == "AssignmentPattern" && p.left === FROM_MOZ_STACK[level + 1] || p.type == "ObjectPattern" - || p.type == "Property" && p[p.shorthand ? "key" : "value"] === FROM_MOZ_STACK[level + 1] + || p.type == "Property" && p.value === FROM_MOZ_STACK[level + 1] || p.type == "VariableDeclarator" && p.id === FROM_MOZ_STACK[level + 1]); var ctor = AST_SymbolRef; switch (p.type) { @@ -731,18 +745,9 @@ }); def_to_moz(AST_ExportDefault, function To_Moz_ExportDefaultDeclaration(M) { - var decl = to_moz(M.body); - switch (decl.type) { - case "ClassExpression": - decl.type = "ClassDeclaration"; - break; - case "FunctionExpression": - decl.type = "FunctionDeclaration"; - break; - } return { type: "ExportDefaultDeclaration", - declaration: decl, + declaration: to_moz(M.body), }; }); @@ -1158,11 +1163,15 @@ var FROM_MOZ_STACK = null; - function from_moz(node) { - FROM_MOZ_STACK.push(node); - var ret = node != null ? MOZ_TO_ME[node.type](node) : null; + function from_moz(moz) { + FROM_MOZ_STACK.push(moz); + var node = null; + if (moz) { + if (!HOP(MOZ_TO_ME, moz.type)) throw new Error("Unsupported type: " + moz.type); + node = MOZ_TO_ME[moz.type](moz); + } FROM_MOZ_STACK.pop(); - return ret; + return node; } AST_Node.from_mozilla_ast = function(node) { diff --git a/test/mozilla-ast.js b/test/mozilla-ast.js index 0f6db3ac02..c10c679a8e 100644 --- a/test/mozilla-ast.js +++ b/test/mozilla-ast.js @@ -5,23 +5,23 @@ var acorn = require("acorn"); var ufuzz = require("./ufuzz"); var UglifyJS = require(".."); -function try_beautify(code) { - var beautified = UglifyJS.minify(code, { +function beautify(ast) { + var beautified = UglifyJS.minify(ast, { compress: false, mangle: false, output: { beautify: true, braces: true, - } + }, + }); + if (beautified.error) return beautified; + return UglifyJS.minify(beautified.code, { + compress: false, + mangle: false, + output: { + ast: true, + }, }); - if (beautified.error) { - console.log("// !!! beautify failed !!!"); - console.log(beautified.error.stack); - console.log(code); - } else { - console.log("// (beautified)"); - console.log(beautified.code); - } } function validate(ast) { @@ -35,22 +35,55 @@ function validate(ast) { return UglifyJS.minify(ast, { compress: false, mangle: false, + output: { + ast: true, + }, validate: true, }); } -function fuzzy(code) { +function patch_import(code) { return code.replace(/\bimport\s*\{\s*\}\s*from\s*(['"])/g, "import$1") .replace(/\b(import\b.*?)\s*,\s*\{\s*\}\s*(from\s*['"])/g, "$1 $2"); } -function test(original, estree, description) { - var transformed = validate(UglifyJS.AST_Node.from_mozilla_ast(estree)); - if (transformed.error || original !== transformed.code && fuzzy(original) !== fuzzy(transformed.code)) { +function equals(input, transformed) { + if (input.code === transformed.code) return true; + return patch_import(input.code) === patch_import(transformed.code); +} + +function test(input, to_moz, description, skip_on_error, beautified) { + try { + var ast = UglifyJS.AST_Node.from_mozilla_ast(to_moz(input)); + } catch (e) { + if (skip_on_error) return true; + console.log("//============================================================="); + console.log("//", description, "failed... round", round); + console.log(e); + console.log("// original code"); + if (beautified === true) console.log("// (beautified)"); + console.log(input.code); + return false; + } + var transformed = validate(ast); + if (transformed.error || !equals(input, transformed)) { + if (!beautified) { + beautified = beautify(input.ast); + if (!beautified.error) { + beautified.raw = beautified.code; + if (!test(beautified, to_moz, description, skip_on_error, true)) return false; + } + } console.log("//============================================================="); console.log("// !!!!!! Failed... round", round); console.log("// original code"); - try_beautify(original); + if (beautified.error) { + console.log("// !!! beautify failed !!!"); + console.log(beautified.error.stack); + } else if (beautified === true) { + console.log("// (beautified)"); + } + console.log(input.raw); console.log(); console.log(); console.log("//-------------------------------------------------------------"); @@ -58,7 +91,15 @@ function test(original, estree, description) { if (transformed.error) { console.log(transformed.error.stack); } else { - try_beautify(transformed.code); + beautified = beautify(transformed.ast); + if (beautified.error) { + console.log("// !!! beautify failed !!!"); + console.log(beautified.error.stack); + console.log(transformed.code); + } else { + console.log("// (beautified)"); + console.log(beautified.code); + } } console.log("!!!!!! Failed... round", round); return false; @@ -73,41 +114,33 @@ for (var round = 1; round <= num_iterations; round++) { process.stdout.write(round + " of " + num_iterations + "\r"); var code = ufuzz.createTopLevelCode(); minify_options.forEach(function(options) { - var input = options ? UglifyJS.minify(code, JSON.parse(options)).code : code; - var uglified = UglifyJS.minify(input, { + var ok = true; + var input = UglifyJS.minify(options ? UglifyJS.minify(code, JSON.parse(options)).code : code, { compress: false, mangle: false, output: { ast: true, }, }); - var ok = true; - try { - var estree = uglified.ast.to_mozilla_ast(); - } catch (e) { + input.raw = options ? input.code : code; + if (input.error) { ok = false; console.log("//============================================================="); - console.log("// AST_Node.to_mozilla_ast() failed... round", round); - console.log(e); + console.log("// minify() failed... round", round); + console.log(input.error); console.log("// original code"); - console.log(input); + console.log(code); } - if (ok) ok = test(uglified.code, estree, "AST_Node.to_mozilla_ast()"); - if (ok) try { - ok = test(uglified.code, acorn.parse(input, { + if (ok) ok = test(input, function(input) { + return input.ast.to_mozilla_ast(); + }, "AST_Node.to_mozilla_ast()"); + if (ok) ok = test(input, function(input) { + return acorn.parse(input.raw, { ecmaVersion: "latest", locations: true, sourceType: "module", - }), "acorn.parse()"); - } catch (e) { - if (ufuzz.verbose) { - console.log("//============================================================="); - console.log("// acorn parser failed... round", round); - console.log(e); - console.log("// original code"); - console.log(input); - } - } + }); + }, "acorn.parse()", !ufuzz.verbose); if (!ok) process.exit(1); }); } diff --git a/test/release/rollup-ts.sh b/test/release/rollup-ts.sh index c80e4d1820..0a960d552a 100755 --- a/test/release/rollup-ts.sh +++ b/test/release/rollup-ts.sh @@ -15,7 +15,7 @@ minify_in_situ() { for i in `find $DIRS -type f -name '*.ts' | grep -v '\.d\.ts'` do echo "$i" - node_modules/.bin/esbuild --loader=ts --target=node14 < "$i" \ + node_modules/.bin/esbuild --loader=ts --target=es2019 < "$i" \ | uglify-js $UGLIFY_OPTIONS -o "$i" done } diff --git a/test/release/sucrase.sh b/test/release/sucrase.sh index c293f3c858..909e633eda 100755 --- a/test/release/sucrase.sh +++ b/test/release/sucrase.sh @@ -19,7 +19,7 @@ minify_in_situ() { for i in `find $DIRS -type f -name '*.ts' | grep -v '\.d\.ts'` do echo "$i" - node_modules/.bin/esbuild --loader=ts --target=node14 < "$i" \ + node_modules/.bin/esbuild --loader=ts --target=es2019 < "$i" \ | uglify-js $UGLIFY_OPTIONS -o "$i" done }