Skip to content

Commit

Permalink
fix(es/codegen): Fix codegen of unicode surrogates (#7985)
Browse files Browse the repository at this point in the history
**Description:**


x-ref: https://vercel.slack.com/archives/C02HY34AKME/p1695334071194139


Reproduction: https://github.com/kdy1/repro-test-mdx-korean



Regression of next.js: `v13.4.11`(swc_core@v0.79.13) => `v13.5.2`
(swc_core@v0.83.12)


 - `next@v13.4.19` works (swc_core@v0.79.59)
 - `next@v13.4.20-canary.3` works (swc_core@v0.79.70)
 -  **`next@v13.4.20-canary.32` fails** (swc_core@v0.83.12)

Commit range:
662f236...e67bf05

- `swc_core@v0.79.70`:
662f236
- `swc_core@v0.83.12`:
e67bf05

Diff: https://gist.github.com/kdy1/047e7e5537c34180d446cb3d5b95fce8

---

I did more investigation by monkey-patching the `next` package.
`.minify()` resolved without an exception.
It means that `.minify()` is producing an invalid ES code.

So... candidates are

 - #7890
 - #7876
 - #7858
 - #7856
 - #7853
 - #7832
  • Loading branch information
kdy1 committed Sep 25, 2023
1 parent 243d68d commit 39c680d
Show file tree
Hide file tree
Showing 27 changed files with 91 additions and 6 deletions.
5 changes: 4 additions & 1 deletion crates/swc/tests/exec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -372,8 +372,11 @@ fn test_file_with_opts(
idx,
res.code
);
println!("external_helpers: {:?}", opts.config.jsc.external_helpers);

println!("target: {:?}", opts.config.jsc.target.unwrap());
println!("minify: {:?}", opts.config.jsc.minify.is_some());
println!("external_helpers: {:?}", opts.config.jsc.external_helpers);
println!("source_map: {:?}", opts.source_maps.is_some());

let actual_stdout = stdout_of(
&res.code,
Expand Down
29 changes: 25 additions & 4 deletions crates/swc_ecma_codegen/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3850,13 +3850,34 @@ fn get_quoted_utf16(v: &str, ascii_only: bool, target: EsVersion) -> String {
inner_buf.push('}');
}

let range = if is_curly {
3..(inner_buf.len() - 1)
} else {
2..6
};

let val_str = &inner_buf[range];

if is_valid {
buf.push_str(&inner_buf);
let v = u32::from_str_radix(val_str, 16).unwrap_or_else(|err| {
unreachable!(
"failed to parse {} as a hex value: {:?}",
val_str, err
)
});

let end = if is_curly { 7 } else { 5 };
if v > 0xffff {
buf.push_str(&inner_buf);

for _ in 0..end {
iter.next();
let end = if is_curly { 7 } else { 5 };

for _ in 0..end {
iter.next();
}
} else if (0xd800..=0xdfff).contains(&v) {
buf.push('\\');
} else {
buf.push_str("\\\\");
}
}
} else if is_curly {
Expand Down
38 changes: 38 additions & 0 deletions crates/swc_ecma_codegen/src/tests.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
use std::{
fmt::Debug,
io::Write,
path::PathBuf,
sync::{Arc, RwLock},
};

use swc_common::{comments::SingleThreadedComments, FileName, SourceMap};
use swc_ecma_parser;
use swc_ecma_testing::{exec_node_js, JsExecOptions};
use testing::DebugUsingDisplay;

use self::swc_ecma_parser::{EsConfig, Parser, StringInput, Syntax};
Expand Down Expand Up @@ -889,3 +891,39 @@ export default {
},
);
}

#[testing::fixture("tests/str-lits/**/*.txt")]
fn test_str_lit(input: PathBuf) {
test_str_lit_inner(input)
}

/// Print text back using `console.log`
fn run_node(code: &str) -> String {
exec_node_js(
code,
JsExecOptions {
cache: true,
module: false,
args: vec![],
},
)
.expect("failed to execute node.js")
}

fn test_str_lit_inner(input: PathBuf) {
let raw_str = std::fs::read_to_string(&*input).unwrap();
let input_code = format!("console.log('{raw_str}')");

let output_code = parse_then_emit(
&input_code,
Config::default()
.with_target(EsVersion::latest())
.with_minify(true),
Syntax::default(),
);

let expected = run_node(&input_code);
let actual = run_node(&output_code);

assert_eq!(actual, expected);
}
1 change: 1 addition & 0 deletions crates/swc_ecma_codegen/tests/str-lits/pr-7985/full.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
****:한국어TürkçeTurkish\\n-**uk**:УкраїнськаUkrainian\\n-**vi**:TiếngViệtVietnamese\\n-**zh-cn**:简体中文SimplifiedChinese\\n-**zh-tw**:繁體中文TraditionalChinese\\n\\n\\n\\n##Updatetranslatedstrings\\n\\nIfyouwant`fr`locales|`Espa\\\\xF1ol(Latinoam\\\\xE9rica)Spanish(LatinAmerica)\\"}),`\\n`,(n\\"fi\\"}),\\":SuomiFinnish\\"}),`\\n`,(n\\"fr\\"}),\\":Fran\\\\xE7aisFrench\\"}),`\\n`,(n\\"fr-be\\"}),\\":Fran\\\\xE7ais(Belgique)French(Belgium)\\"}),`\\n`,(n\\"fr-ca\\"}),\\":Fran\\\\xE7ais(Canada)French(Canada)\\"}),`\\n`,(n\\"fr-ch\\"}),\\":Fran\\\\xE7ais(Suisse)French(Switzerland)\\"}),`\\n`,(n\\"fr-fr\\"}),\\":Fran\\\\xE7ais(France)French(France)\\"}),`\\n`,(n\\"hu\\"}),\\":MagyarHungarian\\"}),`\\n`,(n\\"id\\"}),\\":BahasaIndonesiaIndonesian\\"}),`\\n`,(n\\"it\\"}),\\":ItalianoItalian\\"}),`\\n`,(n\\"ja\\"}),\\":\\\\u65E5\\\\u672C\\\\u8A9EJapanese\\"}),`\\n`,(n\\"ko\\"}),\\":\\\\uD55C\\\\uAD6D\\\\uC5B4Korean\\"}),`\\n`,(n\\"nl\\"}),\\":NederlandsDutch\\"}),`\\n`,(n\\"nl-be\\"}),\\":Nederlands(Belgi\\\\xEB)Dutch(Belgium)\\"}),`\\n`,(n\\"no\\"}),\\":NorskNorwegian\\"}),`\\n`,(n\\"pl\\"}),\\":(n\\"th\\"}),\\":\\\\u0E44\\\\u0E17\\\\u0E22Thai\\"}),`\\n`,(n\\"tr\\"}),\\":T\\\\xFCrk\\\\xE7eTurkish\\"}),`\\n`,(n.li,{()(n.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
****:한국어TürkçeTurkish\\n-**uk**:Українсь
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(Belgium)\\"}),`\\n`,(n\\"fr-ca\\"}),\\":Fran\\\\xE7ais(Canada)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
French(Canada)\\"}),`\\n`,(n\\"fr-ch\\"}),\\":Fran\\\\xE7ais(Suisse)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
French(Switzerland)\\"}),`\\n`,(n\\"fr-fr\\"}),\\":Fran\\\\xE7ais(France)French
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
(France)\\"}),`\\n`,(n\\"hu\\"}),\\":MagyarHungarian\\"}),`\\n`,(n\\"id\\"}),\\":BahasaIndonesia
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Indonesian\\"}),`\\n`,(n\\"it\\"}),\\":Italiano
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Italian\\"}),`\\n`,(n\\"ja\\"}),\\":\\\\u65E5\\\\u672C\\\\u8A9E
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
\\\\uD55C\\\\uAD6D\\\\uC5B4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Thai\\"}),`\\n`,(n\\"tr\\"}),\\":T\\\\xFCrk\\\\xE7eTurkish\\"}),`\\n`,(n.li,{()(n.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Korean\\"}),`\\n`,(n\\"nl\\"}),\\":Nederlands
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Dutch\\"}),`\\n`,(n\\"nl-be\\"}),\\":Nederlands(Belgi\\\\xEB)Dutch(Belgium)\\"}),`\\n`,(n\\"no\\"}),\\":Norsk
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
каUkrainian\\n-**vi**:TiếngViệtVietna
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Norwegian\\"}),`\\n`,(n\\"pl\\"}),\\":(n\\"th\\"}),\\":\\\\u0E44\\\\u0E17\\\\u0E22
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
mese\\n-**zh-cn**:简体中文Simplified
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Chinese\\n-**zh-tw**:繁體中文Traditional
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Chinese\\n\\n\\n\\n##Updatetranslatedstrings\\n\\nIfyou
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
want`fr`locales|`Espa\\\\xF1ol(Lati
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
noam\\\\xE9rica)Spanish(Latin
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
America)\\"}),`\\n`,(n\\"fi\\"}),\\":Suomi
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Finnish\\"}),`\\n`,(n\\"fr\\"}),\\":Fran\\\\xE7aisFrench\\"}),`\\n`,(n\\"fr-be\\"}),\\":Fran\\\\xE7ais(Belgique)French
2 changes: 1 addition & 1 deletion crates/swc_ecma_minifier/scripts/next/add-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ rm -rf .next
# This is not useful data, as this is a script used only by me (@kdy1)
npx next telemetry disable

npx next build | grep '{ name:' | node "$SCRIPT_DIR/evaluate.js"
npm run build | node "$SCRIPT_DIR/evaluate.js"
touch "$SCRIPT_DIR/../../tests/compress.rs"

Large diffs are not rendered by default.

Large diffs are not rendered by default.

1 comment on commit 39c680d

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmark

Benchmark suite Current: 39c680d Previous: d91a554 Ratio
es/full/bugs-1 289009 ns/iter (± 4180) 291271 ns/iter (± 7629) 0.99
es/full/minify/libraries/antd 1335949180 ns/iter (± 15128064) 1290222316 ns/iter (± 6616820) 1.04
es/full/minify/libraries/d3 277101473 ns/iter (± 3180057) 274470290 ns/iter (± 2732425) 1.01
es/full/minify/libraries/echarts 1066188492 ns/iter (± 9902355) 1023013542 ns/iter (± 5681241) 1.04
es/full/minify/libraries/jquery 84115193 ns/iter (± 240716) 83738151 ns/iter (± 140019) 1.00
es/full/minify/libraries/lodash 97537477 ns/iter (± 273068) 96401929 ns/iter (± 186788) 1.01
es/full/minify/libraries/moment 49861779 ns/iter (± 235227) 49550605 ns/iter (± 124381) 1.01
es/full/minify/libraries/react 17943263 ns/iter (± 45433) 17926503 ns/iter (± 40517) 1.00
es/full/minify/libraries/terser 218275344 ns/iter (± 554907) 215057306 ns/iter (± 577178) 1.01
es/full/minify/libraries/three 387824541 ns/iter (± 3848797) 378687630 ns/iter (± 934136) 1.02
es/full/minify/libraries/typescript 2657619646 ns/iter (± 22332243) 2607999632 ns/iter (± 7818306) 1.02
es/full/minify/libraries/victory 578220388 ns/iter (± 7046643) 555094223 ns/iter (± 2978373) 1.04
es/full/minify/libraries/vue 119381019 ns/iter (± 513512) 118243071 ns/iter (± 226987) 1.01
es/full/codegen/es3 33627 ns/iter (± 99) 34915 ns/iter (± 61) 0.96
es/full/codegen/es5 33754 ns/iter (± 117) 34769 ns/iter (± 82) 0.97
es/full/codegen/es2015 33755 ns/iter (± 53) 34812 ns/iter (± 95) 0.97
es/full/codegen/es2016 33716 ns/iter (± 80) 34871 ns/iter (± 52) 0.97
es/full/codegen/es2017 33677 ns/iter (± 137) 34880 ns/iter (± 81) 0.97
es/full/codegen/es2018 33658 ns/iter (± 44) 34834 ns/iter (± 70) 0.97
es/full/codegen/es2019 33716 ns/iter (± 58) 34827 ns/iter (± 83) 0.97
es/full/codegen/es2020 33711 ns/iter (± 84) 34852 ns/iter (± 91) 0.97
es/full/all/es3 165814050 ns/iter (± 1028503) 165909650 ns/iter (± 1306080) 1.00
es/full/all/es5 158607282 ns/iter (± 796684) 158241506 ns/iter (± 1687849) 1.00
es/full/all/es2015 117209465 ns/iter (± 685074) 117248903 ns/iter (± 581802) 1.00
es/full/all/es2016 115938935 ns/iter (± 590553) 116681920 ns/iter (± 599433) 0.99
es/full/all/es2017 115534417 ns/iter (± 1094279) 115713717 ns/iter (± 321588) 1.00
es/full/all/es2018 113320703 ns/iter (± 482366) 114585197 ns/iter (± 581069) 0.99
es/full/all/es2019 112380074 ns/iter (± 718673) 113876823 ns/iter (± 679582) 0.99
es/full/all/es2020 108376586 ns/iter (± 522639) 109497543 ns/iter (± 544870) 0.99
es/full/parser 487834 ns/iter (± 6096) 496045 ns/iter (± 5370) 0.98
es/full/base/fixer 17444 ns/iter (± 55) 19747 ns/iter (± 283) 0.88
es/full/base/resolver_and_hygiene 80416 ns/iter (± 155) 80334 ns/iter (± 163) 1.00
serialization of serde 292 ns/iter (± 2) 292 ns/iter (± 1) 1
css/minify/libraries/bootstrap 28554221 ns/iter (± 67509) 28871922 ns/iter (± 34164) 0.99
css/visitor/compare/clone 1644986 ns/iter (± 4910) 1670471 ns/iter (± 9340) 0.98
css/visitor/compare/visit_mut_span 1767657 ns/iter (± 5067) 1804882 ns/iter (± 5576) 0.98
css/visitor/compare/visit_mut_span_panic 1853611 ns/iter (± 6935) 1882789 ns/iter (± 6647) 0.98
css/visitor/compare/fold_span 2583829 ns/iter (± 7668) 2590382 ns/iter (± 9935) 1.00
css/visitor/compare/fold_span_panic 2752544 ns/iter (± 9888) 2757482 ns/iter (± 2402) 1.00
css/lexer/bootstrap_5_1_3 4463636 ns/iter (± 6334) 4516907 ns/iter (± 38420) 0.99
css/lexer/foundation_6_7_4 3747485 ns/iter (± 4827) 3769995 ns/iter (± 4762) 0.99
css/lexer/tailwind_3_1_1 713221 ns/iter (± 875) 724048 ns/iter (± 228) 0.99
css/parser/bootstrap_5_1_3 19174227 ns/iter (± 15143) 19361853 ns/iter (± 40018) 0.99
css/parser/foundation_6_7_4 15222130 ns/iter (± 20493) 15180174 ns/iter (± 20253) 1.00
css/parser/tailwind_3_1_1 2941572 ns/iter (± 5621) 2934089 ns/iter (± 2697) 1.00
es/codegen/colors 726913 ns/iter (± 397223) 735741 ns/iter (± 400778) 0.99
es/codegen/large 2908098 ns/iter (± 1548888) 3156314 ns/iter (± 1680374) 0.92
es/codegen/with-parser/colors 45872 ns/iter (± 251) 45113 ns/iter (± 91) 1.02
es/codegen/with-parser/large 486065 ns/iter (± 878) 485228 ns/iter (± 776) 1.00
es/minify/libraries/antd 1153605989 ns/iter (± 18773821) 1112818160 ns/iter (± 9187671) 1.04
es/minify/libraries/d3 238657398 ns/iter (± 1455534) 236921883 ns/iter (± 1313709) 1.01
es/minify/libraries/echarts 909062059 ns/iter (± 5753777) 878883992 ns/iter (± 3685684) 1.03
es/minify/libraries/jquery 73043088 ns/iter (± 153766) 72576054 ns/iter (± 95220) 1.01
es/minify/libraries/lodash 86753667 ns/iter (± 200120) 86540669 ns/iter (± 183447) 1.00
es/minify/libraries/moment 43363911 ns/iter (± 144858) 43146257 ns/iter (± 44482) 1.01
es/minify/libraries/react 16020513 ns/iter (± 41503) 15909917 ns/iter (± 50741) 1.01
es/minify/libraries/terser 185402314 ns/iter (± 446827) 184321073 ns/iter (± 346551) 1.01
es/minify/libraries/three 319963839 ns/iter (± 2249170) 316015293 ns/iter (± 725364) 1.01
es/minify/libraries/typescript 2245540931 ns/iter (± 9438747) 2209558324 ns/iter (± 5742651) 1.02
es/minify/libraries/victory 475914460 ns/iter (± 2672778) 469708553 ns/iter (± 3749820) 1.01
es/minify/libraries/vue 105867927 ns/iter (± 258162) 104708175 ns/iter (± 174936) 1.01
es/visitor/compare/clone 1966598 ns/iter (± 4014) 1946030 ns/iter (± 2918) 1.01
es/visitor/compare/visit_mut_span 2313909 ns/iter (± 4509) 2278689 ns/iter (± 4481) 1.02
es/visitor/compare/visit_mut_span_panic 2353777 ns/iter (± 4805) 2324604 ns/iter (± 1720) 1.01
es/visitor/compare/fold_span 3418358 ns/iter (± 7240) 3364353 ns/iter (± 6519) 1.02
es/visitor/compare/fold_span_panic 3540656 ns/iter (± 10296) 3504965 ns/iter (± 9474) 1.01
es/lexer/colors 13151 ns/iter (± 25) 13393 ns/iter (± 16) 0.98
es/lexer/angular 6126636 ns/iter (± 9998) 6156340 ns/iter (± 3869) 1.00
es/lexer/backbone 791774 ns/iter (± 292) 800635 ns/iter (± 11161) 0.99
es/lexer/jquery 4531191 ns/iter (± 4233) 4553174 ns/iter (± 7186) 1.00
es/lexer/jquery mobile 6937740 ns/iter (± 11068) 6945867 ns/iter (± 3811) 1.00
es/lexer/mootools 3580732 ns/iter (± 4113) 3607527 ns/iter (± 3939) 0.99
es/lexer/underscore 668222 ns/iter (± 358) 674340 ns/iter (± 1583) 0.99
es/lexer/three 21556952 ns/iter (± 36039) 21773567 ns/iter (± 46855) 0.99
es/lexer/yui 3841424 ns/iter (± 3782) 3881098 ns/iter (± 4105) 0.99
es/parser/colors 27615 ns/iter (± 72) 26885 ns/iter (± 62) 1.03
es/parser/angular 13674175 ns/iter (± 55266) 13435870 ns/iter (± 56139) 1.02
es/parser/backbone 1996965 ns/iter (± 7900) 1973488 ns/iter (± 8051) 1.01
es/parser/jquery 10985595 ns/iter (± 35485) 10788276 ns/iter (± 46993) 1.02
es/parser/jquery mobile 16853459 ns/iter (± 101159) 16539323 ns/iter (± 68374) 1.02
es/parser/mootools 8453532 ns/iter (± 19746) 8327372 ns/iter (± 15731) 1.02
es/parser/underscore 1727303 ns/iter (± 7396) 1697510 ns/iter (± 8168) 1.02
es/parser/three 47445388 ns/iter (± 223408) 46251040 ns/iter (± 126315) 1.03
es/parser/yui 8322920 ns/iter (± 34851) 8213383 ns/iter (± 22007) 1.01
es/preset-env/usage/builtin_type 135560 ns/iter (± 31843) 134987 ns/iter (± 31204) 1.00
es/preset-env/usage/property 17269 ns/iter (± 53) 16585 ns/iter (± 47) 1.04
es/resolver/typescript 90314118 ns/iter (± 1615966) 90323700 ns/iter (± 945550) 1.00
es/fixer/typescript 63493640 ns/iter (± 536934) 61776922 ns/iter (± 304281) 1.03
es/hygiene/typescript 129485950 ns/iter (± 1043491) 127962658 ns/iter (± 811728) 1.01
es/resolver_with_hygiene/typescript 242965079 ns/iter (± 1402835) 242209076 ns/iter (± 817999) 1.00
es/visitor/base-perf/module_clone 59076 ns/iter (± 251) 59538 ns/iter (± 276) 0.99
es/visitor/base-perf/fold_empty 63365 ns/iter (± 311) 63278 ns/iter (± 406) 1.00
es/visitor/base-perf/fold_noop_impl_all 63374 ns/iter (± 331) 63713 ns/iter (± 209) 0.99
es/visitor/base-perf/fold_noop_impl_vec 63908 ns/iter (± 349) 64051 ns/iter (± 379) 1.00
es/visitor/base-perf/boxing_boxed_clone 55 ns/iter (± 0) 59 ns/iter (± 0) 0.93
es/visitor/base-perf/boxing_unboxed_clone 39 ns/iter (± 0) 39 ns/iter (± 0) 1
es/visitor/base-perf/boxing_boxed 108 ns/iter (± 0) 110 ns/iter (± 0) 0.98
es/visitor/base-perf/boxing_unboxed 77 ns/iter (± 0) 77 ns/iter (± 0) 1
es/visitor/base-perf/visit_empty 0 ns/iter (± 0) 0 ns/iter (± 0) NaN
es/visitor/base-perf/visit_contains_this 2544 ns/iter (± 15) 2432 ns/iter (± 10) 1.05
es/base/parallel/resolver/typescript 4269831962 ns/iter (± 190598371) 3742564513 ns/iter (± 321996582) 1.14
es/base/parallel/hygiene/typescript 1450551781 ns/iter (± 6545858) 1442586867 ns/iter (± 13333550) 1.01
misc/visitors/time-complexity/time 5 133 ns/iter (± 1) 134 ns/iter (± 3) 0.99
misc/visitors/time-complexity/time 10 370 ns/iter (± 10) 355 ns/iter (± 1) 1.04
misc/visitors/time-complexity/time 15 674 ns/iter (± 3) 669 ns/iter (± 3) 1.01
misc/visitors/time-complexity/time 20 1111 ns/iter (± 4) 1126 ns/iter (± 15) 0.99
misc/visitors/time-complexity/time 40 3563 ns/iter (± 9) 3565 ns/iter (± 7) 1.00
misc/visitors/time-complexity/time 60 7327 ns/iter (± 461) 7444 ns/iter (± 33) 0.98

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.