1
1
//! ECMAScript lexer.
2
2
3
- use std:: { cell:: RefCell , char, iter:: FusedIterator , rc:: Rc } ;
3
+ use std:: { cell:: RefCell , char, iter:: FusedIterator , mem :: transmute , rc:: Rc } ;
4
4
5
5
use either:: Either :: { Left , Right } ;
6
6
use smallvec:: { smallvec, SmallVec } ;
7
7
use swc_atoms:: { Atom , AtomStoreCell } ;
8
8
use swc_common:: { comments:: Comments , input:: StringInput , BytePos , Span } ;
9
- use swc_ecma_ast:: { op, AssignOp , EsVersion } ;
9
+ use swc_ecma_ast:: { op, AssignOp , EsVersion , Ident } ;
10
10
11
11
use self :: {
12
12
comments_buffer:: CommentsBuffer ,
@@ -793,49 +793,50 @@ impl<'a> Lexer<'a> {
793
793
debug_assert ! ( self . cur( ) . is_some( ) ) ;
794
794
let mut first = true ;
795
795
let mut can_be_keyword = true ;
796
+ let mut slice_start = self . cur_pos ( ) ;
797
+ let mut has_escape = false ;
796
798
797
799
self . with_buf ( |l, buf| {
798
- let mut has_escape = false ;
799
-
800
- while let Some ( c) = {
801
- // Optimization
802
- {
803
- let s = l. input . uncons_while ( |c| {
804
- if !c. is_ident_part ( ) {
805
- return false ;
806
- }
807
-
808
- // Performance optimization
809
- if c. is_ascii_uppercase ( ) || c. is_ascii_digit ( ) || !c. is_ascii ( ) {
810
- can_be_keyword = false ;
811
- }
812
-
813
- true
814
- } ) ;
815
- if !s. is_empty ( ) {
816
- first = false ;
800
+ loop {
801
+ if let Some ( c) = l. input . cur_as_ascii ( ) {
802
+ // Performance optimization
803
+ if c. is_ascii_uppercase ( ) || c. is_ascii_digit ( ) {
804
+ can_be_keyword = false ;
817
805
}
818
- buf. push_str ( s)
819
- }
820
806
821
- l. cur ( )
822
- } {
823
- let start = l. cur_pos ( ) ;
824
-
825
- match c {
826
- c if c. is_ident_part ( ) => {
807
+ if Ident :: is_valid_continue ( c as _ ) {
808
+ l. bump ( ) ;
809
+ continue ;
810
+ } else if first && Ident :: is_valid_start ( c as _ ) {
827
811
l. bump ( ) ;
828
- buf. push ( c) ;
812
+ first = false ;
813
+ continue ;
829
814
}
815
+
830
816
// unicode escape
831
- '\\' => {
817
+ if c == b'\\' {
818
+ first = false ;
819
+ has_escape = true ;
820
+ let start = l. cur_pos ( ) ;
832
821
l. bump ( ) ;
833
822
834
823
if !l. is ( b'u' ) {
835
824
l. error_span ( pos_span ( start) , SyntaxError :: ExpectedUnicodeEscape ) ?
836
825
}
837
826
838
- has_escape = true ;
827
+ {
828
+ let end = l. input . cur_pos ( ) ;
829
+ let s = unsafe {
830
+ // Safety: start and end are valid position because we got them from
831
+ // `self.input`
832
+ l. input . slice ( slice_start, start)
833
+ } ;
834
+ buf. push_str ( s) ;
835
+ unsafe {
836
+ // Safety: We got end from `self.input`
837
+ l. input . reset_to ( end) ;
838
+ }
839
+ }
839
840
840
841
let chars = l. read_unicode_escape ( ) ?;
841
842
@@ -854,14 +855,56 @@ impl<'a> Lexer<'a> {
854
855
for c in chars {
855
856
buf. extend ( c) ;
856
857
}
858
+
859
+ slice_start = l. cur_pos ( ) ;
860
+ continue ;
857
861
}
858
- _ => {
859
- break ;
862
+
863
+ // ASCII but not a valid identifier
864
+
865
+ break ;
866
+ }
867
+
868
+ if let Some ( c) = l. input . cur ( ) {
869
+ if Ident :: is_valid_continue ( c) {
870
+ l. bump ( ) ;
871
+ continue ;
872
+ } else if first && Ident :: is_valid_start ( c) {
873
+ l. bump ( ) ;
874
+ first = false ;
875
+ continue ;
860
876
}
861
877
}
862
- first = false ;
878
+
879
+ break ;
863
880
}
864
- let value = convert ( l, buf, has_escape, can_be_keyword) ;
881
+
882
+ let end = l. cur_pos ( ) ;
883
+
884
+ let value = if !has_escape {
885
+ // Fast path: raw slice is enough if there's no escape.
886
+
887
+ let s = unsafe {
888
+ // Safety: slice_start and end are valid position because we got them from
889
+ // `self.input`
890
+ l. input . slice ( slice_start, end)
891
+ } ;
892
+ let s = unsafe {
893
+ // Safety: We don't use 'static. We just bypass the lifetime check.
894
+ transmute :: < & str , & ' static str > ( s)
895
+ } ;
896
+
897
+ convert ( l, s, has_escape, can_be_keyword)
898
+ } else {
899
+ let s = unsafe {
900
+ // Safety: slice_start and end are valid position because we got them from
901
+ // `self.input`
902
+ l. input . slice ( slice_start, end)
903
+ } ;
904
+ buf. push_str ( s) ;
905
+
906
+ convert ( l, buf, has_escape, can_be_keyword)
907
+ } ;
865
908
866
909
Ok ( ( value, has_escape) )
867
910
} )
1 commit comments
github-actions[bot] commentedon Jun 18, 2024
Benchmark
es/full/bugs-1
278153
ns/iter (± 9089
)273315
ns/iter (± 13666
)1.02
es/full/minify/libraries/antd
1819198281
ns/iter (± 57720988
)1875331502
ns/iter (± 48111854
)0.97
es/full/minify/libraries/d3
345031970
ns/iter (± 4045546
)354431576
ns/iter (± 4487176
)0.97
es/full/minify/libraries/echarts
1523908003
ns/iter (± 15617743
)1587068452
ns/iter (± 24860863
)0.96
es/full/minify/libraries/jquery
102081200
ns/iter (± 578867
)104756977
ns/iter (± 596776
)0.97
es/full/minify/libraries/lodash
113603995
ns/iter (± 154691
)116361001
ns/iter (± 863675
)0.98
es/full/minify/libraries/moment
59254132
ns/iter (± 198043
)60229654
ns/iter (± 197422
)0.98
es/full/minify/libraries/react
18326213
ns/iter (± 37822
)18514501
ns/iter (± 68139
)0.99
es/full/minify/libraries/terser
274221608
ns/iter (± 1854787
)294793222
ns/iter (± 5755381
)0.93
es/full/minify/libraries/three
473906951
ns/iter (± 4937534
)500099343
ns/iter (± 6031104
)0.95
es/full/minify/libraries/typescript
3253823797
ns/iter (± 49622131
)3283670699
ns/iter (± 38211593
)0.99
es/full/minify/libraries/victory
768963122
ns/iter (± 11833889
)800527265
ns/iter (± 30158638
)0.96
es/full/minify/libraries/vue
145790166
ns/iter (± 779753
)149154830
ns/iter (± 778973
)0.98
es/full/codegen/es3
58904
ns/iter (± 128
)59670
ns/iter (± 464
)0.99
es/full/codegen/es5
59735
ns/iter (± 906
)59701
ns/iter (± 170
)1.00
es/full/codegen/es2015
59024
ns/iter (± 224
)59838
ns/iter (± 132
)0.99
es/full/codegen/es2016
59108
ns/iter (± 86
)59775
ns/iter (± 916
)0.99
es/full/codegen/es2017
59153
ns/iter (± 235
)59755
ns/iter (± 151
)0.99
es/full/codegen/es2018
59128
ns/iter (± 129
)60325
ns/iter (± 442
)0.98
es/full/codegen/es2019
59211
ns/iter (± 133
)60214
ns/iter (± 360
)0.98
es/full/codegen/es2020
59390
ns/iter (± 883
)60755
ns/iter (± 232
)0.98
es/full/all/es3
160882837
ns/iter (± 1055856
)162322991
ns/iter (± 912394
)0.99
es/full/all/es5
156138775
ns/iter (± 978958
)156882552
ns/iter (± 1064118
)1.00
es/full/all/es2015
116509608
ns/iter (± 558051
)116576381
ns/iter (± 794122
)1.00
es/full/all/es2016
114736971
ns/iter (± 1211669
)115321560
ns/iter (± 722814
)0.99
es/full/all/es2017
115036255
ns/iter (± 880117
)115480182
ns/iter (± 752673
)1.00
es/full/all/es2018
112525221
ns/iter (± 666768
)112385488
ns/iter (± 634677
)1.00
es/full/all/es2019
112457058
ns/iter (± 871171
)111802739
ns/iter (± 517531
)1.01
es/full/all/es2020
108333577
ns/iter (± 705811
)108225624
ns/iter (± 723372
)1.00
es/full/parser
462823
ns/iter (± 2424
)466283
ns/iter (± 2069
)0.99
es/full/base/fixer
16418
ns/iter (± 329
)16208
ns/iter (± 196
)1.01
es/full/base/resolver_and_hygiene
81722
ns/iter (± 709
)81098
ns/iter (± 146
)1.01
serialization of serde
271
ns/iter (± 2
)276
ns/iter (± 6
)0.98
css/minify/libraries/bootstrap
22277915
ns/iter (± 75848
)22377661
ns/iter (± 166018
)1.00
css/visitor/compare/clone
1398600
ns/iter (± 9071
)1398601
ns/iter (± 18530
)1.00
css/visitor/compare/visit_mut_span
1521644
ns/iter (± 7204
)1535754
ns/iter (± 20597
)0.99
css/visitor/compare/visit_mut_span_panic
1546027
ns/iter (± 34982
)1542691
ns/iter (± 29365
)1.00
css/visitor/compare/fold_span
2056693
ns/iter (± 32781
)2036408
ns/iter (± 11176
)1.01
css/visitor/compare/fold_span_panic
2165633
ns/iter (± 8747
)2163000
ns/iter (± 34252
)1.00
css/lexer/bootstrap_5_1_3
3441904
ns/iter (± 2695
)3442449
ns/iter (± 47200
)1.00
css/lexer/foundation_6_7_4
2786676
ns/iter (± 4038
)2792809
ns/iter (± 7793
)1.00
css/lexer/tailwind_3_1_1
535315
ns/iter (± 135
)532626
ns/iter (± 28963
)1.01
css/parser/bootstrap_5_1_3
15964546
ns/iter (± 224885
)16221265
ns/iter (± 107573
)0.98
css/parser/foundation_6_7_4
12726858
ns/iter (± 19235
)13040985
ns/iter (± 195431
)0.98
css/parser/tailwind_3_1_1
2527643
ns/iter (± 4956
)2533231
ns/iter (± 38561
)1.00
es/codegen/colors
729504
ns/iter (± 395515
)729550
ns/iter (± 402946
)1.00
es/codegen/large
3032653
ns/iter (± 1601838
)3055805
ns/iter (± 1614404
)0.99
es/codegen/with-parser/colors
42375
ns/iter (± 800
)42045
ns/iter (± 222
)1.01
es/codegen/with-parser/large
459716
ns/iter (± 4122
)455895
ns/iter (± 3960
)1.01
es/minify/libraries/antd
1456080207
ns/iter (± 26856361
)1438903021
ns/iter (± 31497960
)1.01
es/minify/libraries/d3
293672441
ns/iter (± 5058883
)282781904
ns/iter (± 5193951
)1.04
es/minify/libraries/echarts
1220861891
ns/iter (± 24815249
)1161394999
ns/iter (± 15118643
)1.05
es/minify/libraries/jquery
82247533
ns/iter (± 301289
)82594751
ns/iter (± 220033
)1.00
es/minify/libraries/lodash
103991985
ns/iter (± 462906
)104361795
ns/iter (± 366858
)1.00
es/minify/libraries/moment
48306268
ns/iter (± 113007
)48132569
ns/iter (± 125636
)1.00
es/minify/libraries/react
16274126
ns/iter (± 60968
)16168719
ns/iter (± 15812
)1.01
es/minify/libraries/terser
219614293
ns/iter (± 2830842
)217384124
ns/iter (± 279040
)1.01
es/minify/libraries/three
370923728
ns/iter (± 4127582
)362936763
ns/iter (± 1070722
)1.02
es/minify/libraries/typescript
2807390861
ns/iter (± 16398799
)2709351465
ns/iter (± 25866724
)1.04
es/minify/libraries/victory
591629433
ns/iter (± 13037282
)577062492
ns/iter (± 10963378
)1.03
es/minify/libraries/vue
121517015
ns/iter (± 421651
)121019079
ns/iter (± 363143
)1.00
es/visitor/compare/clone
1685016
ns/iter (± 7173
)1716597
ns/iter (± 24197
)0.98
es/visitor/compare/visit_mut_span
2030665
ns/iter (± 17658
)2066102
ns/iter (± 30572
)0.98
es/visitor/compare/visit_mut_span_panic
2023440
ns/iter (± 35945
)2050179
ns/iter (± 71382
)0.99
es/visitor/compare/fold_span
2770661
ns/iter (± 21714
)2768895
ns/iter (± 10236
)1.00
es/visitor/compare/fold_span_panic
2796537
ns/iter (± 18626
)2802691
ns/iter (± 6067
)1.00
es/lexer/colors
9111
ns/iter (± 69
)9779
ns/iter (± 85
)0.93
es/lexer/angular
4740428
ns/iter (± 29379
)5245044
ns/iter (± 23059
)0.90
es/lexer/backbone
634441
ns/iter (± 1614
)637692
ns/iter (± 3779
)0.99
es/lexer/jquery
3485495
ns/iter (± 7102
)3718545
ns/iter (± 26902
)0.94
es/lexer/jquery mobile
5248554
ns/iter (± 35276
)5652660
ns/iter (± 7681
)0.93
es/lexer/mootools
2730502
ns/iter (± 17706
)2860997
ns/iter (± 1648
)0.95
es/lexer/underscore
510412
ns/iter (± 1440
)538161
ns/iter (± 613
)0.95
es/lexer/three
15429692
ns/iter (± 87241
)16259224
ns/iter (± 69600
)0.95
es/lexer/yui
2807780
ns/iter (± 6560
)2989888
ns/iter (± 12024
)0.94
es/lexer/cal-com
12498953
ns/iter (± 65691
)12826096
ns/iter (± 17368
)0.97
es/lexer/typescript
87439926
ns/iter (± 180063
)90786058
ns/iter (± 447925
)0.96
es/parser/colors
23688
ns/iter (± 75
)24644
ns/iter (± 77
)0.96
es/parser/angular
12474519
ns/iter (± 89962
)12476036
ns/iter (± 56728
)1.00
es/parser/backbone
1788087
ns/iter (± 7356
)1843279
ns/iter (± 11856
)0.97
es/parser/jquery
9944892
ns/iter (± 84324
)10111584
ns/iter (± 77560
)0.98
es/parser/jquery mobile
15037262
ns/iter (± 175004
)15323886
ns/iter (± 83830
)0.98
es/parser/mootools
7703896
ns/iter (± 62622
)7907871
ns/iter (± 31436
)0.97
es/parser/underscore
1570045
ns/iter (± 5931
)1608133
ns/iter (± 4608
)0.98
es/parser/three
45075102
ns/iter (± 930870
)44386499
ns/iter (± 667571
)1.02
es/parser/yui
7377172
ns/iter (± 31375
)7465653
ns/iter (± 38237
)0.99
es/parser/cal-com
42191728
ns/iter (± 1221076
)39841493
ns/iter (± 141524
)1.06
es/parser/typescript
268219669
ns/iter (± 3380804
)267590754
ns/iter (± 867934
)1.00
es/preset-env/usage/builtin_type
140153
ns/iter (± 34435
)135913
ns/iter (± 34839
)1.03
es/preset-env/usage/property
16165
ns/iter (± 74
)16532
ns/iter (± 57
)0.98
es/resolver/typescript
106125080
ns/iter (± 3003335
)102170531
ns/iter (± 3317082
)1.04
es/fixer/typescript
85720375
ns/iter (± 3249375
)81553860
ns/iter (± 3098715
)1.05
es/hygiene/typescript
181387749
ns/iter (± 2998280
)190743227
ns/iter (± 7231408
)0.95
es/resolver_with_hygiene/typescript
297306926
ns/iter (± 3777129
)310795961
ns/iter (± 3623176
)0.96
es/visitor/base-perf/module_clone
57379
ns/iter (± 1449
)58044
ns/iter (± 2045
)0.99
es/visitor/base-perf/fold_empty
60324
ns/iter (± 542
)61310
ns/iter (± 816
)0.98
es/visitor/base-perf/fold_noop_impl_all
60005
ns/iter (± 533
)61413
ns/iter (± 741
)0.98
es/visitor/base-perf/fold_noop_impl_vec
60415
ns/iter (± 727
)61898
ns/iter (± 1201
)0.98
es/visitor/base-perf/boxing_boxed_clone
59
ns/iter (± 0
)60
ns/iter (± 1
)0.98
es/visitor/base-perf/boxing_unboxed_clone
47
ns/iter (± 0
)47
ns/iter (± 0
)1
es/visitor/base-perf/boxing_boxed
105
ns/iter (± 0
)105
ns/iter (± 0
)1
es/visitor/base-perf/boxing_unboxed
81
ns/iter (± 0
)80
ns/iter (± 0
)1.01
es/visitor/base-perf/visit_contains_this
2468
ns/iter (± 9
)2527
ns/iter (± 23
)0.98
es/base/parallel/resolver/typescript
2425209465
ns/iter (± 110957191
)2307668317
ns/iter (± 148917439
)1.05
es/base/parallel/hygiene/typescript
3276197503
ns/iter (± 86818197
)3197996952
ns/iter (± 66385536
)1.02
babelify-only
479110
ns/iter (± 4389
)475654
ns/iter (± 13969
)1.01
parse_and_babelify_angular
42403348
ns/iter (± 1914563
)42555573
ns/iter (± 1067590
)1.00
parse_and_babelify_backbone
4259849
ns/iter (± 61517
)4336189
ns/iter (± 24888
)0.98
parse_and_babelify_jquery
29416897
ns/iter (± 679298
)28397292
ns/iter (± 202083
)1.04
parse_and_babelify_jquery_mobile
51258248
ns/iter (± 988172
)48732674
ns/iter (± 668160
)1.05
parse_and_babelify_mootools
25937939
ns/iter (± 257035
)25478322
ns/iter (± 148549
)1.02
parse_and_babelify_underscore
3607959
ns/iter (± 34402
)3675846
ns/iter (± 15368
)0.98
parse_and_babelify_yui
25995895
ns/iter (± 1675827
)23380429
ns/iter (± 218225
)1.11
html/minify/document/css_spec
46850633
ns/iter (± 467019
)46850237
ns/iter (± 533417
)1.00
html/minify/document/github
18687386
ns/iter (± 112722
)18562822
ns/iter (± 59242
)1.01
html/minify/document/stackoverflow
16840126
ns/iter (± 333626
)16952254
ns/iter (± 433898
)0.99
html/minify/document_fragment/css_spec
45318798
ns/iter (± 266835
)47979371
ns/iter (± 529780
)0.94
html/minify/document_fragment/github
18198479
ns/iter (± 42169
)18414403
ns/iter (± 78804
)0.99
html/minify/document_fragment/stackoverflow
16466728
ns/iter (± 101771
)16781322
ns/iter (± 49917
)0.98
html/document/visitor/compare/clone
165604
ns/iter (± 1318
)167704
ns/iter (± 2485
)0.99
html/document/visitor/compare/visit_mut_span
184068
ns/iter (± 3443
)182054
ns/iter (± 1824
)1.01
html/document/visitor/compare/visit_mut_span_panic
187603
ns/iter (± 2257
)186077
ns/iter (± 4278
)1.01
html/document/visitor/compare/fold_span
229585
ns/iter (± 1512
)227959
ns/iter (± 1145
)1.01
html/document/visitor/compare/fold_span_panic
281709
ns/iter (± 1657
)280083
ns/iter (± 845
)1.01
html/document_fragment/visitor/compare/clone
167717
ns/iter (± 1655
)164404
ns/iter (± 1456
)1.02
html/document_fragment/visitor/compare/visit_mut_span
182067
ns/iter (± 1306
)184777
ns/iter (± 1110
)0.99
html/document_fragment/visitor/compare/visit_mut_span_panic
190272
ns/iter (± 2010
)188851
ns/iter (± 1184
)1.01
html/document_fragment/visitor/compare/fold_span
228697
ns/iter (± 699
)229400
ns/iter (± 1130
)1.00
html/document_fragment/visitor/compare/fold_span_panic
280150
ns/iter (± 1532
)279645
ns/iter (± 1247
)1.00
html/lexer/css_2021_spec
12906440
ns/iter (± 32882
)12948537
ns/iter (± 85065
)1.00
html/lexer/github_com_17_05_2022
4958124
ns/iter (± 10902
)4927845
ns/iter (± 6181
)1.01
html/lexer/stackoverflow_com_17_05_2022
4689983
ns/iter (± 6467
)4681862
ns/iter (± 15539
)1.00
html/parser/parser_document/css_2021_spec
25675345
ns/iter (± 251245
)25316715
ns/iter (± 80065
)1.01
html/parser/parser_document/github_com_17_05_2022
8544762
ns/iter (± 29953
)8528045
ns/iter (± 77810
)1.00
html/parser/parser_document/stackoverflow_com_17_05_2022
7480302
ns/iter (± 19136
)7461070
ns/iter (± 44005
)1.00
html/parser/parser_document_fragment/css_2021_spec
25330683
ns/iter (± 109620
)25174648
ns/iter (± 101167
)1.01
html/parser/parser_document_fragment/github_com_17_05_2022
8456146
ns/iter (± 12883
)8457509
ns/iter (± 35072
)1.00
html/parser/parser_document_fragment/stackoverflow_com_17_05_2022
7427630
ns/iter (± 29160
)7453820
ns/iter (± 7830
)1.00
This comment was automatically generated by workflow using github-action-benchmark.