diff --git a/CHANGELOG.md b/CHANGELOG.md index f735d43be18..4a890c6d138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Unreleased + +* Add support for ES5-style identifiers ([#1349](https://github.com/evanw/esbuild/issues/1349)) + + In ES6 and above, an identifier is a character sequence starting with a character in the `ID_Start` Unicode category and followed by zero or more characters in the `ID_Continue` Unicode category, and these categories must be drawn from Unicode version 5.1 or above. + + But in ES5, an identifier is a character sequence starting with a character in one of the `Lu, Ll, Lt, Lm, Lo, Nl` Unicode categories and followed by zero or more characters in the `Lu, Ll, Lt, Lm, Lo, Nl, Mn, Mc, Nd, Pc` Unicode categories, and these categories must be drawn from Unicode version 3.0. + + Previously esbuild always used the ES6+ identifier validation test but with this release, it will use the ES5 validation test when the target environment doesn't support ES6+ identifiers. This identifier validation test is used in decisions related to code printing. For example: + + ``` + $ echo x.ꓷꓶꓲꓵꓭꓢꓱ | ./esbuild --charset=utf8 + x.ꓷꓶꓲꓵꓭꓢꓱ; + + $ echo x.ꓷꓶꓲꓵꓭꓢꓱ | ./esbuild --charset=utf8 --target=es5 + x["ꓷꓶꓲꓵꓭꓢꓱ"]; + ``` + ## 0.12.6 * Improve template literal lowering transformation conformance ([#1327](https://github.com/evanw/esbuild/issues/1327)) diff --git a/internal/bundler/linker.go b/internal/bundler/linker.go index bfa71d54ea5..c18a3af9934 100644 --- a/internal/bundler/linker.go +++ b/internal/bundler/linker.go @@ -1646,7 +1646,7 @@ func (c *linkerContext) generateCodeForLazyExport(sourceIndex uint32) { clone.Properties = append(make([]js_ast.Property, 0, len(clone.Properties)), clone.Properties...) for i, property := range clone.Properties { if str, ok := property.Key.Data.(*js_ast.EString); ok && - (!file.IsEntryPoint() || js_lexer.IsIdentifierUTF16(str.Value) || + (!file.IsEntryPoint() || js_lexer.IsIdentifierUTF16(str.Value, 0) || !c.options.UnsupportedJSFeatures.Has(compat.ArbitraryModuleNamespaceNames)) { name := js_lexer.UTF16ToString(str.Value) exportRef := generateExport(name, name, property.ValueOrNil).ref @@ -4168,7 +4168,7 @@ func (c *linkerContext) renameSymbolsInChunk(chunk *chunkInfo, filesInOrder []ui } // When we're not minifying, just append numbers to symbol names to avoid collisions - r := renamer.NewNumberRenamer(c.graph.Symbols, reservedNames) + r := renamer.NewNumberRenamer(c.graph.Symbols, reservedNames, c.options.UnsupportedJSFeatures) nestedScopes := make(map[uint32][]*js_ast.Scope) timer.Begin("Add top-level symbols") diff --git a/internal/compat/js_table.go b/internal/compat/js_table.go index 425a8f896ca..f1c80ca1de7 100644 --- a/internal/compat/js_table.go +++ b/internal/compat/js_table.go @@ -79,6 +79,7 @@ const ( TemplateLiteral TopLevelAwait UnicodeEscapes + UpdatedIdentifiers ) func (features JSFeature) Has(feature JSFeature) bool { @@ -417,6 +418,15 @@ var jsTable = map[JSFeature]map[Engine][]int{ Node: {4}, Safari: {9}, }, + UpdatedIdentifiers: { + Chrome: {58}, + Edge: {14}, + ES: {2015}, + Firefox: {52}, + IOS: {14}, + Node: {8}, + Safari: {14}, + }, } func isVersionLessThan(a []int, b []int) bool { diff --git a/internal/js_lexer/js_lexer.go b/internal/js_lexer/js_lexer.go index fc92e6d83ae..d66dcc8c3cb 100644 --- a/internal/js_lexer/js_lexer.go +++ b/internal/js_lexer/js_lexer.go @@ -20,6 +20,7 @@ import ( "unicode" "unicode/utf8" + "github.com/evanw/esbuild/internal/compat" "github.com/evanw/esbuild/internal/js_ast" "github.com/evanw/esbuild/internal/logger" ) @@ -559,17 +560,17 @@ func (lexer *Lexer) maybeExpandEquals() { } } -func IsIdentifier(text string) bool { +func IsIdentifier(text string, unsupportedJSFeatures compat.JSFeature) bool { if len(text) == 0 { return false } for i, codePoint := range text { if i == 0 { - if !IsIdentifierStart(codePoint) { + if !IsIdentifierStart(codePoint, unsupportedJSFeatures) { return false } } else { - if !IsIdentifierContinue(codePoint) { + if !IsIdentifierContinue(codePoint, unsupportedJSFeatures) { return false } } @@ -577,8 +578,8 @@ func IsIdentifier(text string) bool { return true } -func ForceValidIdentifier(text string) string { - if IsIdentifier(text) { +func ForceValidIdentifier(text string, unsupportedJSFeatures compat.JSFeature) string { + if IsIdentifier(text, unsupportedJSFeatures) { return text } sb := strings.Builder{} @@ -586,7 +587,7 @@ func ForceValidIdentifier(text string) string { // Identifier start c, width := utf8.DecodeRuneInString(text) text = text[width:] - if IsIdentifierStart(c) { + if IsIdentifierStart(c, unsupportedJSFeatures) { sb.WriteRune(c) } else { sb.WriteRune('_') @@ -596,7 +597,7 @@ func ForceValidIdentifier(text string) string { for text != "" { c, width := utf8.DecodeRuneInString(text) text = text[width:] - if IsIdentifierContinue(c) { + if IsIdentifierContinue(c, unsupportedJSFeatures) { sb.WriteRune(c) } else { sb.WriteRune('_') @@ -607,7 +608,7 @@ func ForceValidIdentifier(text string) string { } // This does "IsIdentifier(UTF16ToString(text))" without any allocations -func IsIdentifierUTF16(text []uint16) bool { +func IsIdentifierUTF16(text []uint16, unsupportedJSFeatures compat.JSFeature) bool { n := len(text) if n == 0 { return false @@ -622,11 +623,11 @@ func IsIdentifierUTF16(text []uint16) bool { } } if isStart { - if !IsIdentifierStart(r1) { + if !IsIdentifierStart(r1, unsupportedJSFeatures) { return false } } else { - if !IsIdentifierContinue(r1) { + if !IsIdentifierContinue(r1, unsupportedJSFeatures) { return false } } @@ -634,7 +635,7 @@ func IsIdentifierUTF16(text []uint16) bool { return true } -func IsIdentifierStart(codePoint rune) bool { +func IsIdentifierStart(codePoint rune, unsupportedJSFeatures compat.JSFeature) bool { switch codePoint { case '_', '$', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', @@ -649,10 +650,14 @@ func IsIdentifierStart(codePoint rune) bool { return false } - return unicode.Is(idStart, codePoint) + if unsupportedJSFeatures.Has(compat.UpdatedIdentifiers) { + return unicode.Is(idStartES5, codePoint) + } else { + return unicode.Is(idStart, codePoint) + } } -func IsIdentifierContinue(codePoint rune) bool { +func IsIdentifierContinue(codePoint rune, unsupportedJSFeatures compat.JSFeature) bool { switch codePoint { case '_', '$', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', @@ -672,7 +677,11 @@ func IsIdentifierContinue(codePoint rune) bool { return true } - return unicode.Is(idContinue, codePoint) + if unsupportedJSFeatures.Has(compat.UpdatedIdentifiers) { + return unicode.Is(idContinueES5, codePoint) + } else { + return unicode.Is(idContinue, codePoint) + } } // See the "White Space Code Points" table in the ECMAScript standard @@ -725,7 +734,7 @@ func RangeOfIdentifier(source logger.Source, loc logger.Loc) logger.Range { c, _ = utf8.DecodeRuneInString(text[i:]) } - if IsIdentifierStart(c) || c == '\\' { + if IsIdentifierStart(c, 0) || c == '\\' { // Search for the end of the identifier for i < len(text) { c2, width2 := utf8.DecodeRuneInString(text[i:]) @@ -743,7 +752,7 @@ func RangeOfIdentifier(source logger.Source, loc logger.Loc) logger.Range { i++ } } - } else if !IsIdentifierContinue(c2) { + } else if !IsIdentifierContinue(c2, 0) { return logger.Range{Loc: loc, Len: int32(i)} } else { i += width2 @@ -1001,9 +1010,9 @@ func (lexer *Lexer) NextInsideJSXElement() { continue } - if IsIdentifierStart(lexer.codePoint) { + if IsIdentifierStart(lexer.codePoint, 0) { lexer.step() - for IsIdentifierContinue(lexer.codePoint) || lexer.codePoint == '-' { + for IsIdentifierContinue(lexer.codePoint, 0) || lexer.codePoint == '-' { lexer.step() } @@ -1013,9 +1022,9 @@ func (lexer *Lexer) NextInsideJSXElement() { // can't use this feature to reference JavaScript identifiers. if lexer.codePoint == ':' { lexer.step() - if IsIdentifierStart(lexer.codePoint) { + if IsIdentifierStart(lexer.codePoint, 0) { lexer.step() - for IsIdentifierContinue(lexer.codePoint) || lexer.codePoint == '-' { + for IsIdentifierContinue(lexer.codePoint, 0) || lexer.codePoint == '-' { lexer.step() } } else { @@ -1073,11 +1082,11 @@ func (lexer *Lexer) Next() { if lexer.codePoint == '\\' { lexer.Identifier, _ = lexer.scanIdentifierWithEscapes(privateIdentifier) } else { - if !IsIdentifierStart(lexer.codePoint) { + if !IsIdentifierStart(lexer.codePoint, 0) { lexer.SyntaxError() } lexer.step() - for IsIdentifierContinue(lexer.codePoint) { + for IsIdentifierContinue(lexer.codePoint, 0) { lexer.step() } if lexer.codePoint == '\\' { @@ -1592,7 +1601,7 @@ func (lexer *Lexer) Next() { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': lexer.step() - for IsIdentifierContinue(lexer.codePoint) { + for IsIdentifierContinue(lexer.codePoint, 0) { lexer.step() } if lexer.codePoint == '\\' { @@ -1619,9 +1628,9 @@ func (lexer *Lexer) Next() { continue } - if IsIdentifierStart(lexer.codePoint) { + if IsIdentifierStart(lexer.codePoint, 0) { lexer.step() - for IsIdentifierContinue(lexer.codePoint) { + for IsIdentifierContinue(lexer.codePoint, 0) { lexer.step() } if lexer.codePoint == '\\' { @@ -1692,7 +1701,7 @@ func (lexer *Lexer) scanIdentifierWithEscapes(kind identifierKind) (string, T) { } // Stop when we reach the end of the identifier - if !IsIdentifierContinue(lexer.codePoint) { + if !IsIdentifierContinue(lexer.codePoint, 0) { break } lexer.step() @@ -1711,7 +1720,7 @@ func (lexer *Lexer) scanIdentifierWithEscapes(kind identifierKind) (string, T) { if kind == privateIdentifier { identifier = identifier[1:] // Skip over the "#" } - if !IsIdentifier(identifier) { + if !IsIdentifier(identifier, 0) { lexer.addRangeError(logger.Range{Loc: logger.Loc{Start: int32(lexer.start)}, Len: int32(lexer.end - lexer.start)}, fmt.Sprintf("Invalid identifier: %q", text)) } @@ -2029,7 +2038,7 @@ func (lexer *Lexer) parseNumericLiteralOrDot() { } // Identifiers can't occur immediately after numbers - if IsIdentifierStart(lexer.codePoint) { + if IsIdentifierStart(lexer.codePoint, 0) { lexer.SyntaxError() } } @@ -2058,7 +2067,7 @@ func (lexer *Lexer) ScanRegExp() { case '/': lexer.step() bits := uint32(0) - for IsIdentifierContinue(lexer.codePoint) { + for IsIdentifierContinue(lexer.codePoint, 0) { switch lexer.codePoint { case 'g', 'i', 'm', 's', 'u', 'y': bit := uint32(1) << uint32(lexer.codePoint-'a') @@ -2528,7 +2537,7 @@ func hasPrefixWithWordBoundary(text string, prefix string) bool { return true } c, _ := utf8.DecodeRuneInString(text[p:]) - if !IsIdentifierContinue(c) { + if !IsIdentifierContinue(c, 0) { return true } } diff --git a/internal/js_lexer/unicode.go b/internal/js_lexer/unicode.go index 3242149afec..49ff38fd64a 100644 --- a/internal/js_lexer/unicode.go +++ b/internal/js_lexer/unicode.go @@ -3,6 +3,626 @@ package js_lexer import "unicode" +var idStartES5 = &unicode.RangeTable{ + LatinOffset: 117, + R16: []unicode.Range16{ + {Lo: 0x41, Hi: 0x5a, Stride: 1}, + {Lo: 0x61, Hi: 0x7a, Stride: 1}, + {Lo: 0xaa, Hi: 0xaa, Stride: 1}, + {Lo: 0xb5, Hi: 0xb5, Stride: 1}, + {Lo: 0xba, Hi: 0xba, Stride: 1}, + {Lo: 0xc0, Hi: 0xd6, Stride: 1}, + {Lo: 0xd8, Hi: 0xf6, Stride: 1}, + {Lo: 0xf8, Hi: 0x21f, Stride: 1}, + {Lo: 0x222, Hi: 0x233, Stride: 1}, + {Lo: 0x250, Hi: 0x2ad, Stride: 1}, + {Lo: 0x2b0, Hi: 0x2b8, Stride: 1}, + {Lo: 0x2bb, Hi: 0x2c1, Stride: 1}, + {Lo: 0x2d0, Hi: 0x2d1, Stride: 1}, + {Lo: 0x2e0, Hi: 0x2e4, Stride: 1}, + {Lo: 0x2ee, Hi: 0x2ee, Stride: 1}, + {Lo: 0x37a, Hi: 0x37a, Stride: 1}, + {Lo: 0x386, Hi: 0x386, Stride: 1}, + {Lo: 0x388, Hi: 0x38a, Stride: 1}, + {Lo: 0x38c, Hi: 0x38c, Stride: 1}, + {Lo: 0x38e, Hi: 0x3a1, Stride: 1}, + {Lo: 0x3a3, Hi: 0x3ce, Stride: 1}, + {Lo: 0x3d0, Hi: 0x3d7, Stride: 1}, + {Lo: 0x3da, Hi: 0x3f3, Stride: 1}, + {Lo: 0x400, Hi: 0x481, Stride: 1}, + {Lo: 0x48c, Hi: 0x4c4, Stride: 1}, + {Lo: 0x4c7, Hi: 0x4c8, Stride: 1}, + {Lo: 0x4cb, Hi: 0x4cc, Stride: 1}, + {Lo: 0x4d0, Hi: 0x4f5, Stride: 1}, + {Lo: 0x4f8, Hi: 0x4f9, Stride: 1}, + {Lo: 0x531, Hi: 0x556, Stride: 1}, + {Lo: 0x559, Hi: 0x559, Stride: 1}, + {Lo: 0x561, Hi: 0x587, Stride: 1}, + {Lo: 0x5d0, Hi: 0x5ea, Stride: 1}, + {Lo: 0x5f0, Hi: 0x5f2, Stride: 1}, + {Lo: 0x621, Hi: 0x63a, Stride: 1}, + {Lo: 0x640, Hi: 0x64a, Stride: 1}, + {Lo: 0x671, Hi: 0x6d3, Stride: 1}, + {Lo: 0x6d5, Hi: 0x6d5, Stride: 1}, + {Lo: 0x6e5, Hi: 0x6e6, Stride: 1}, + {Lo: 0x6fa, Hi: 0x6fc, Stride: 1}, + {Lo: 0x710, Hi: 0x710, Stride: 1}, + {Lo: 0x712, Hi: 0x72c, Stride: 1}, + {Lo: 0x780, Hi: 0x7a5, Stride: 1}, + {Lo: 0x905, Hi: 0x939, Stride: 1}, + {Lo: 0x93d, Hi: 0x93d, Stride: 1}, + {Lo: 0x950, Hi: 0x950, Stride: 1}, + {Lo: 0x958, Hi: 0x961, Stride: 1}, + {Lo: 0x985, Hi: 0x98c, Stride: 1}, + {Lo: 0x98f, Hi: 0x990, Stride: 1}, + {Lo: 0x993, Hi: 0x9a8, Stride: 1}, + {Lo: 0x9aa, Hi: 0x9b0, Stride: 1}, + {Lo: 0x9b2, Hi: 0x9b2, Stride: 1}, + {Lo: 0x9b6, Hi: 0x9b9, Stride: 1}, + {Lo: 0x9dc, Hi: 0x9dd, Stride: 1}, + {Lo: 0x9df, Hi: 0x9e1, Stride: 1}, + {Lo: 0x9f0, Hi: 0x9f1, Stride: 1}, + {Lo: 0xa05, Hi: 0xa0a, Stride: 1}, + {Lo: 0xa0f, Hi: 0xa10, Stride: 1}, + {Lo: 0xa13, Hi: 0xa28, Stride: 1}, + {Lo: 0xa2a, Hi: 0xa30, Stride: 1}, + {Lo: 0xa32, Hi: 0xa33, Stride: 1}, + {Lo: 0xa35, Hi: 0xa36, Stride: 1}, + {Lo: 0xa38, Hi: 0xa39, Stride: 1}, + {Lo: 0xa59, Hi: 0xa5c, Stride: 1}, + {Lo: 0xa5e, Hi: 0xa5e, Stride: 1}, + {Lo: 0xa72, Hi: 0xa74, Stride: 1}, + {Lo: 0xa85, Hi: 0xa8b, Stride: 1}, + {Lo: 0xa8d, Hi: 0xa8d, Stride: 1}, + {Lo: 0xa8f, Hi: 0xa91, Stride: 1}, + {Lo: 0xa93, Hi: 0xaa8, Stride: 1}, + {Lo: 0xaaa, Hi: 0xab0, Stride: 1}, + {Lo: 0xab2, Hi: 0xab3, Stride: 1}, + {Lo: 0xab5, Hi: 0xab9, Stride: 1}, + {Lo: 0xabd, Hi: 0xabd, Stride: 1}, + {Lo: 0xad0, Hi: 0xad0, Stride: 1}, + {Lo: 0xae0, Hi: 0xae0, Stride: 1}, + {Lo: 0xb05, Hi: 0xb0c, Stride: 1}, + {Lo: 0xb0f, Hi: 0xb10, Stride: 1}, + {Lo: 0xb13, Hi: 0xb28, Stride: 1}, + {Lo: 0xb2a, Hi: 0xb30, Stride: 1}, + {Lo: 0xb32, Hi: 0xb33, Stride: 1}, + {Lo: 0xb36, Hi: 0xb39, Stride: 1}, + {Lo: 0xb3d, Hi: 0xb3d, Stride: 1}, + {Lo: 0xb5c, Hi: 0xb5d, Stride: 1}, + {Lo: 0xb5f, Hi: 0xb61, Stride: 1}, + {Lo: 0xb85, Hi: 0xb8a, Stride: 1}, + {Lo: 0xb8e, Hi: 0xb90, Stride: 1}, + {Lo: 0xb92, Hi: 0xb95, Stride: 1}, + {Lo: 0xb99, Hi: 0xb9a, Stride: 1}, + {Lo: 0xb9c, Hi: 0xb9c, Stride: 1}, + {Lo: 0xb9e, Hi: 0xb9f, Stride: 1}, + {Lo: 0xba3, Hi: 0xba4, Stride: 1}, + {Lo: 0xba8, Hi: 0xbaa, Stride: 1}, + {Lo: 0xbae, Hi: 0xbb5, Stride: 1}, + {Lo: 0xbb7, Hi: 0xbb9, Stride: 1}, + {Lo: 0xc05, Hi: 0xc0c, Stride: 1}, + {Lo: 0xc0e, Hi: 0xc10, Stride: 1}, + {Lo: 0xc12, Hi: 0xc28, Stride: 1}, + {Lo: 0xc2a, Hi: 0xc33, Stride: 1}, + {Lo: 0xc35, Hi: 0xc39, Stride: 1}, + {Lo: 0xc60, Hi: 0xc61, Stride: 1}, + {Lo: 0xc85, Hi: 0xc8c, Stride: 1}, + {Lo: 0xc8e, Hi: 0xc90, Stride: 1}, + {Lo: 0xc92, Hi: 0xca8, Stride: 1}, + {Lo: 0xcaa, Hi: 0xcb3, Stride: 1}, + {Lo: 0xcb5, Hi: 0xcb9, Stride: 1}, + {Lo: 0xcde, Hi: 0xcde, Stride: 1}, + {Lo: 0xce0, Hi: 0xce1, Stride: 1}, + {Lo: 0xd05, Hi: 0xd0c, Stride: 1}, + {Lo: 0xd0e, Hi: 0xd10, Stride: 1}, + {Lo: 0xd12, Hi: 0xd28, Stride: 1}, + {Lo: 0xd2a, Hi: 0xd39, Stride: 1}, + {Lo: 0xd60, Hi: 0xd61, Stride: 1}, + {Lo: 0xd85, Hi: 0xd96, Stride: 1}, + {Lo: 0xd9a, Hi: 0xdb1, Stride: 1}, + {Lo: 0xdb3, Hi: 0xdbb, Stride: 1}, + {Lo: 0xdbd, Hi: 0xdbd, Stride: 1}, + {Lo: 0xdc0, Hi: 0xdc6, Stride: 1}, + {Lo: 0xe01, Hi: 0xe30, Stride: 1}, + {Lo: 0xe32, Hi: 0xe33, Stride: 1}, + {Lo: 0xe40, Hi: 0xe46, Stride: 1}, + {Lo: 0xe81, Hi: 0xe82, Stride: 1}, + {Lo: 0xe84, Hi: 0xe84, Stride: 1}, + {Lo: 0xe87, Hi: 0xe88, Stride: 1}, + {Lo: 0xe8a, Hi: 0xe8a, Stride: 1}, + {Lo: 0xe8d, Hi: 0xe8d, Stride: 1}, + {Lo: 0xe94, Hi: 0xe97, Stride: 1}, + {Lo: 0xe99, Hi: 0xe9f, Stride: 1}, + {Lo: 0xea1, Hi: 0xea3, Stride: 1}, + {Lo: 0xea5, Hi: 0xea5, Stride: 1}, + {Lo: 0xea7, Hi: 0xea7, Stride: 1}, + {Lo: 0xeaa, Hi: 0xeab, Stride: 1}, + {Lo: 0xead, Hi: 0xeb0, Stride: 1}, + {Lo: 0xeb2, Hi: 0xeb3, Stride: 1}, + {Lo: 0xebd, Hi: 0xebd, Stride: 1}, + {Lo: 0xec0, Hi: 0xec4, Stride: 1}, + {Lo: 0xec6, Hi: 0xec6, Stride: 1}, + {Lo: 0xedc, Hi: 0xedd, Stride: 1}, + {Lo: 0xf00, Hi: 0xf00, Stride: 1}, + {Lo: 0xf40, Hi: 0xf47, Stride: 1}, + {Lo: 0xf49, Hi: 0xf6a, Stride: 1}, + {Lo: 0xf88, Hi: 0xf8b, Stride: 1}, + }, + R32: []unicode.Range32{ + {Lo: 0x1000, Hi: 0x1021, Stride: 1}, + {Lo: 0x1023, Hi: 0x1027, Stride: 1}, + {Lo: 0x1029, Hi: 0x102a, Stride: 1}, + {Lo: 0x1050, Hi: 0x1055, Stride: 1}, + {Lo: 0x10a0, Hi: 0x10c5, Stride: 1}, + {Lo: 0x10d0, Hi: 0x10f6, Stride: 1}, + {Lo: 0x1100, Hi: 0x1159, Stride: 1}, + {Lo: 0x115f, Hi: 0x11a2, Stride: 1}, + {Lo: 0x11a8, Hi: 0x11f9, Stride: 1}, + {Lo: 0x1200, Hi: 0x1206, Stride: 1}, + {Lo: 0x1208, Hi: 0x1246, Stride: 1}, + {Lo: 0x1248, Hi: 0x1248, Stride: 1}, + {Lo: 0x124a, Hi: 0x124d, Stride: 1}, + {Lo: 0x1250, Hi: 0x1256, Stride: 1}, + {Lo: 0x1258, Hi: 0x1258, Stride: 1}, + {Lo: 0x125a, Hi: 0x125d, Stride: 1}, + {Lo: 0x1260, Hi: 0x1286, Stride: 1}, + {Lo: 0x1288, Hi: 0x1288, Stride: 1}, + {Lo: 0x128a, Hi: 0x128d, Stride: 1}, + {Lo: 0x1290, Hi: 0x12ae, Stride: 1}, + {Lo: 0x12b0, Hi: 0x12b0, Stride: 1}, + {Lo: 0x12b2, Hi: 0x12b5, Stride: 1}, + {Lo: 0x12b8, Hi: 0x12be, Stride: 1}, + {Lo: 0x12c0, Hi: 0x12c0, Stride: 1}, + {Lo: 0x12c2, Hi: 0x12c5, Stride: 1}, + {Lo: 0x12c8, Hi: 0x12ce, Stride: 1}, + {Lo: 0x12d0, Hi: 0x12d6, Stride: 1}, + {Lo: 0x12d8, Hi: 0x12ee, Stride: 1}, + {Lo: 0x12f0, Hi: 0x130e, Stride: 1}, + {Lo: 0x1310, Hi: 0x1310, Stride: 1}, + {Lo: 0x1312, Hi: 0x1315, Stride: 1}, + {Lo: 0x1318, Hi: 0x131e, Stride: 1}, + {Lo: 0x1320, Hi: 0x1346, Stride: 1}, + {Lo: 0x1348, Hi: 0x135a, Stride: 1}, + {Lo: 0x13a0, Hi: 0x13f4, Stride: 1}, + {Lo: 0x1401, Hi: 0x166c, Stride: 1}, + {Lo: 0x166f, Hi: 0x1676, Stride: 1}, + {Lo: 0x1681, Hi: 0x169a, Stride: 1}, + {Lo: 0x16a0, Hi: 0x16ea, Stride: 1}, + {Lo: 0x1780, Hi: 0x17b3, Stride: 1}, + {Lo: 0x1820, Hi: 0x1877, Stride: 1}, + {Lo: 0x1880, Hi: 0x18a8, Stride: 1}, + {Lo: 0x1e00, Hi: 0x1e9b, Stride: 1}, + {Lo: 0x1ea0, Hi: 0x1ef9, Stride: 1}, + {Lo: 0x1f00, Hi: 0x1f15, Stride: 1}, + {Lo: 0x1f18, Hi: 0x1f1d, Stride: 1}, + {Lo: 0x1f20, Hi: 0x1f45, Stride: 1}, + {Lo: 0x1f48, Hi: 0x1f4d, Stride: 1}, + {Lo: 0x1f50, Hi: 0x1f57, Stride: 1}, + {Lo: 0x1f59, Hi: 0x1f59, Stride: 1}, + {Lo: 0x1f5b, Hi: 0x1f5b, Stride: 1}, + {Lo: 0x1f5d, Hi: 0x1f5d, Stride: 1}, + {Lo: 0x1f5f, Hi: 0x1f7d, Stride: 1}, + {Lo: 0x1f80, Hi: 0x1fb4, Stride: 1}, + {Lo: 0x1fb6, Hi: 0x1fbc, Stride: 1}, + {Lo: 0x1fbe, Hi: 0x1fbe, Stride: 1}, + {Lo: 0x1fc2, Hi: 0x1fc4, Stride: 1}, + {Lo: 0x1fc6, Hi: 0x1fcc, Stride: 1}, + {Lo: 0x1fd0, Hi: 0x1fd3, Stride: 1}, + {Lo: 0x1fd6, Hi: 0x1fdb, Stride: 1}, + {Lo: 0x1fe0, Hi: 0x1fec, Stride: 1}, + {Lo: 0x1ff2, Hi: 0x1ff4, Stride: 1}, + {Lo: 0x1ff6, Hi: 0x1ffc, Stride: 1}, + {Lo: 0x207f, Hi: 0x207f, Stride: 1}, + {Lo: 0x2102, Hi: 0x2102, Stride: 1}, + {Lo: 0x2107, Hi: 0x2107, Stride: 1}, + {Lo: 0x210a, Hi: 0x2113, Stride: 1}, + {Lo: 0x2115, Hi: 0x2115, Stride: 1}, + {Lo: 0x2119, Hi: 0x211d, Stride: 1}, + {Lo: 0x2124, Hi: 0x2124, Stride: 1}, + {Lo: 0x2126, Hi: 0x2126, Stride: 1}, + {Lo: 0x2128, Hi: 0x2128, Stride: 1}, + {Lo: 0x212a, Hi: 0x212d, Stride: 1}, + {Lo: 0x212f, Hi: 0x2131, Stride: 1}, + {Lo: 0x2133, Hi: 0x2139, Stride: 1}, + {Lo: 0x2160, Hi: 0x2183, Stride: 1}, + {Lo: 0x3005, Hi: 0x3007, Stride: 1}, + {Lo: 0x3021, Hi: 0x3029, Stride: 1}, + {Lo: 0x3031, Hi: 0x3035, Stride: 1}, + {Lo: 0x3038, Hi: 0x303a, Stride: 1}, + {Lo: 0x3041, Hi: 0x3094, Stride: 1}, + {Lo: 0x309d, Hi: 0x309e, Stride: 1}, + {Lo: 0x30a1, Hi: 0x30fa, Stride: 1}, + {Lo: 0x30fc, Hi: 0x30fe, Stride: 1}, + {Lo: 0x3105, Hi: 0x312c, Stride: 1}, + {Lo: 0x3131, Hi: 0x318e, Stride: 1}, + {Lo: 0x31a0, Hi: 0x31b7, Stride: 1}, + {Lo: 0x3400, Hi: 0x4db5, Stride: 1}, + {Lo: 0x4e00, Hi: 0x9fa5, Stride: 1}, + {Lo: 0xa000, Hi: 0xa48c, Stride: 1}, + {Lo: 0xac00, Hi: 0xd7a3, Stride: 1}, + {Lo: 0xf900, Hi: 0xfa2d, Stride: 1}, + {Lo: 0xfb00, Hi: 0xfb06, Stride: 1}, + {Lo: 0xfb13, Hi: 0xfb17, Stride: 1}, + {Lo: 0xfb1d, Hi: 0xfb1d, Stride: 1}, + {Lo: 0xfb1f, Hi: 0xfb28, Stride: 1}, + {Lo: 0xfb2a, Hi: 0xfb36, Stride: 1}, + {Lo: 0xfb38, Hi: 0xfb3c, Stride: 1}, + {Lo: 0xfb3e, Hi: 0xfb3e, Stride: 1}, + {Lo: 0xfb40, Hi: 0xfb41, Stride: 1}, + {Lo: 0xfb43, Hi: 0xfb44, Stride: 1}, + {Lo: 0xfb46, Hi: 0xfbb1, Stride: 1}, + {Lo: 0xfbd3, Hi: 0xfd3d, Stride: 1}, + {Lo: 0xfd50, Hi: 0xfd8f, Stride: 1}, + {Lo: 0xfd92, Hi: 0xfdc7, Stride: 1}, + {Lo: 0xfdf0, Hi: 0xfdfb, Stride: 1}, + {Lo: 0xfe70, Hi: 0xfe72, Stride: 1}, + {Lo: 0xfe74, Hi: 0xfe74, Stride: 1}, + {Lo: 0xfe76, Hi: 0xfefc, Stride: 1}, + {Lo: 0xff21, Hi: 0xff3a, Stride: 1}, + {Lo: 0xff41, Hi: 0xff5a, Stride: 1}, + {Lo: 0xff66, Hi: 0xffbe, Stride: 1}, + {Lo: 0xffc2, Hi: 0xffc7, Stride: 1}, + {Lo: 0xffca, Hi: 0xffcf, Stride: 1}, + {Lo: 0xffd2, Hi: 0xffd7, Stride: 1}, + {Lo: 0xffda, Hi: 0xffdc, Stride: 1}, + }, +} + +var idContinueES5 = &unicode.RangeTable{ + LatinOffset: 128, + R16: []unicode.Range16{ + {Lo: 0x30, Hi: 0x39, Stride: 1}, + {Lo: 0x41, Hi: 0x5a, Stride: 1}, + {Lo: 0x5f, Hi: 0x5f, Stride: 1}, + {Lo: 0x61, Hi: 0x7a, Stride: 1}, + {Lo: 0xaa, Hi: 0xaa, Stride: 1}, + {Lo: 0xb5, Hi: 0xb5, Stride: 1}, + {Lo: 0xba, Hi: 0xba, Stride: 1}, + {Lo: 0xc0, Hi: 0xd6, Stride: 1}, + {Lo: 0xd8, Hi: 0xf6, Stride: 1}, + {Lo: 0xf8, Hi: 0x21f, Stride: 1}, + {Lo: 0x222, Hi: 0x233, Stride: 1}, + {Lo: 0x250, Hi: 0x2ad, Stride: 1}, + {Lo: 0x2b0, Hi: 0x2b8, Stride: 1}, + {Lo: 0x2bb, Hi: 0x2c1, Stride: 1}, + {Lo: 0x2d0, Hi: 0x2d1, Stride: 1}, + {Lo: 0x2e0, Hi: 0x2e4, Stride: 1}, + {Lo: 0x2ee, Hi: 0x2ee, Stride: 1}, + {Lo: 0x300, Hi: 0x34e, Stride: 1}, + {Lo: 0x360, Hi: 0x362, Stride: 1}, + {Lo: 0x37a, Hi: 0x37a, Stride: 1}, + {Lo: 0x386, Hi: 0x386, Stride: 1}, + {Lo: 0x388, Hi: 0x38a, Stride: 1}, + {Lo: 0x38c, Hi: 0x38c, Stride: 1}, + {Lo: 0x38e, Hi: 0x3a1, Stride: 1}, + {Lo: 0x3a3, Hi: 0x3ce, Stride: 1}, + {Lo: 0x3d0, Hi: 0x3d7, Stride: 1}, + {Lo: 0x3da, Hi: 0x3f3, Stride: 1}, + {Lo: 0x400, Hi: 0x481, Stride: 1}, + {Lo: 0x483, Hi: 0x486, Stride: 1}, + {Lo: 0x48c, Hi: 0x4c4, Stride: 1}, + {Lo: 0x4c7, Hi: 0x4c8, Stride: 1}, + {Lo: 0x4cb, Hi: 0x4cc, Stride: 1}, + {Lo: 0x4d0, Hi: 0x4f5, Stride: 1}, + {Lo: 0x4f8, Hi: 0x4f9, Stride: 1}, + {Lo: 0x531, Hi: 0x556, Stride: 1}, + {Lo: 0x559, Hi: 0x559, Stride: 1}, + {Lo: 0x561, Hi: 0x587, Stride: 1}, + {Lo: 0x591, Hi: 0x5a1, Stride: 1}, + {Lo: 0x5a3, Hi: 0x5b9, Stride: 1}, + {Lo: 0x5bb, Hi: 0x5bd, Stride: 1}, + {Lo: 0x5bf, Hi: 0x5bf, Stride: 1}, + {Lo: 0x5c1, Hi: 0x5c2, Stride: 1}, + {Lo: 0x5c4, Hi: 0x5c4, Stride: 1}, + {Lo: 0x5d0, Hi: 0x5ea, Stride: 1}, + {Lo: 0x5f0, Hi: 0x5f2, Stride: 1}, + {Lo: 0x621, Hi: 0x63a, Stride: 1}, + {Lo: 0x640, Hi: 0x655, Stride: 1}, + {Lo: 0x660, Hi: 0x669, Stride: 1}, + {Lo: 0x670, Hi: 0x6d3, Stride: 1}, + {Lo: 0x6d5, Hi: 0x6dc, Stride: 1}, + {Lo: 0x6df, Hi: 0x6e8, Stride: 1}, + {Lo: 0x6ea, Hi: 0x6ed, Stride: 1}, + {Lo: 0x6f0, Hi: 0x6fc, Stride: 1}, + {Lo: 0x710, Hi: 0x72c, Stride: 1}, + {Lo: 0x730, Hi: 0x74a, Stride: 1}, + {Lo: 0x780, Hi: 0x7b0, Stride: 1}, + {Lo: 0x901, Hi: 0x903, Stride: 1}, + {Lo: 0x905, Hi: 0x939, Stride: 1}, + {Lo: 0x93c, Hi: 0x94d, Stride: 1}, + {Lo: 0x950, Hi: 0x954, Stride: 1}, + {Lo: 0x958, Hi: 0x963, Stride: 1}, + {Lo: 0x966, Hi: 0x96f, Stride: 1}, + {Lo: 0x981, Hi: 0x983, Stride: 1}, + {Lo: 0x985, Hi: 0x98c, Stride: 1}, + {Lo: 0x98f, Hi: 0x990, Stride: 1}, + {Lo: 0x993, Hi: 0x9a8, Stride: 1}, + {Lo: 0x9aa, Hi: 0x9b0, Stride: 1}, + {Lo: 0x9b2, Hi: 0x9b2, Stride: 1}, + {Lo: 0x9b6, Hi: 0x9b9, Stride: 1}, + {Lo: 0x9bc, Hi: 0x9bc, Stride: 1}, + {Lo: 0x9be, Hi: 0x9c4, Stride: 1}, + {Lo: 0x9c7, Hi: 0x9c8, Stride: 1}, + {Lo: 0x9cb, Hi: 0x9cd, Stride: 1}, + {Lo: 0x9d7, Hi: 0x9d7, Stride: 1}, + {Lo: 0x9dc, Hi: 0x9dd, Stride: 1}, + {Lo: 0x9df, Hi: 0x9e3, Stride: 1}, + {Lo: 0x9e6, Hi: 0x9f1, Stride: 1}, + {Lo: 0xa02, Hi: 0xa02, Stride: 1}, + {Lo: 0xa05, Hi: 0xa0a, Stride: 1}, + {Lo: 0xa0f, Hi: 0xa10, Stride: 1}, + {Lo: 0xa13, Hi: 0xa28, Stride: 1}, + {Lo: 0xa2a, Hi: 0xa30, Stride: 1}, + {Lo: 0xa32, Hi: 0xa33, Stride: 1}, + {Lo: 0xa35, Hi: 0xa36, Stride: 1}, + {Lo: 0xa38, Hi: 0xa39, Stride: 1}, + {Lo: 0xa3c, Hi: 0xa3c, Stride: 1}, + {Lo: 0xa3e, Hi: 0xa42, Stride: 1}, + {Lo: 0xa47, Hi: 0xa48, Stride: 1}, + {Lo: 0xa4b, Hi: 0xa4d, Stride: 1}, + {Lo: 0xa59, Hi: 0xa5c, Stride: 1}, + {Lo: 0xa5e, Hi: 0xa5e, Stride: 1}, + {Lo: 0xa66, Hi: 0xa74, Stride: 1}, + {Lo: 0xa81, Hi: 0xa83, Stride: 1}, + {Lo: 0xa85, Hi: 0xa8b, Stride: 1}, + {Lo: 0xa8d, Hi: 0xa8d, Stride: 1}, + {Lo: 0xa8f, Hi: 0xa91, Stride: 1}, + {Lo: 0xa93, Hi: 0xaa8, Stride: 1}, + {Lo: 0xaaa, Hi: 0xab0, Stride: 1}, + {Lo: 0xab2, Hi: 0xab3, Stride: 1}, + {Lo: 0xab5, Hi: 0xab9, Stride: 1}, + {Lo: 0xabc, Hi: 0xac5, Stride: 1}, + {Lo: 0xac7, Hi: 0xac9, Stride: 1}, + {Lo: 0xacb, Hi: 0xacd, Stride: 1}, + {Lo: 0xad0, Hi: 0xad0, Stride: 1}, + {Lo: 0xae0, Hi: 0xae0, Stride: 1}, + {Lo: 0xae6, Hi: 0xaef, Stride: 1}, + {Lo: 0xb01, Hi: 0xb03, Stride: 1}, + {Lo: 0xb05, Hi: 0xb0c, Stride: 1}, + {Lo: 0xb0f, Hi: 0xb10, Stride: 1}, + {Lo: 0xb13, Hi: 0xb28, Stride: 1}, + {Lo: 0xb2a, Hi: 0xb30, Stride: 1}, + {Lo: 0xb32, Hi: 0xb33, Stride: 1}, + {Lo: 0xb36, Hi: 0xb39, Stride: 1}, + {Lo: 0xb3c, Hi: 0xb43, Stride: 1}, + {Lo: 0xb47, Hi: 0xb48, Stride: 1}, + {Lo: 0xb4b, Hi: 0xb4d, Stride: 1}, + {Lo: 0xb56, Hi: 0xb57, Stride: 1}, + {Lo: 0xb5c, Hi: 0xb5d, Stride: 1}, + {Lo: 0xb5f, Hi: 0xb61, Stride: 1}, + {Lo: 0xb66, Hi: 0xb6f, Stride: 1}, + {Lo: 0xb82, Hi: 0xb83, Stride: 1}, + {Lo: 0xb85, Hi: 0xb8a, Stride: 1}, + {Lo: 0xb8e, Hi: 0xb90, Stride: 1}, + {Lo: 0xb92, Hi: 0xb95, Stride: 1}, + {Lo: 0xb99, Hi: 0xb9a, Stride: 1}, + {Lo: 0xb9c, Hi: 0xb9c, Stride: 1}, + {Lo: 0xb9e, Hi: 0xb9f, Stride: 1}, + {Lo: 0xba3, Hi: 0xba4, Stride: 1}, + {Lo: 0xba8, Hi: 0xbaa, Stride: 1}, + {Lo: 0xbae, Hi: 0xbb5, Stride: 1}, + {Lo: 0xbb7, Hi: 0xbb9, Stride: 1}, + {Lo: 0xbbe, Hi: 0xbc2, Stride: 1}, + {Lo: 0xbc6, Hi: 0xbc8, Stride: 1}, + {Lo: 0xbca, Hi: 0xbcd, Stride: 1}, + {Lo: 0xbd7, Hi: 0xbd7, Stride: 1}, + {Lo: 0xbe7, Hi: 0xbef, Stride: 1}, + {Lo: 0xc01, Hi: 0xc03, Stride: 1}, + {Lo: 0xc05, Hi: 0xc0c, Stride: 1}, + {Lo: 0xc0e, Hi: 0xc10, Stride: 1}, + {Lo: 0xc12, Hi: 0xc28, Stride: 1}, + {Lo: 0xc2a, Hi: 0xc33, Stride: 1}, + {Lo: 0xc35, Hi: 0xc39, Stride: 1}, + {Lo: 0xc3e, Hi: 0xc44, Stride: 1}, + {Lo: 0xc46, Hi: 0xc48, Stride: 1}, + {Lo: 0xc4a, Hi: 0xc4d, Stride: 1}, + {Lo: 0xc55, Hi: 0xc56, Stride: 1}, + {Lo: 0xc60, Hi: 0xc61, Stride: 1}, + {Lo: 0xc66, Hi: 0xc6f, Stride: 1}, + {Lo: 0xc82, Hi: 0xc83, Stride: 1}, + {Lo: 0xc85, Hi: 0xc8c, Stride: 1}, + {Lo: 0xc8e, Hi: 0xc90, Stride: 1}, + {Lo: 0xc92, Hi: 0xca8, Stride: 1}, + {Lo: 0xcaa, Hi: 0xcb3, Stride: 1}, + {Lo: 0xcb5, Hi: 0xcb9, Stride: 1}, + {Lo: 0xcbe, Hi: 0xcc4, Stride: 1}, + {Lo: 0xcc6, Hi: 0xcc8, Stride: 1}, + {Lo: 0xcca, Hi: 0xccd, Stride: 1}, + {Lo: 0xcd5, Hi: 0xcd6, Stride: 1}, + {Lo: 0xcde, Hi: 0xcde, Stride: 1}, + {Lo: 0xce0, Hi: 0xce1, Stride: 1}, + {Lo: 0xce6, Hi: 0xcef, Stride: 1}, + {Lo: 0xd02, Hi: 0xd03, Stride: 1}, + {Lo: 0xd05, Hi: 0xd0c, Stride: 1}, + {Lo: 0xd0e, Hi: 0xd10, Stride: 1}, + {Lo: 0xd12, Hi: 0xd28, Stride: 1}, + {Lo: 0xd2a, Hi: 0xd39, Stride: 1}, + {Lo: 0xd3e, Hi: 0xd43, Stride: 1}, + {Lo: 0xd46, Hi: 0xd48, Stride: 1}, + {Lo: 0xd4a, Hi: 0xd4d, Stride: 1}, + {Lo: 0xd57, Hi: 0xd57, Stride: 1}, + {Lo: 0xd60, Hi: 0xd61, Stride: 1}, + {Lo: 0xd66, Hi: 0xd6f, Stride: 1}, + {Lo: 0xd82, Hi: 0xd83, Stride: 1}, + {Lo: 0xd85, Hi: 0xd96, Stride: 1}, + {Lo: 0xd9a, Hi: 0xdb1, Stride: 1}, + {Lo: 0xdb3, Hi: 0xdbb, Stride: 1}, + {Lo: 0xdbd, Hi: 0xdbd, Stride: 1}, + {Lo: 0xdc0, Hi: 0xdc6, Stride: 1}, + {Lo: 0xdca, Hi: 0xdca, Stride: 1}, + {Lo: 0xdcf, Hi: 0xdd4, Stride: 1}, + {Lo: 0xdd6, Hi: 0xdd6, Stride: 1}, + {Lo: 0xdd8, Hi: 0xddf, Stride: 1}, + {Lo: 0xdf2, Hi: 0xdf3, Stride: 1}, + {Lo: 0xe01, Hi: 0xe3a, Stride: 1}, + {Lo: 0xe40, Hi: 0xe4e, Stride: 1}, + {Lo: 0xe50, Hi: 0xe59, Stride: 1}, + {Lo: 0xe81, Hi: 0xe82, Stride: 1}, + {Lo: 0xe84, Hi: 0xe84, Stride: 1}, + {Lo: 0xe87, Hi: 0xe88, Stride: 1}, + {Lo: 0xe8a, Hi: 0xe8a, Stride: 1}, + {Lo: 0xe8d, Hi: 0xe8d, Stride: 1}, + {Lo: 0xe94, Hi: 0xe97, Stride: 1}, + {Lo: 0xe99, Hi: 0xe9f, Stride: 1}, + {Lo: 0xea1, Hi: 0xea3, Stride: 1}, + {Lo: 0xea5, Hi: 0xea5, Stride: 1}, + {Lo: 0xea7, Hi: 0xea7, Stride: 1}, + {Lo: 0xeaa, Hi: 0xeab, Stride: 1}, + {Lo: 0xead, Hi: 0xeb9, Stride: 1}, + {Lo: 0xebb, Hi: 0xebd, Stride: 1}, + {Lo: 0xec0, Hi: 0xec4, Stride: 1}, + {Lo: 0xec6, Hi: 0xec6, Stride: 1}, + {Lo: 0xec8, Hi: 0xecd, Stride: 1}, + {Lo: 0xed0, Hi: 0xed9, Stride: 1}, + {Lo: 0xedc, Hi: 0xedd, Stride: 1}, + {Lo: 0xf00, Hi: 0xf00, Stride: 1}, + {Lo: 0xf18, Hi: 0xf19, Stride: 1}, + {Lo: 0xf20, Hi: 0xf29, Stride: 1}, + {Lo: 0xf35, Hi: 0xf35, Stride: 1}, + {Lo: 0xf37, Hi: 0xf37, Stride: 1}, + {Lo: 0xf39, Hi: 0xf39, Stride: 1}, + {Lo: 0xf3e, Hi: 0xf47, Stride: 1}, + {Lo: 0xf49, Hi: 0xf6a, Stride: 1}, + {Lo: 0xf71, Hi: 0xf84, Stride: 1}, + {Lo: 0xf86, Hi: 0xf8b, Stride: 1}, + {Lo: 0xf90, Hi: 0xf97, Stride: 1}, + {Lo: 0xf99, Hi: 0xfbc, Stride: 1}, + {Lo: 0xfc6, Hi: 0xfc6, Stride: 1}, + }, + R32: []unicode.Range32{ + {Lo: 0x1000, Hi: 0x1021, Stride: 1}, + {Lo: 0x1023, Hi: 0x1027, Stride: 1}, + {Lo: 0x1029, Hi: 0x102a, Stride: 1}, + {Lo: 0x102c, Hi: 0x1032, Stride: 1}, + {Lo: 0x1036, Hi: 0x1039, Stride: 1}, + {Lo: 0x1040, Hi: 0x1049, Stride: 1}, + {Lo: 0x1050, Hi: 0x1059, Stride: 1}, + {Lo: 0x10a0, Hi: 0x10c5, Stride: 1}, + {Lo: 0x10d0, Hi: 0x10f6, Stride: 1}, + {Lo: 0x1100, Hi: 0x1159, Stride: 1}, + {Lo: 0x115f, Hi: 0x11a2, Stride: 1}, + {Lo: 0x11a8, Hi: 0x11f9, Stride: 1}, + {Lo: 0x1200, Hi: 0x1206, Stride: 1}, + {Lo: 0x1208, Hi: 0x1246, Stride: 1}, + {Lo: 0x1248, Hi: 0x1248, Stride: 1}, + {Lo: 0x124a, Hi: 0x124d, Stride: 1}, + {Lo: 0x1250, Hi: 0x1256, Stride: 1}, + {Lo: 0x1258, Hi: 0x1258, Stride: 1}, + {Lo: 0x125a, Hi: 0x125d, Stride: 1}, + {Lo: 0x1260, Hi: 0x1286, Stride: 1}, + {Lo: 0x1288, Hi: 0x1288, Stride: 1}, + {Lo: 0x128a, Hi: 0x128d, Stride: 1}, + {Lo: 0x1290, Hi: 0x12ae, Stride: 1}, + {Lo: 0x12b0, Hi: 0x12b0, Stride: 1}, + {Lo: 0x12b2, Hi: 0x12b5, Stride: 1}, + {Lo: 0x12b8, Hi: 0x12be, Stride: 1}, + {Lo: 0x12c0, Hi: 0x12c0, Stride: 1}, + {Lo: 0x12c2, Hi: 0x12c5, Stride: 1}, + {Lo: 0x12c8, Hi: 0x12ce, Stride: 1}, + {Lo: 0x12d0, Hi: 0x12d6, Stride: 1}, + {Lo: 0x12d8, Hi: 0x12ee, Stride: 1}, + {Lo: 0x12f0, Hi: 0x130e, Stride: 1}, + {Lo: 0x1310, Hi: 0x1310, Stride: 1}, + {Lo: 0x1312, Hi: 0x1315, Stride: 1}, + {Lo: 0x1318, Hi: 0x131e, Stride: 1}, + {Lo: 0x1320, Hi: 0x1346, Stride: 1}, + {Lo: 0x1348, Hi: 0x135a, Stride: 1}, + {Lo: 0x1369, Hi: 0x1371, Stride: 1}, + {Lo: 0x13a0, Hi: 0x13f4, Stride: 1}, + {Lo: 0x1401, Hi: 0x166c, Stride: 1}, + {Lo: 0x166f, Hi: 0x1676, Stride: 1}, + {Lo: 0x1681, Hi: 0x169a, Stride: 1}, + {Lo: 0x16a0, Hi: 0x16ea, Stride: 1}, + {Lo: 0x1780, Hi: 0x17d3, Stride: 1}, + {Lo: 0x17e0, Hi: 0x17e9, Stride: 1}, + {Lo: 0x1810, Hi: 0x1819, Stride: 1}, + {Lo: 0x1820, Hi: 0x1877, Stride: 1}, + {Lo: 0x1880, Hi: 0x18a9, Stride: 1}, + {Lo: 0x1e00, Hi: 0x1e9b, Stride: 1}, + {Lo: 0x1ea0, Hi: 0x1ef9, Stride: 1}, + {Lo: 0x1f00, Hi: 0x1f15, Stride: 1}, + {Lo: 0x1f18, Hi: 0x1f1d, Stride: 1}, + {Lo: 0x1f20, Hi: 0x1f45, Stride: 1}, + {Lo: 0x1f48, Hi: 0x1f4d, Stride: 1}, + {Lo: 0x1f50, Hi: 0x1f57, Stride: 1}, + {Lo: 0x1f59, Hi: 0x1f59, Stride: 1}, + {Lo: 0x1f5b, Hi: 0x1f5b, Stride: 1}, + {Lo: 0x1f5d, Hi: 0x1f5d, Stride: 1}, + {Lo: 0x1f5f, Hi: 0x1f7d, Stride: 1}, + {Lo: 0x1f80, Hi: 0x1fb4, Stride: 1}, + {Lo: 0x1fb6, Hi: 0x1fbc, Stride: 1}, + {Lo: 0x1fbe, Hi: 0x1fbe, Stride: 1}, + {Lo: 0x1fc2, Hi: 0x1fc4, Stride: 1}, + {Lo: 0x1fc6, Hi: 0x1fcc, Stride: 1}, + {Lo: 0x1fd0, Hi: 0x1fd3, Stride: 1}, + {Lo: 0x1fd6, Hi: 0x1fdb, Stride: 1}, + {Lo: 0x1fe0, Hi: 0x1fec, Stride: 1}, + {Lo: 0x1ff2, Hi: 0x1ff4, Stride: 1}, + {Lo: 0x1ff6, Hi: 0x1ffc, Stride: 1}, + {Lo: 0x203f, Hi: 0x2040, Stride: 1}, + {Lo: 0x207f, Hi: 0x207f, Stride: 1}, + {Lo: 0x20d0, Hi: 0x20dc, Stride: 1}, + {Lo: 0x20e1, Hi: 0x20e1, Stride: 1}, + {Lo: 0x2102, Hi: 0x2102, Stride: 1}, + {Lo: 0x2107, Hi: 0x2107, Stride: 1}, + {Lo: 0x210a, Hi: 0x2113, Stride: 1}, + {Lo: 0x2115, Hi: 0x2115, Stride: 1}, + {Lo: 0x2119, Hi: 0x211d, Stride: 1}, + {Lo: 0x2124, Hi: 0x2124, Stride: 1}, + {Lo: 0x2126, Hi: 0x2126, Stride: 1}, + {Lo: 0x2128, Hi: 0x2128, Stride: 1}, + {Lo: 0x212a, Hi: 0x212d, Stride: 1}, + {Lo: 0x212f, Hi: 0x2131, Stride: 1}, + {Lo: 0x2133, Hi: 0x2139, Stride: 1}, + {Lo: 0x2160, Hi: 0x2183, Stride: 1}, + {Lo: 0x3005, Hi: 0x3007, Stride: 1}, + {Lo: 0x3021, Hi: 0x302f, Stride: 1}, + {Lo: 0x3031, Hi: 0x3035, Stride: 1}, + {Lo: 0x3038, Hi: 0x303a, Stride: 1}, + {Lo: 0x3041, Hi: 0x3094, Stride: 1}, + {Lo: 0x3099, Hi: 0x309a, Stride: 1}, + {Lo: 0x309d, Hi: 0x309e, Stride: 1}, + {Lo: 0x30a1, Hi: 0x30fe, Stride: 1}, + {Lo: 0x3105, Hi: 0x312c, Stride: 1}, + {Lo: 0x3131, Hi: 0x318e, Stride: 1}, + {Lo: 0x31a0, Hi: 0x31b7, Stride: 1}, + {Lo: 0x3400, Hi: 0x4db5, Stride: 1}, + {Lo: 0x4e00, Hi: 0x9fa5, Stride: 1}, + {Lo: 0xa000, Hi: 0xa48c, Stride: 1}, + {Lo: 0xac00, Hi: 0xd7a3, Stride: 1}, + {Lo: 0xf900, Hi: 0xfa2d, Stride: 1}, + {Lo: 0xfb00, Hi: 0xfb06, Stride: 1}, + {Lo: 0xfb13, Hi: 0xfb17, Stride: 1}, + {Lo: 0xfb1d, Hi: 0xfb28, Stride: 1}, + {Lo: 0xfb2a, Hi: 0xfb36, Stride: 1}, + {Lo: 0xfb38, Hi: 0xfb3c, Stride: 1}, + {Lo: 0xfb3e, Hi: 0xfb3e, Stride: 1}, + {Lo: 0xfb40, Hi: 0xfb41, Stride: 1}, + {Lo: 0xfb43, Hi: 0xfb44, Stride: 1}, + {Lo: 0xfb46, Hi: 0xfbb1, Stride: 1}, + {Lo: 0xfbd3, Hi: 0xfd3d, Stride: 1}, + {Lo: 0xfd50, Hi: 0xfd8f, Stride: 1}, + {Lo: 0xfd92, Hi: 0xfdc7, Stride: 1}, + {Lo: 0xfdf0, Hi: 0xfdfb, Stride: 1}, + {Lo: 0xfe20, Hi: 0xfe23, Stride: 1}, + {Lo: 0xfe33, Hi: 0xfe34, Stride: 1}, + {Lo: 0xfe4d, Hi: 0xfe4f, Stride: 1}, + {Lo: 0xfe70, Hi: 0xfe72, Stride: 1}, + {Lo: 0xfe74, Hi: 0xfe74, Stride: 1}, + {Lo: 0xfe76, Hi: 0xfefc, Stride: 1}, + {Lo: 0xff10, Hi: 0xff19, Stride: 1}, + {Lo: 0xff21, Hi: 0xff3a, Stride: 1}, + {Lo: 0xff3f, Hi: 0xff3f, Stride: 1}, + {Lo: 0xff41, Hi: 0xff5a, Stride: 1}, + {Lo: 0xff65, Hi: 0xffbe, Stride: 1}, + {Lo: 0xffc2, Hi: 0xffc7, Stride: 1}, + {Lo: 0xffca, Hi: 0xffcf, Stride: 1}, + {Lo: 0xffd2, Hi: 0xffd7, Stride: 1}, + {Lo: 0xffda, Hi: 0xffdc, Stride: 1}, + }, +} + var idStart = &unicode.RangeTable{ LatinOffset: 117, R16: []unicode.Range16{ diff --git a/internal/js_parser/js_parser.go b/internal/js_parser/js_parser.go index 0b00a4abf44..4c9319c24c5 100644 --- a/internal/js_parser/js_parser.go +++ b/internal/js_parser/js_parser.go @@ -33,44 +33,44 @@ import ( // to have at least two separate passes to handle variable hoisting. See the // comment about scopesInOrder below for more information. type parser struct { - options Options - log logger.Log - source logger.Source - tracker logger.LineColumnTracker - lexer js_lexer.Lexer - allowIn bool - allowPrivateIdentifiers bool - hasTopLevelReturn bool - latestReturnHadSemicolon bool - hasImportMeta bool - hasESModuleSyntax bool - warnedThisIsUndefined bool - topLevelAwaitKeyword logger.Range - fnOrArrowDataParse fnOrArrowDataParse - fnOrArrowDataVisit fnOrArrowDataVisit - fnOnlyDataVisit fnOnlyDataVisit - allocatedNames []string - latestArrowArgLoc logger.Loc - forbidSuffixAfterAsLoc logger.Loc - currentScope *js_ast.Scope - scopesForCurrentPart []*js_ast.Scope - symbols []js_ast.Symbol - tsUseCounts []uint32 - exportsRef js_ast.Ref - requireRef js_ast.Ref - moduleRef js_ast.Ref - importMetaRef js_ast.Ref - promiseRef js_ast.Ref - findSymbolHelper func(loc logger.Loc, name string) js_ast.Ref - symbolForDefineHelper func(int) js_ast.Ref - injectedDefineSymbols []js_ast.Ref - injectedSymbolSources map[js_ast.Ref]injectedSymbolSource - symbolUses map[js_ast.Ref]js_ast.SymbolUse - declaredSymbols []js_ast.DeclaredSymbol - runtimeImports map[string]js_ast.Ref - duplicateCaseChecker duplicateCaseChecker - nonBMPIdentifiers map[string]bool - legacyOctalLiterals map[js_ast.E]logger.Range + options Options + log logger.Log + source logger.Source + tracker logger.LineColumnTracker + lexer js_lexer.Lexer + allowIn bool + allowPrivateIdentifiers bool + hasTopLevelReturn bool + latestReturnHadSemicolon bool + hasImportMeta bool + hasESModuleSyntax bool + warnedThisIsUndefined bool + topLevelAwaitKeyword logger.Range + fnOrArrowDataParse fnOrArrowDataParse + fnOrArrowDataVisit fnOrArrowDataVisit + fnOnlyDataVisit fnOnlyDataVisit + allocatedNames []string + latestArrowArgLoc logger.Loc + forbidSuffixAfterAsLoc logger.Loc + currentScope *js_ast.Scope + scopesForCurrentPart []*js_ast.Scope + symbols []js_ast.Symbol + tsUseCounts []uint32 + exportsRef js_ast.Ref + requireRef js_ast.Ref + moduleRef js_ast.Ref + importMetaRef js_ast.Ref + promiseRef js_ast.Ref + findSymbolHelper func(loc logger.Loc, name string) js_ast.Ref + symbolForDefineHelper func(int) js_ast.Ref + injectedDefineSymbols []js_ast.Ref + injectedSymbolSources map[js_ast.Ref]injectedSymbolSource + symbolUses map[js_ast.Ref]js_ast.SymbolUse + declaredSymbols []js_ast.DeclaredSymbol + runtimeImports map[string]js_ast.Ref + duplicateCaseChecker duplicateCaseChecker + unrepresentableIdentifiers map[string]bool + legacyOctalLiterals map[js_ast.E]logger.Range // For strict mode handling hoistedRefForSloppyModeBlockFn map[js_ast.Ref]js_ast.Ref @@ -1381,7 +1381,7 @@ func (p *parser) canMergeSymbols(scope *js_ast.Scope, existing js_ast.SymbolKind } func (p *parser) declareSymbol(kind js_ast.SymbolKind, loc logger.Loc, name string) js_ast.Ref { - p.checkForNonBMPCodePoint(loc, name) + p.checkForUnrepresentableIdentifier(loc, name) // Allocate a new symbol ref := p.newSymbol(kind, name) @@ -4636,7 +4636,7 @@ func (p *parser) parseClauseAlias(kind string) string { } alias := p.lexer.Identifier - p.checkForNonBMPCodePoint(loc, alias) + p.checkForUnrepresentableIdentifier(loc, alias) return alias } @@ -6341,7 +6341,7 @@ func (p *parser) parseStmt(opts parseStmtOpts) js_ast.Stmt { for i, item := range *stmt.Items { name := p.loadNameFromRef(item.Name.Ref) ref := p.declareSymbol(js_ast.SymbolImport, item.Name.Loc, name) - p.checkForNonBMPCodePoint(item.AliasLoc, item.Alias) + p.checkForUnrepresentableIdentifier(item.AliasLoc, item.Alias) p.isImportItem[ref] = true (*stmt.Items)[i].Name.Ref = ref itemRefs[item.Alias] = js_ast.LocRef{Loc: item.Name.Loc, Ref: ref} @@ -6785,7 +6785,7 @@ func (p *parser) findSymbol(loc logger.Loc, name string) findSymbolResult { s = s.Parent if s == nil { // Allocate an "unbound" symbol - p.checkForNonBMPCodePoint(loc, name) + p.checkForUnrepresentableIdentifier(loc, name) ref = p.newSymbol(js_ast.SymbolUnbound, name) declareLoc = loc p.moduleScope.Members[name] = js_ast.ScopeMember{Ref: ref, Loc: logger.Loc{Start: -1}} @@ -9107,7 +9107,7 @@ func (p *parser) visitAndAppendStmt(stmts []js_ast.Stmt, stmt js_ast.Stmt) []js_ value.ValueOrNil = js_ast.Expr{Loc: value.Loc, Data: js_ast.EUndefinedShared} } - if p.options.mangleSyntax && js_lexer.IsIdentifier(name) { + if p.options.mangleSyntax && js_lexer.IsIdentifier(name, 0) { // "Enum.Name = value" assignTarget = js_ast.Assign( js_ast.Expr{Loc: value.Loc, Data: &js_ast.EDot{ @@ -9561,7 +9561,7 @@ func (p *parser) visitClass(nameScopeLoc logger.Loc, class *js_ast.Class) js_ast // "class {['x'] = y}" => "class {x = y}" if p.options.mangleSyntax && property.IsComputed { - if str, ok := key.Data.(*js_ast.EString); ok && js_lexer.IsIdentifierUTF16(str.Value) { + if str, ok := key.Data.(*js_ast.EString); ok && js_lexer.IsIdentifierUTF16(str.Value, 0) { isInvalidConstructor := false if js_lexer.UTF16EqualsString(str.Value, "constructor") { if !property.IsMethod { @@ -9821,17 +9821,29 @@ func (p *parser) jsxStringsToMemberExpression(loc logger.Loc, parts []string) js return value } -func (p *parser) checkForNonBMPCodePoint(loc logger.Loc, name string) { +func (p *parser) checkForUnrepresentableIdentifier(loc logger.Loc, name string) { if p.options.asciiOnly && p.options.unsupportedJSFeatures.Has(compat.UnicodeEscapes) && js_lexer.ContainsNonBMPCodePoint(name) { - if p.nonBMPIdentifiers == nil { - p.nonBMPIdentifiers = make(map[string]bool) + if p.unrepresentableIdentifiers == nil { + p.unrepresentableIdentifiers = make(map[string]bool) } - if !p.nonBMPIdentifiers[name] { - p.nonBMPIdentifiers[name] = true + if !p.unrepresentableIdentifiers[name] { + p.unrepresentableIdentifiers[name] = true + where, notes := p.prettyPrintTargetEnvironment(compat.UpdatedIdentifiers) r := js_lexer.RangeOfIdentifier(p.source, loc) - p.log.AddRangeError(&p.tracker, r, fmt.Sprintf("%q cannot be escaped in the target environment ("+ - "consider setting the charset to \"utf8\" or changing the target)", name)) + p.log.AddRangeErrorWithNotes(&p.tracker, r, fmt.Sprintf("%q cannot be escaped in %s but you "+ + "can set the charset to \"utf8\" to allow unescaped Unicode characters", name, where), notes) + } + } else if p.options.unsupportedJSFeatures.Has(compat.UpdatedIdentifiers) && !strings.HasPrefix(name, "#") && + !js_lexer.IsIdentifier(name, p.options.unsupportedJSFeatures) { + if p.unrepresentableIdentifiers == nil { + p.unrepresentableIdentifiers = make(map[string]bool) + } + if !p.unrepresentableIdentifiers[name] { + p.unrepresentableIdentifiers[name] = true + where, notes := p.prettyPrintTargetEnvironment(compat.UpdatedIdentifiers) + r := js_lexer.RangeOfIdentifier(p.source, loc) + p.log.AddRangeErrorWithNotes(&p.tracker, r, fmt.Sprintf("%q is not considered a valid identifier in %s", name, where), notes) } } } @@ -11263,7 +11275,7 @@ func (p *parser) visitExprInOut(expr js_ast.Expr, in exprIn) (js_ast.Expr, exprO // "a['b']" => "a.b" if p.options.mangleSyntax { - if str, ok := e.Index.Data.(*js_ast.EString); ok && js_lexer.IsIdentifierUTF16(str.Value) { + if str, ok := e.Index.Data.(*js_ast.EString); ok && js_lexer.IsIdentifierUTF16(str.Value, 0) { dot := &js_ast.EDot{ Target: e.Target, Name: js_lexer.UTF16ToString(str.Value), @@ -11747,7 +11759,7 @@ func (p *parser) visitExprInOut(expr js_ast.Expr, in exprIn) (js_ast.Expr, exprO // "{['x']: y}" => "{x: y}" if p.options.mangleSyntax && property.IsComputed { - if str, ok := key.Data.(*js_ast.EString); ok && js_lexer.IsIdentifierUTF16(str.Value) && !js_lexer.UTF16EqualsString(str.Value, "__proto__") { + if str, ok := key.Data.(*js_ast.EString); ok && js_lexer.IsIdentifierUTF16(str.Value, 0) && !js_lexer.UTF16EqualsString(str.Value, "__proto__") { property.IsComputed = false } } @@ -13861,7 +13873,7 @@ func ParseJSXExpr(text string, kind JSXExprKind) (config.JSXExpr, bool) { // Try a property chain parts := strings.Split(text, ".") for _, part := range parts { - if !js_lexer.IsIdentifier(part) { + if !js_lexer.IsIdentifier(part, 0) { parts = nil break } diff --git a/internal/js_parser/js_parser_lower.go b/internal/js_parser/js_parser_lower.go index e08aa86e60d..524314e4130 100644 --- a/internal/js_parser/js_parser_lower.go +++ b/internal/js_parser/js_parser_lower.go @@ -14,6 +14,19 @@ import ( "github.com/evanw/esbuild/internal/logger" ) +func (p *parser) prettyPrintTargetEnvironment(feature compat.JSFeature) (where string, notes []logger.MsgData) { + where = "the configured target environment" + if tsTarget := p.options.tsTarget; tsTarget != nil && tsTarget.UnsupportedJSFeatures.Has(feature) { + tracker := logger.MakeLineColumnTracker(&tsTarget.Source) + where = fmt.Sprintf("%s (%q)", where, tsTarget.Target) + notes = []logger.MsgData{logger.RangeData(&tracker, tsTarget.Range, fmt.Sprintf( + "The target environment was set to %q here", tsTarget.Target))} + } else if p.options.originalTargetEnv != "" { + where = fmt.Sprintf("%s (%s)", where, p.options.originalTargetEnv) + } + return +} + func (p *parser) markSyntaxFeature(feature compat.JSFeature, r logger.Range) (didGenerateError bool) { didGenerateError = true @@ -29,17 +42,7 @@ func (p *parser) markSyntaxFeature(feature compat.JSFeature, r logger.Range) (di } var name string - var notes []logger.MsgData - where := "the configured target environment" - - if tsTarget := p.options.tsTarget; tsTarget != nil && tsTarget.UnsupportedJSFeatures.Has(feature) { - tracker := logger.MakeLineColumnTracker(&tsTarget.Source) - where = fmt.Sprintf("%s (%q)", where, tsTarget.Target) - notes = []logger.MsgData{logger.RangeData(&tracker, tsTarget.Range, fmt.Sprintf( - "The target environment was set to %q here", tsTarget.Target))} - } else if p.options.originalTargetEnv != "" { - where = fmt.Sprintf("%s (%s)", where, p.options.originalTargetEnv) - } + where, notes := p.prettyPrintTargetEnvironment(feature) switch feature { case compat.DefaultArgument: diff --git a/internal/js_parser/js_parser_test.go b/internal/js_parser/js_parser_test.go index a76d86fd9a1..1df90c115aa 100644 --- a/internal/js_parser/js_parser_test.go +++ b/internal/js_parser/js_parser_test.go @@ -149,6 +149,30 @@ func expectPrintedJSX(t *testing.T, contents string, expected string) { }) } +func expectParseErrorTargetJSX(t *testing.T, esVersion int, contents string, expected string) { + t.Helper() + expectParseErrorCommon(t, contents, expected, config.Options{ + UnsupportedJSFeatures: compat.UnsupportedJSFeatures(map[compat.Engine][]int{ + compat.ES: {esVersion}, + }), + JSX: config.JSXOptions{ + Parse: true, + }, + }) +} + +func expectPrintedTargetJSX(t *testing.T, esVersion int, contents string, expected string) { + t.Helper() + expectPrintedCommon(t, contents, expected, config.Options{ + UnsupportedJSFeatures: compat.UnsupportedJSFeatures(map[compat.Engine][]int{ + compat.ES: {esVersion}, + }), + JSX: config.JSXOptions{ + Parse: true, + }, + }) +} + func TestBinOp(t *testing.T) { for code, entry := range js_ast.OpTable { opCode := js_ast.OpCode(code) @@ -4606,8 +4630,8 @@ func TestES5(t *testing.T) { } func TestASCIIOnly(t *testing.T) { - es5 := ": error: \"𐀀\" cannot be escaped in the target environment " + - "(consider setting the charset to \"utf8\" or changing the target)\n" + es5 := ": error: \"𐀀\" cannot be escaped in the configured target environment " + + "but you can set the charset to \"utf8\" to allow unescaped Unicode characters\n" // Some context: "π" is in the BMP (i.e. has a code point ≤0xFFFF) and "𐀀" is // not in the BMP (i.e. has a code point >0xFFFF). This distinction matters @@ -4753,3 +4777,24 @@ func TestASCIIOnly(t *testing.T) { expectPrintedTargetASCII(t, 5, "export var π", "export var \\u03C0;\n") expectParseErrorTargetASCII(t, 5, "export var 𐀀", es5) } + +func TestUpdatedIdentifiers(t *testing.T) { + // Some context: The text "ꓷꓶꓲꓵꓭꓢꓱ" is all non-BMP code points and is a valid + // identifier in ES6+ but not in ES5. It must either be quoted or forbidden + // when it's used in ES5. + + expectPrinted(t, "x.ꓷꓶꓲꓵꓭꓢꓱ", "x.ꓷꓶꓲꓵꓭꓢꓱ;\n") + expectPrinted(t, "var ꓷꓶꓲꓵꓭꓢꓱ", "var ꓷꓶꓲꓵꓭꓢꓱ;\n") + expectPrintedTarget(t, 5, "x.ꓷꓶꓲꓵꓭꓢꓱ", "x[\"ꓷꓶꓲꓵꓭꓢꓱ\"];\n") + expectPrintedTarget(t, 5, "x = {ꓷꓶꓲꓵꓭꓢꓱ: 0}", "x = { \"ꓷꓶꓲꓵꓭꓢꓱ\": 0 };\n") + expectParseErrorTarget(t, 5, "ꓷꓶꓲꓵꓭꓢꓱ", + ": error: \"ꓷꓶꓲꓵꓭꓢꓱ\" is not considered a valid identifier in the configured target environment\n") + expectParseErrorTarget(t, 5, "var ꓷꓶꓲꓵꓭꓢꓱ", + ": error: \"ꓷꓶꓲꓵꓭꓢꓱ\" is not considered a valid identifier in the configured target environment\n") + + expectPrintedJSX(t, "", "/* @__PURE__ */ React.createElement(\"x\", {\n ꓷꓶꓲꓵꓭꓢꓱ: true\n});\n") + expectPrintedJSX(t, "<ꓷꓶꓲꓵꓭꓢꓱ/>", "/* @__PURE__ */ React.createElement(ꓷꓶꓲꓵꓭꓢꓱ, null);\n") + expectPrintedTargetJSX(t, 5, "", "/* @__PURE__ */ React.createElement(\"x\", {\n \"ꓷꓶꓲꓵꓭꓢꓱ\": true\n});\n") + expectParseErrorTargetJSX(t, 5, "<ꓷꓶꓲꓵꓭꓢꓱ/>", + ": error: \"ꓷꓶꓲꓵꓭꓢꓱ\" is not considered a valid identifier in the configured target environment\n") +} diff --git a/internal/js_parser/ts_parser.go b/internal/js_parser/ts_parser.go index 6485eaf31d6..582e2c09aea 100644 --- a/internal/js_parser/ts_parser.go +++ b/internal/js_parser/ts_parser.go @@ -842,7 +842,7 @@ func (p *parser) parseTypeScriptEnumStmt(loc logger.Loc, opts parseStmtOpts) js_ p.lexer.Next() // Identifiers can be referenced by other values - if !opts.isTypeScriptDeclare && js_lexer.IsIdentifierUTF16(value.Name) { + if !opts.isTypeScriptDeclare && js_lexer.IsIdentifierUTF16(value.Name, 0) { value.Ref = p.declareSymbol(js_ast.SymbolOther, value.Loc, js_lexer.UTF16ToString(value.Name)) } diff --git a/internal/js_printer/js_printer.go b/internal/js_printer/js_printer.go index 1bc89293b90..4f541c5b9e9 100644 --- a/internal/js_printer/js_printer.go +++ b/internal/js_printer/js_printer.go @@ -805,7 +805,7 @@ func (p *printer) printSymbol(ref js_ast.Ref) { } func (p *printer) printClauseAlias(alias string) { - if js_lexer.IsIdentifier(alias) { + if js_lexer.IsIdentifier(alias, p.options.UnsupportedFeatures) { p.printSpaceBeforeIdentifier() p.printIdentifier(alias) } else { @@ -814,19 +814,19 @@ func (p *printer) printClauseAlias(alias string) { } func CanQuoteIdentifier(name string, unsupportedJSFeatures compat.JSFeature, asciiOnly bool) bool { - return js_lexer.IsIdentifier(name) && (!asciiOnly || + return js_lexer.IsIdentifier(name, unsupportedJSFeatures) && (!asciiOnly || !unsupportedJSFeatures.Has(compat.UnicodeEscapes) || !js_lexer.ContainsNonBMPCodePoint(name)) } func (p *printer) canPrintIdentifier(name string) bool { - return js_lexer.IsIdentifier(name) && (!p.options.ASCIIOnly || + return js_lexer.IsIdentifier(name, p.options.UnsupportedFeatures) && (!p.options.ASCIIOnly || !p.options.UnsupportedFeatures.Has(compat.UnicodeEscapes) || !js_lexer.ContainsNonBMPCodePoint(name)) } func (p *printer) canPrintIdentifierUTF16(name []uint16) bool { - return js_lexer.IsIdentifierUTF16(name) && (!p.options.ASCIIOnly || + return js_lexer.IsIdentifierUTF16(name, p.options.UnsupportedFeatures) && (!p.options.ASCIIOnly || !p.options.UnsupportedFeatures.Has(compat.UnicodeEscapes) || !js_lexer.ContainsNonBMPCodePointUTF16(name)) } @@ -1058,7 +1058,7 @@ func (p *printer) printSemicolonIfNeeded() { func (p *printer) printSpaceBeforeIdentifier() { buffer := p.js n := len(buffer) - if n > 0 && (js_lexer.IsIdentifierContinue(rune(buffer[n-1])) || n == p.prevRegExpEnd) { + if n > 0 && (js_lexer.IsIdentifierContinue(rune(buffer[n-1]), 0) || n == p.prevRegExpEnd) { p.print(" ") } } diff --git a/internal/renamer/renamer.go b/internal/renamer/renamer.go index 866a817599b..04cf254e963 100644 --- a/internal/renamer/renamer.go +++ b/internal/renamer/renamer.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "github.com/evanw/esbuild/internal/ast" + "github.com/evanw/esbuild/internal/compat" "github.com/evanw/esbuild/internal/js_ast" "github.com/evanw/esbuild/internal/js_lexer" ) @@ -386,12 +387,13 @@ func (a slotAndCountArray) Less(i int, j int) bool { // NumberRenamer type NumberRenamer struct { - symbols js_ast.SymbolMap - names [][]string - root numberScope + symbols js_ast.SymbolMap + names [][]string + root numberScope + unsupportedJSFeatures compat.JSFeature } -func NewNumberRenamer(symbols js_ast.SymbolMap, reservedNames map[string]uint32) *NumberRenamer { +func NewNumberRenamer(symbols js_ast.SymbolMap, reservedNames map[string]uint32, unsupportedJSFeatures compat.JSFeature) *NumberRenamer { return &NumberRenamer{ symbols: symbols, names: make([][]string, len(symbols.SymbolsForSource)), @@ -437,7 +439,7 @@ func (r *NumberRenamer) assignName(scope *numberScope, ref js_ast.Ref) { } // Compute a new name - name := scope.findUnusedName(originalName) + name := scope.findUnusedName(originalName, r.unsupportedJSFeatures) // Store the new name if inner == nil { @@ -534,8 +536,8 @@ func (s *numberScope) findNameUse(name string) nameUse { } } -func (s *numberScope) findUnusedName(name string) string { - name = js_lexer.ForceValidIdentifier(name) +func (s *numberScope) findUnusedName(name string, unsupportedJSFeatures compat.JSFeature) string { + name = js_lexer.ForceValidIdentifier(name, unsupportedJSFeatures) if use := s.findNameUse(name); use != nameUnused { // If the name is already in use, generate a new name by appending a number diff --git a/internal/resolver/tsconfig_json.go b/internal/resolver/tsconfig_json.go index 9897732e928..fefa6b8e7cf 100644 --- a/internal/resolver/tsconfig_json.go +++ b/internal/resolver/tsconfig_json.go @@ -234,7 +234,7 @@ func parseMemberExpressionForJSX(log logger.Log, source *logger.Source, tracker } parts := strings.Split(text, ".") for _, part := range parts { - if !js_lexer.IsIdentifier(part) { + if !js_lexer.IsIdentifier(part, 0) { warnRange := source.RangeOfString(loc) log.AddRangeWarning(tracker, warnRange, fmt.Sprintf("Invalid JSX member expression: %q", text)) return nil diff --git a/pkg/api/api_impl.go b/pkg/api/api_impl.go index 89a4e9d612e..5fe3d9301c1 100644 --- a/pkg/api/api_impl.go +++ b/pkg/api/api_impl.go @@ -438,7 +438,7 @@ func validateDefines( for key, value := range defines { // The key must be a dot-separated identifier list for _, part := range strings.Split(key, ".") { - if !js_lexer.IsIdentifier(part) { + if !js_lexer.IsIdentifier(part, 0) { if part == key { log.AddError(nil, logger.Loc{}, fmt.Sprintf("The define key %q must be a valid identifier", key)) } else { @@ -449,7 +449,7 @@ func validateDefines( } // Allow substituting for an identifier - if js_lexer.IsIdentifier(value) { + if js_lexer.IsIdentifier(value, 0) { if _, ok := js_lexer.Keywords[value]; !ok { name := value // The closure must close over a variable inside the loop rawDefines[key] = config.DefineData{ @@ -538,7 +538,7 @@ func validateDefines( for _, key := range pureFns { // The key must be a dot-separated identifier list for _, part := range strings.Split(key, ".") { - if !js_lexer.IsIdentifier(part) { + if !js_lexer.IsIdentifier(part, 0) { log.AddError(nil, logger.Loc{}, fmt.Sprintf("Invalid pure function: %q", key)) continue } diff --git a/scripts/compat-table.js b/scripts/compat-table.js index a29e12ff466..98cc53952d1 100644 --- a/scripts/compat-table.js +++ b/scripts/compat-table.js @@ -37,6 +37,7 @@ const features = { 'class': { target: 'Class' }, 'generators': { target: 'Generator' }, 'Unicode code point escapes': { target: 'UnicodeEscapes' }, + 'Updated identifier syntax': { target: 'UpdatedIdentifiers' }, // >ES6 features 'exponentiation (**) operator': { target: 'ExponentOperator' }, @@ -140,6 +141,7 @@ mergeVersions('ObjectExtensions', { es2015: true }) mergeVersions('RestArgument', { es2015: true }) mergeVersions('TemplateLiteral', { es2015: true }) mergeVersions('UnicodeEscapes', { es2015: true }) +mergeVersions('UpdatedIdentifiers', { es2015: true }) // >ES6 features mergeVersions('ExponentOperator', { es2016: true }) diff --git a/scripts/gen-unicode-table.js b/scripts/gen-unicode-table.js index 6afbf6d383c..eaffeb2947d 100644 --- a/scripts/gen-unicode-table.js +++ b/scripts/gen-unicode-table.js @@ -1,8 +1,49 @@ const fs = require('fs'); const path = require('path'); -const version = '@unicode/unicode-13.0.0'; -const idStart = require(`${version}/Binary_Property/ID_Start/code-points`); -const idContinue = require(`${version}/Binary_Property/ID_Continue/code-points`); + +// ES5 reference: https://es5.github.io/ +// +// A conforming implementation of this International standard shall interpret +// characters in conformance with the Unicode Standard, Version 3.0 or later +// and ISO/IEC 10646-1 with either UCS-2 or UTF-16 as the adopted encoding +// form, implementation level 3. If the adopted ISO/IEC 10646-1 subset is not +// otherwise specified, it is presumed to be the BMP subset, collection 300. +// +// UnicodeLetter: any character in the Unicode categories “Uppercase letter (Lu)”, +// “Lowercase letter (Ll)”, “Titlecase letter (Lt)”, “Modifier letter (Lm)”, +// “Other letter (Lo)”, or “Letter number (Nl)”. +const idStartES5 = [].concat( + require('@unicode/unicode-3.0.0/General_Category/Uppercase_Letter/code-points'), + require('@unicode/unicode-3.0.0/General_Category/Lowercase_Letter/code-points'), + require('@unicode/unicode-3.0.0/General_Category/Titlecase_Letter/code-points'), + require('@unicode/unicode-3.0.0/General_Category/Modifier_Letter/code-points'), + require('@unicode/unicode-3.0.0/General_Category/Other_Letter/code-points'), + require('@unicode/unicode-3.0.0/General_Category/Letter_Number/code-points'), +).sort((a, b) => a - b) + +// UnicodeCombiningMark: any character in the Unicode categories “Non-spacing mark (Mn)” +// or “Combining spacing mark (Mc)” +// UnicodeDigit: any character in the Unicode category “Decimal number (Nd)” +// UnicodeConnectorPunctuation: any character in the Unicode category “Connector punctuation (Pc)” +const idContinueES5 = idStartES5.concat( + require('@unicode/unicode-3.0.0/General_Category/Nonspacing_Mark/code-points'), + require('@unicode/unicode-3.0.0/General_Category/Spacing_Mark/code-points'), + require('@unicode/unicode-3.0.0/General_Category/Decimal_Number/code-points'), + require('@unicode/unicode-3.0.0/General_Category/Connector_Punctuation/code-points'), +).sort((a, b) => a - b) + +// ESNext reference: https://tc39.es/ecma262/ +// +// A conforming implementation of ECMAScript must interpret source text input +// in conformance with the Unicode Standard, Version 5.1.0 or later and ISO/IEC +// 10646. If the adopted ISO/IEC 10646-1 subset is not otherwise specified, it +// is presumed to be the Unicode set, collection 10646. +// +// UnicodeIDStart: any Unicode code point with the Unicode property “ID_Start” +const idStart = require('@unicode/unicode-13.0.0/Binary_Property/ID_Start/code-points'); + +// UnicodeIDContinue: any Unicode code point with the Unicode property “ID_Continue” +const idContinue = require('@unicode/unicode-13.0.0/Binary_Property/ID_Continue/code-points'); function generateRangeTable(codePoints) { let lines = []; @@ -59,6 +100,10 @@ package js_lexer import "unicode" +var idStartES5 = ${generateRangeTable(idStartES5)} + +var idContinueES5 = ${generateRangeTable(idContinueES5)} + var idStart = ${generateRangeTable(idStart)} var idContinue = ${generateRangeTable(idContinue)} diff --git a/scripts/package-lock.json b/scripts/package-lock.json index 6ec296a0b34..b71d4e24610 100644 --- a/scripts/package-lock.json +++ b/scripts/package-lock.json @@ -7,6 +7,7 @@ "dependencies": { "@types/node": "14.14.6", "@unicode/unicode-13.0.0": "1.0.6", + "@unicode/unicode-3.0.0": "1.0.6", "fuse.js": "3.2.0", "js-yaml": "3.14.0", "react": "17.0.1", @@ -24,6 +25,11 @@ "resolved": "https://registry.npmjs.org/@unicode/unicode-13.0.0/-/unicode-13.0.0-1.0.6.tgz", "integrity": "sha512-ywSlML1wdeBSGH12b6PkblGtjzsQNMZ7+JlJj+eoPj0VuCeg52hcUTeVWfd4GmISU+zOQHWjNC6c0xmsIuVy8Q==" }, + "node_modules/@unicode/unicode-3.0.0": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@unicode/unicode-3.0.0/-/unicode-3.0.0-1.0.6.tgz", + "integrity": "sha512-eHhtHouHRCE3r0Sq4Spp/P6z1S000pEYkimT2d1xRoYrUDBUJ2TXbIfn96DgdslhBDtoIrnq2kdsCHWnh5RmBg==" + }, "node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -134,6 +140,11 @@ "resolved": "https://registry.npmjs.org/@unicode/unicode-13.0.0/-/unicode-13.0.0-1.0.6.tgz", "integrity": "sha512-ywSlML1wdeBSGH12b6PkblGtjzsQNMZ7+JlJj+eoPj0VuCeg52hcUTeVWfd4GmISU+zOQHWjNC6c0xmsIuVy8Q==" }, + "@unicode/unicode-3.0.0": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@unicode/unicode-3.0.0/-/unicode-3.0.0-1.0.6.tgz", + "integrity": "sha512-eHhtHouHRCE3r0Sq4Spp/P6z1S000pEYkimT2d1xRoYrUDBUJ2TXbIfn96DgdslhBDtoIrnq2kdsCHWnh5RmBg==" + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", diff --git a/scripts/package.json b/scripts/package.json index 0bafdae32d8..4451f22250c 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -2,6 +2,7 @@ "dependencies": { "@types/node": "14.14.6", "@unicode/unicode-13.0.0": "1.0.6", + "@unicode/unicode-3.0.0": "1.0.6", "fuse.js": "3.2.0", "js-yaml": "3.14.0", "react": "17.0.1",