Merge pull request #3603 from kitbellew/sbt299_17

LegacyScanner: allow repeated number separators
scalameta · Feb 29, 2024 · adc2272 · adc2272
2 parents 48a0221 + 2fff6ae
commit adc2272
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 10 deletions.
diff --git a/...ameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/LegacyScanner.scala b/...ameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/LegacyScanner.scala
@@ -765,12 +765,12 @@ class LegacyScanner(input: Input, dialect: Dialect) {
       putChar(ch)
       nextChar()
       readDigits(base)
-    } else if (wasSeparator) {
-      val pos = if (ch == SU) begCharOffset else begCharOffset - 1
-      syntaxError("trailing number separator", at = pos)
     } else if (isNumberSeparator()) {
       nextChar()
       readDigits(base, true)
+    } else if (wasSeparator) {
+      val pos = if (ch == SU) begCharOffset else begCharOffset - 1
+      syntaxError("trailing number separator", at = pos)
     }
 
   /**

diff --git a/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala b/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala
@@ -1136,7 +1136,17 @@ class TokenizerSuite extends BaseTokenizerSuite {
     }
   }
 
-  Seq("1_024", "1_024L", "3_14e-2", "3_14E-2_1").foreach { value =>
+  Seq(
+    "1_024",
+    "1_024L",
+    "3_14e-2",
+    "3_14E-2_1",
+    "3_14E-1__2",
+    "0x_1234",
+    "0b_0101",
+    "123_456__789",
+    "0x__123_456__789"
+  ).foreach { value =>
     test(s"numeric literal separator ok scala213: $value") {
       dialects.Scala213(value).tokenize.get // no exception
     }
@@ -1191,12 +1201,6 @@ class TokenizerSuite extends BaseTokenizerSuite {
          |3_14ef
          |    ^""".stripMargin
     ),
-    (
-      "3_14E-1__2",
-      """|<input>:1: error: trailing number separator
-         |3_14E-1__2
-         |       ^""".stripMargin
-    ),
     (
       "3_14E-2_",
       """|<input>:1: error: trailing number separator