[Rust] Fix string whitespace skip

- Recognize string prefixes as `LiteralStringAffix` token - Remove an unnecessary rule?
alecthomas · Jan 14, 2023 · 482cb62 · 482cb62
1 parent 4aaf294
commit 482cb62
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 16 deletions.
diff --git a/lexers/embedded/rust.xml b/lexers/embedded/rust.xml
@@ -113,7 +113,7 @@
         <token type="LiteralString"/>
         <pop depth="1"/>
       </rule>
-      <rule pattern="\\[&#39;&#34;\\nrt]|\\x[0-7][0-9a-fA-F]|\\0|\\u\{[0-9a-fA-F]{1,6}\}">
+      <rule pattern="\\[&#39;&#34;\\nrt]|\\(?=\n)|\\x[0-7][0-9a-fA-F]|\\0|\\u\{[0-9a-fA-F]{1,6}\}">
         <token type="LiteralStringEscape"/>
       </rule>
       <rule pattern="[^\\&#34;]+">
@@ -171,11 +171,9 @@
         <token type="CommentMultiline"/>
         <push state="comment"/>
       </rule>
-      <rule pattern="r#*&#34;(?:\\.|[^\\;])*&#34;#*">
-        <token type="LiteralString"/>
-      </rule>
-      <rule pattern="&#34;(?:\\.|[^\\&#34;])*&#34;">
+      <rule pattern="&#34;">
         <token type="LiteralString"/>
+        <push state="string"/>
       </rule>
       <rule pattern="\$([a-zA-Z_]\w*|\(,?|\),?|,?)">
         <token type="CommentPreproc"/>
@@ -243,8 +241,11 @@
       <rule pattern="&#39;(\\[&#39;&#34;\\nrt]|\\x[0-7][0-9a-fA-F]|\\0|\\u\{[0-9a-fA-F]{1,6}\}|.)&#39;">
         <token type="LiteralStringChar"/>
       </rule>
-      <rule pattern="b&#39;(\\[&#39;&#34;\\nrt]|\\x[0-9a-fA-F]{2}|\\0|\\u\{[0-9a-fA-F]{1,6}\}|.)&#39;">
-        <token type="LiteralStringChar"/>
+      <rule pattern="(b)(&#39;(\\[&#39;&#34;\\nrt]|\\x[0-9a-fA-F]{2}|\\0|\\u\{[0-9a-fA-F]{1,6}\}|.)&#39;)">
+        <bygroups>
+          <token type="LiteralStringAffix"/>
+          <token type="LiteralStringChar"/>
+        </bygroups>
       </rule>
       <rule pattern="0b[01_]+">
         <token type="LiteralNumberBin"/>
@@ -266,12 +267,19 @@
         <token type="LiteralNumberInteger"/>
         <push state="number_lit"/>
       </rule>
-      <rule pattern="b&#34;">
-        <token type="LiteralString"/>
+      <rule pattern="(b)(&#34;)">
+        <bygroups>
+          <token type="LiteralStringAffix"/>
+          <token type="LiteralString"/>
+        </bygroups>
         <push state="bytestring"/>
       </rule>
-      <rule pattern="(?s)b?r(#*)&#34;.*?&#34;\1">
-        <token type="LiteralString"/>
+      <rule pattern="(?s)(b?r)(#*)(&#34;.*?&#34;\2)">
+        <bygroups>
+          <token type="LiteralStringAffix"/>
+          <token type="LiteralString"/>
+          <token type="LiteralString"/>
+        </bygroups>
       </rule>
       <rule pattern="&#39;">
         <token type="Operator"/>

diff --git a/lexers/testdata/rust.actual b/lexers/testdata/rust.actual
@@ -15,9 +15,11 @@ fn main() {
     let r#type = "valid";
     let i: i128 = 1117;
 
-    let s = "March
+    let s = "March\n
 April";
-    let s = r"March
+    let s = r"March \
 April";
 
+    let s = "String split over\n multiple \
+    lines";
 }
diff --git a/lexers/testdata/rust.expected b/lexers/testdata/rust.expected
@@ -92,7 +92,9 @@
   {"type":"TextWhitespace","value":" "},
   {"type":"Operator","value":"="},
   {"type":"TextWhitespace","value":" "},
-  {"type":"LiteralString","value":"\"March\nApril\""},
+  {"type":"LiteralString","value":"\"March"},
+  {"type":"LiteralStringEscape","value":"\\n"},
+  {"type":"LiteralString","value":"\nApril\""},
   {"type":"Punctuation","value":";"},
   {"type":"TextWhitespace","value":"\n    "},
   {"type":"KeywordDeclaration","value":"let"},
@@ -101,9 +103,23 @@
   {"type":"TextWhitespace","value":" "},
   {"type":"Operator","value":"="},
   {"type":"TextWhitespace","value":" "},
-  {"type":"LiteralString","value":"r\"March\nApril\""},
+  {"type":"LiteralStringAffix","value":"r"},
+  {"type":"LiteralString","value":"\"March \\\nApril\""},
   {"type":"Punctuation","value":";"},
-  {"type":"TextWhitespace","value":"\n\n"},
+  {"type":"TextWhitespace","value":"\n\n    "},
+  {"type":"KeywordDeclaration","value":"let"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Name","value":"s"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralString","value":"\"String split over"},
+  {"type":"LiteralStringEscape","value":"\\n"},
+  {"type":"LiteralString","value":" multiple "},
+  {"type":"LiteralStringEscape","value":"\\"},
+  {"type":"LiteralString","value":"\n    lines\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"Punctuation","value":"}"},
   {"type":"TextWhitespace","value":"\n"}
 ]