From 86028adba6f7a17c9cfebe50d969c8be8b08a8db Mon Sep 17 00:00:00 2001 From: Michael Schmidt Date: Mon, 22 Nov 2021 13:25:30 +0100 Subject: [PATCH] Ruby: Improved tokenization (#3193) --- components/prism-crystal.js | 2 +- components/prism-crystal.min.js | 2 +- components/prism-ruby.js | 217 +++--- components/prism-ruby.min.js | 2 +- .../languages/crystal/attribute_feature.test | 41 +- tests/languages/crystal/keyword_feature.test | 132 +++- tests/languages/erb+haml/erb_inclusion.test | 8 +- tests/languages/erb/erb_feature.test | 4 +- .../languages/erb/erb_in_markup_feature.test | 4 +- .../languages/haml/interpolation_feature.test | 4 +- tests/languages/haml/tag_feature.test | 30 +- tests/languages/ruby+haml/ruby_inclusion.test | 16 +- tests/languages/ruby/class-name_feature.test | 73 ++ tests/languages/ruby/command_feature.test | 105 +++ tests/languages/ruby/constant_feature.test | 4 +- tests/languages/ruby/issue1336.test | 6 +- tests/languages/ruby/keyword_feature.test | 4 +- .../ruby/method_definition_feature.test | 10 +- tests/languages/ruby/operator_feature.test | 80 +++ tests/languages/ruby/punctuation_feature.test | 20 + tests/languages/ruby/regex_feature.test | 142 ++-- tests/languages/ruby/string_feature.test | 661 +++++++++--------- tests/languages/ruby/symbol_feature.test | 67 +- 23 files changed, 1109 insertions(+), 525 deletions(-) create mode 100644 tests/languages/ruby/class-name_feature.test create mode 100644 tests/languages/ruby/command_feature.test create mode 100644 tests/languages/ruby/operator_feature.test create mode 100644 tests/languages/ruby/punctuation_feature.test diff --git a/components/prism-crystal.js b/components/prism-crystal.js index cda3f38b7b..cf877587d0 100644 --- a/components/prism-crystal.js +++ b/components/prism-crystal.js @@ -11,7 +11,7 @@ number: /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/ }); - Prism.languages.insertBefore('crystal', 'string', { + Prism.languages.insertBefore('crystal', 'string-literal', { attribute: { pattern: /@\[.+?\]/, alias: 'attr-name', diff --git a/components/prism-crystal.min.js b/components/prism-crystal.min.js index 0d58c05795..aca3593c69 100644 --- a/components/prism-crystal.min.js +++ b/components/prism-crystal.min.js @@ -1 +1 @@ -!function(e){e.languages.crystal=e.languages.extend("ruby",{keyword:[/\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|uninitialized|union|unless|until|when|while|with|yield)\b/,{pattern:/(\.\s*)(?:is_a|responds_to)\?/,lookbehind:!0}],number:/\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/}),e.languages.insertBefore("crystal","string",{attribute:{pattern:/@\[.+?\]/,alias:"attr-name",inside:{delimiter:{pattern:/^@\[|\]$/,alias:"tag"},rest:e.languages.crystal}},expansion:[{pattern:/\{\{.+?\}\}/,inside:{delimiter:{pattern:/^\{\{|\}\}$/,alias:"tag"},rest:e.languages.crystal}},{pattern:/\{%.+?%\}/,inside:{delimiter:{pattern:/^\{%|%\}$/,alias:"tag"},rest:e.languages.crystal}}]})}(Prism); \ No newline at end of file +!function(e){e.languages.crystal=e.languages.extend("ruby",{keyword:[/\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|uninitialized|union|unless|until|when|while|with|yield)\b/,{pattern:/(\.\s*)(?:is_a|responds_to)\?/,lookbehind:!0}],number:/\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/}),e.languages.insertBefore("crystal","string-literal",{attribute:{pattern:/@\[.+?\]/,alias:"attr-name",inside:{delimiter:{pattern:/^@\[|\]$/,alias:"tag"},rest:e.languages.crystal}},expansion:[{pattern:/\{\{.+?\}\}/,inside:{delimiter:{pattern:/^\{\{|\}\}$/,alias:"tag"},rest:e.languages.crystal}},{pattern:/\{%.+?%\}/,inside:{delimiter:{pattern:/^\{%|%\}$/,alias:"tag"},rest:e.languages.crystal}}]})}(Prism); \ No newline at end of file diff --git a/components/prism-ruby.js b/components/prism-ruby.js index 3fab30373d..79256dcf8a 100644 --- a/components/prism-ruby.js +++ b/components/prism-ruby.js @@ -6,50 +6,65 @@ */ (function (Prism) { Prism.languages.ruby = Prism.languages.extend('clike', { - 'comment': [ - /#.*/, - { - pattern: /^=begin\s[\s\S]*?^=end/m, - greedy: true - } - ], + 'comment': { + pattern: /#.*|^=begin\s[\s\S]*?^=end/m, + greedy: true + }, 'class-name': { - pattern: /(\b(?:class)\s+|\bcatch\s+\()[\w.\\]+/i, + pattern: /(\b(?:class|module)\s+|\bcatch\s+\()[\w.\\]+|\b[A-Z_]\w*(?=\s*\.\s*new\b)/, lookbehind: true, inside: { 'punctuation': /[.\\]/ } }, - 'keyword': /\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/ + 'keyword': /\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/, + 'operator': /\.{2,3}|&\.|===||[!=]?~|(?:&&|\|\||<<|>>|\*\*|[+\-*/%<>!^&|=])=?|[?:]/, + 'punctuation': /[(){}[\].,;]/, + }); + + Prism.languages.insertBefore('ruby', 'operator', { + 'double-colon': { + pattern: /::/, + alias: 'punctuation' + }, }); var interpolation = { - pattern: /#\{[^}]+\}/, + pattern: /((?:^|[^\\])(?:\\{2})*)#\{(?:[^{}]|\{[^{}]*\})*\}/, + lookbehind: true, inside: { + 'content': { + pattern: /^(#\{)[\s\S]+(?=\}$)/, + lookbehind: true, + inside: Prism.languages.ruby + }, 'delimiter': { pattern: /^#\{|\}$/, - alias: 'tag' - }, - rest: Prism.languages.ruby + alias: 'punctuation' + } } }; delete Prism.languages.ruby.function; + var percentExpression = '(?:' + [ + /([^a-zA-Z0-9\s{(\[<=])(?:(?!\1)[^\\]|\\[\s\S])*\1/.source, + /\((?:[^()\\]|\\[\s\S]|\((?:[^()\\]|\\[\s\S])*\))*\)/.source, + /\{(?:[^{}\\]|\\[\s\S]|\{(?:[^{}\\]|\\[\s\S])*\})*\}/.source, + /\[(?:[^\[\]\\]|\\[\s\S]|\[(?:[^\[\]\\]|\\[\s\S])*\])*\]/.source, + /<(?:[^<>\\]|\\[\s\S]|<(?:[^<>\\]|\\[\s\S])*>)*>/.source + ].join('|') + ')'; + + var symbolName = /(?:"(?:\\.|[^"\\\r\n])*"|(?:\b[a-zA-Z_]\w*|[^\s\0-\x7F]+)[?!]?|\$.)/.source; + Prism.languages.insertBefore('ruby', 'keyword', { - 'regex': [ + 'regex-literal': [ { - pattern: RegExp(/%r/.source + '(?:' + [ - /([^a-zA-Z0-9\s{(\[<])(?:(?!\1)[^\\]|\\[\s\S])*\1/.source, - /\((?:[^()\\]|\\[\s\S])*\)/.source, - // Here we need to specifically allow interpolation - /\{(?:[^#{}\\]|#(?:\{[^}]+\})?|\\[\s\S])*\}/.source, - /\[(?:[^\[\]\\]|\\[\s\S])*\]/.source, - /<(?:[^<>\\]|\\[\s\S])*>/.source - ].join('|') + ')' + /[egimnosux]{0,6}/.source), + pattern: RegExp(/%r/.source + percentExpression + /[egimnosux]{0,6}/.source), greedy: true, inside: { - 'interpolation': interpolation + 'interpolation': interpolation, + 'regex': /[\s\S]+/ } }, { @@ -57,82 +72,118 @@ lookbehind: true, greedy: true, inside: { - 'interpolation': interpolation + 'interpolation': interpolation, + 'regex': /[\s\S]+/ } } ], 'variable': /[@$]+[a-zA-Z_]\w*(?:[?!]|\b)/, - 'symbol': { - pattern: /(^|[^:]):[a-zA-Z_]\w*(?:[?!]|\b)/, - lookbehind: true - }, + 'symbol': [ + { + pattern: RegExp(/(^|[^:]):/.source + symbolName), + lookbehind: true, + greedy: true + }, + { + pattern: RegExp(/([\r\n{(,][ \t]*)/.source + symbolName + /(?=:(?!:))/.source), + lookbehind: true, + greedy: true + }, + ], 'method-definition': { - pattern: /(\bdef\s+)[\w.]+/, + pattern: /(\bdef\s+)\w+(?:\s*\.\s*\w+)?/, lookbehind: true, inside: { - 'function': /\w+$/, - rest: Prism.languages.ruby + 'function': /\b\w+$/, + 'keyword': /^self\b/, + 'class-name': /^\w+/, + 'punctuation': /\./ } } }); - Prism.languages.insertBefore('ruby', 'number', { - 'builtin': /\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/, - 'constant': /\b[A-Z]\w*(?:[?!]|\b)/ - }); - - Prism.languages.ruby.string = [ - { - pattern: RegExp(/%[qQiIwWxs]?/.source + '(?:' + [ - /([^a-zA-Z0-9\s{(\[<])(?:(?!\1)[^\\]|\\[\s\S])*\1/.source, - /\((?:[^()\\]|\\[\s\S])*\)/.source, - // Here we need to specifically allow interpolation - /\{(?:[^#{}\\]|#(?:\{[^}]+\})?|\\[\s\S])*\}/.source, - /\[(?:[^\[\]\\]|\\[\s\S])*\]/.source, - /<(?:[^<>\\]|\\[\s\S])*>/.source - ].join('|') + ')'), - greedy: true, - inside: { - 'interpolation': interpolation - } - }, - { - pattern: /("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/, - greedy: true, - inside: { - 'interpolation': interpolation + Prism.languages.insertBefore('ruby', 'string', { + 'string-literal': [ + { + pattern: RegExp(/%[qQiIwWs]?/.source + percentExpression), + greedy: true, + inside: { + 'interpolation': interpolation, + 'string': /[\s\S]+/ + } + }, + { + pattern: /("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/, + greedy: true, + inside: { + 'interpolation': interpolation, + 'string': /[\s\S]+/ + } + }, + { + pattern: /<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i, + alias: 'heredoc-string', + greedy: true, + inside: { + 'delimiter': { + pattern: /^<<[-~]?[a-z_]\w*|\b[a-z_]\w*$/i, + inside: { + 'symbol': /\b\w+/, + 'punctuation': /^<<[-~]?/ + } + }, + 'interpolation': interpolation, + 'string': /[\s\S]+/ + } + }, + { + pattern: /<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i, + alias: 'heredoc-string', + greedy: true, + inside: { + 'delimiter': { + pattern: /^<<[-~]?'[a-z_]\w*'|\b[a-z_]\w*$/i, + inside: { + 'symbol': /\b\w+/, + 'punctuation': /^<<[-~]?'|'$/, + } + }, + 'string': /[\s\S]+/ + } } - }, - { - pattern: /<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i, - alias: 'heredoc-string', - greedy: true, - inside: { - 'delimiter': { - pattern: /^<<[-~]?[a-z_]\w*|[a-z_]\w*$/i, - alias: 'symbol', - inside: { - 'punctuation': /^<<[-~]?/ + ], + 'command-literal': [ + { + pattern: RegExp(/%x/.source + percentExpression), + greedy: true, + inside: { + 'interpolation': interpolation, + 'command': { + pattern: /[\s\S]+/, + alias: 'string' } - }, - 'interpolation': interpolation - } - }, - { - pattern: /<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i, - alias: 'heredoc-string', - greedy: true, - inside: { - 'delimiter': { - pattern: /^<<[-~]?'[a-z_]\w*'|[a-z_]\w*$/i, - alias: 'symbol', - inside: { - 'punctuation': /^<<[-~]?'|'$/, + } + }, + { + pattern: /`(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|[^\\`#\r\n])*`/, + greedy: true, + inside: { + 'interpolation': interpolation, + 'command': { + pattern: /[\s\S]+/, + alias: 'string' } } } - } - ]; + ] + }); + + delete Prism.languages.ruby.string; + + Prism.languages.insertBefore('ruby', 'number', { + 'builtin': /\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/, + 'constant': /\b[A-Z][A-Z0-9_]*(?:[?!]|\b)/ + }); Prism.languages.rb = Prism.languages.ruby; }(Prism)); diff --git a/components/prism-ruby.min.js b/components/prism-ruby.min.js index 31bb49d909..7c5bec2aae 100644 --- a/components/prism-ruby.min.js +++ b/components/prism-ruby.min.js @@ -1 +1 @@ -!function(e){e.languages.ruby=e.languages.extend("clike",{comment:[/#.*/,{pattern:/^=begin\s[\s\S]*?^=end/m,greedy:!0}],"class-name":{pattern:/(\b(?:class)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/});var n={pattern:/#\{[^}]+\}/,inside:{delimiter:{pattern:/^#\{|\}$/,alias:"tag"},rest:e.languages.ruby}};delete e.languages.ruby.function,e.languages.insertBefore("ruby","keyword",{regex:[{pattern:RegExp("%r(?:"+["([^a-zA-Z0-9\\s{(\\[<])(?:(?!\\1)[^\\\\]|\\\\[^])*\\1","\\((?:[^()\\\\]|\\\\[^])*\\)","\\{(?:[^#{}\\\\]|#(?:\\{[^}]+\\})?|\\\\[^])*\\}","\\[(?:[^\\[\\]\\\\]|\\\\[^])*\\]","<(?:[^<>\\\\]|\\\\[^])*>"].join("|")+")[egimnosux]{0,6}"),greedy:!0,inside:{interpolation:n}},{pattern:/(^|[^/])\/(?!\/)(?:\[[^\r\n\]]+\]|\\.|[^[/\\\r\n])+\/[egimnosux]{0,6}(?=\s*(?:$|[\r\n,.;})#]))/,lookbehind:!0,greedy:!0,inside:{interpolation:n}}],variable:/[@$]+[a-zA-Z_]\w*(?:[?!]|\b)/,symbol:{pattern:/(^|[^:]):[a-zA-Z_]\w*(?:[?!]|\b)/,lookbehind:!0},"method-definition":{pattern:/(\bdef\s+)[\w.]+/,lookbehind:!0,inside:{function:/\w+$/,rest:e.languages.ruby}}}),e.languages.insertBefore("ruby","number",{builtin:/\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/,constant:/\b[A-Z]\w*(?:[?!]|\b)/}),e.languages.ruby.string=[{pattern:RegExp("%[qQiIwWxs]?(?:"+["([^a-zA-Z0-9\\s{(\\[<])(?:(?!\\1)[^\\\\]|\\\\[^])*\\1","\\((?:[^()\\\\]|\\\\[^])*\\)","\\{(?:[^#{}\\\\]|#(?:\\{[^}]+\\})?|\\\\[^])*\\}","\\[(?:[^\\[\\]\\\\]|\\\\[^])*\\]","<(?:[^<>\\\\]|\\\\[^])*>"].join("|")+")"),greedy:!0,inside:{interpolation:n}},{pattern:/("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/,greedy:!0,inside:{interpolation:n}},{pattern:/<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i,alias:"heredoc-string",greedy:!0,inside:{delimiter:{pattern:/^<<[-~]?[a-z_]\w*|[a-z_]\w*$/i,alias:"symbol",inside:{punctuation:/^<<[-~]?/}},interpolation:n}},{pattern:/<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i,alias:"heredoc-string",greedy:!0,inside:{delimiter:{pattern:/^<<[-~]?'[a-z_]\w*'|[a-z_]\w*$/i,alias:"symbol",inside:{punctuation:/^<<[-~]?'|'$/}}}}],e.languages.rb=e.languages.ruby}(Prism); \ No newline at end of file +!function(e){e.languages.ruby=e.languages.extend("clike",{comment:{pattern:/#.*|^=begin\s[\s\S]*?^=end/m,greedy:!0},"class-name":{pattern:/(\b(?:class|module)\s+|\bcatch\s+\()[\w.\\]+|\b[A-Z_]\w*(?=\s*\.\s*new\b)/,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/,operator:/\.{2,3}|&\.|===||[!=]?~|(?:&&|\|\||<<|>>|\*\*|[+\-*/%<>!^&|=])=?|[?:]/,punctuation:/[(){}[\].,;]/}),e.languages.insertBefore("ruby","operator",{"double-colon":{pattern:/::/,alias:"punctuation"}});var n={pattern:/((?:^|[^\\])(?:\\{2})*)#\{(?:[^{}]|\{[^{}]*\})*\}/,lookbehind:!0,inside:{content:{pattern:/^(#\{)[\s\S]+(?=\}$)/,lookbehind:!0,inside:e.languages.ruby},delimiter:{pattern:/^#\{|\}$/,alias:"punctuation"}}};delete e.languages.ruby.function;var t="(?:"+["([^a-zA-Z0-9\\s{(\\[<=])(?:(?!\\1)[^\\\\]|\\\\[^])*\\1","\\((?:[^()\\\\]|\\\\[^]|\\((?:[^()\\\\]|\\\\[^])*\\))*\\)","\\{(?:[^{}\\\\]|\\\\[^]|\\{(?:[^{}\\\\]|\\\\[^])*\\})*\\}","\\[(?:[^\\[\\]\\\\]|\\\\[^]|\\[(?:[^\\[\\]\\\\]|\\\\[^])*\\])*\\]","<(?:[^<>\\\\]|\\\\[^]|<(?:[^<>\\\\]|\\\\[^])*>)*>"].join("|")+")",i='(?:"(?:\\\\.|[^"\\\\\r\n])*"|(?:\\b[a-zA-Z_]\\w*|[^\\s\0-\\x7F]+)[?!]?|\\$.)';e.languages.insertBefore("ruby","keyword",{"regex-literal":[{pattern:RegExp("%r"+t+"[egimnosux]{0,6}"),greedy:!0,inside:{interpolation:n,regex:/[\s\S]+/}},{pattern:/(^|[^/])\/(?!\/)(?:\[[^\r\n\]]+\]|\\.|[^[/\\\r\n])+\/[egimnosux]{0,6}(?=\s*(?:$|[\r\n,.;})#]))/,lookbehind:!0,greedy:!0,inside:{interpolation:n,regex:/[\s\S]+/}}],variable:/[@$]+[a-zA-Z_]\w*(?:[?!]|\b)/,symbol:[{pattern:RegExp("(^|[^:]):"+i),lookbehind:!0,greedy:!0},{pattern:RegExp("([\r\n{(,][ \t]*)"+i+"(?=:(?!:))"),lookbehind:!0,greedy:!0}],"method-definition":{pattern:/(\bdef\s+)\w+(?:\s*\.\s*\w+)?/,lookbehind:!0,inside:{function:/\b\w+$/,keyword:/^self\b/,"class-name":/^\w+/,punctuation:/\./}}}),e.languages.insertBefore("ruby","string",{"string-literal":[{pattern:RegExp("%[qQiIwWs]?"+t),greedy:!0,inside:{interpolation:n,string:/[\s\S]+/}},{pattern:/("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/,greedy:!0,inside:{interpolation:n,string:/[\s\S]+/}},{pattern:/<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i,alias:"heredoc-string",greedy:!0,inside:{delimiter:{pattern:/^<<[-~]?[a-z_]\w*|\b[a-z_]\w*$/i,inside:{symbol:/\b\w+/,punctuation:/^<<[-~]?/}},interpolation:n,string:/[\s\S]+/}},{pattern:/<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i,alias:"heredoc-string",greedy:!0,inside:{delimiter:{pattern:/^<<[-~]?'[a-z_]\w*'|\b[a-z_]\w*$/i,inside:{symbol:/\b\w+/,punctuation:/^<<[-~]?'|'$/}},string:/[\s\S]+/}}],"command-literal":[{pattern:RegExp("%x"+t),greedy:!0,inside:{interpolation:n,command:{pattern:/[\s\S]+/,alias:"string"}}},{pattern:/`(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|[^\\`#\r\n])*`/,greedy:!0,inside:{interpolation:n,command:{pattern:/[\s\S]+/,alias:"string"}}}]}),delete e.languages.ruby.string,e.languages.insertBefore("ruby","number",{builtin:/\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/,constant:/\b[A-Z][A-Z0-9_]*(?:[?!]|\b)/}),e.languages.rb=e.languages.ruby}(Prism); \ No newline at end of file diff --git a/tests/languages/crystal/attribute_feature.test b/tests/languages/crystal/attribute_feature.test index 4fb0bad1d0..61b486882b 100644 --- a/tests/languages/crystal/attribute_feature.test +++ b/tests/languages/crystal/attribute_feature.test @@ -1,17 +1,54 @@ @[AlwaysInline] @[CallConvention("X86_StdCall")] +@[MyAnnotation(key: "value", value: 123)] +@[MyAnnotation("foo", 123, false)] ---------------------------------------------------- [ ["attribute", [ ["delimiter", "@["], - ["constant", "AlwaysInline"], + "AlwaysInline", ["delimiter", "]"] ]], ["attribute", [ ["delimiter", "@["], - ["constant", "CallConvention"], ["punctuation", "("], ["string", [ "\"X86_StdCall\"" ]], ["punctuation", ")"], + "CallConvention", + ["punctuation", "("], + ["string-literal", [ + ["string", "\"X86_StdCall\""] + ]], + ["punctuation", ")"], + ["delimiter", "]"] + ]], + ["attribute", [ + ["delimiter", "@["], + "MyAnnotation", + ["punctuation", "("], + ["symbol", "key"], + ["operator", ":"], + ["string-literal", [ + ["string", "\"value\""] + ]], + ["punctuation", ","], + ["symbol", "value"], + ["operator", ":"], + ["number", "123"], + ["punctuation", ")"], + ["delimiter", "]"] + ]], + ["attribute", [ + ["delimiter", "@["], + "MyAnnotation", + ["punctuation", "("], + ["string-literal", [ + ["string", "\"foo\""] + ]], + ["punctuation", ","], + ["number", "123"], + ["punctuation", ","], + ["boolean", "false"], + ["punctuation", ")"], ["delimiter", "]"] ]] ] diff --git a/tests/languages/crystal/keyword_feature.test b/tests/languages/crystal/keyword_feature.test index 962beadf84..d97c9d6063 100644 --- a/tests/languages/crystal/keyword_feature.test +++ b/tests/languages/crystal/keyword_feature.test @@ -1,35 +1,119 @@ -abstract alias as asm begin break case -class; -def; -do else elsif -end ensure enum extend for fun -if include instance_sizeof .is_a? -lib macro module next of out pointerof -private protected rescue .responds_to? -return require select self sizeof struct super -then type typeof uninitialized union unless -until when while with yield __DIR__ __END_LINE__ -__FILE__ __LINE__ +abstract +alias +as +asm +begin +break +case +class; +def; +do +else +elsif +end +ensure +enum +extend +for +fun +if +include +instance_sizeof +lib +macro +module; +next +of +out +pointerof +private +protected +require +rescue +return +select +self +sizeof +struct +super +then +type +typeof +uninitialized +union +unless +until +when +while +with +yield + +__DIR__ +__END_LINE__ +__FILE__ +__LINE__ ---------------------------------------------------- [ - ["keyword", "abstract"], ["keyword", "alias"], ["keyword", "as"], ["keyword", "asm"], ["keyword", "begin"], ["keyword", "break"], ["keyword", "case"], - ["keyword", "class"], ["punctuation", ";"], - ["keyword", "def"], ["punctuation", ";"], - ["keyword", "do"], ["keyword", "else"], ["keyword", "elsif"], - ["keyword", "end"], ["keyword", "ensure"], ["keyword", "enum"], ["keyword", "extend"], ["keyword", "for"], ["keyword", "fun"], - ["keyword", "if"], ["keyword", "include"], ["keyword", "instance_sizeof"], ["punctuation", "."], ["keyword", "is_a?"], - ["keyword", "lib"], ["keyword", "macro"], ["keyword", "module"], ["keyword", "next"], ["keyword", "of"], ["keyword", "out"], ["keyword", "pointerof"], - ["keyword", "private"], ["keyword", "protected"], ["keyword", "rescue"], ["punctuation", "."], ["keyword", "responds_to?"], - ["keyword", "return"], ["keyword", "require"], ["keyword", "select"], ["keyword", "self"], ["keyword", "sizeof"], ["keyword", "struct"], ["keyword", "super"], - ["keyword", "then"], ["keyword", "type"], ["keyword", "typeof"], ["keyword", "uninitialized"], ["keyword", "union"], ["keyword", "unless"], - ["keyword", "until"], ["keyword", "when"], ["keyword", "while"], ["keyword", "with"], ["keyword", "yield"], ["keyword", "__DIR__"], ["keyword", "__END_LINE__"], - ["keyword", "__FILE__"], ["keyword", "__LINE__"] + ["keyword", "abstract"], + ["keyword", "alias"], + ["keyword", "as"], + ["keyword", "asm"], + ["keyword", "begin"], + ["keyword", "break"], + ["keyword", "case"], + ["keyword", "class"], ["punctuation", ";"], + ["keyword", "def"], ["punctuation", ";"], + ["keyword", "do"], + ["keyword", "else"], + ["keyword", "elsif"], + ["keyword", "end"], + ["keyword", "ensure"], + ["keyword", "enum"], + ["keyword", "extend"], + ["keyword", "for"], + ["keyword", "fun"], + ["keyword", "if"], + ["keyword", "include"], + ["keyword", "instance_sizeof"], + ["keyword", "lib"], + ["keyword", "macro"], + ["keyword", "module"], ["punctuation", ";"], + ["keyword", "next"], + ["keyword", "of"], + ["keyword", "out"], + ["keyword", "pointerof"], + ["keyword", "private"], + ["keyword", "protected"], + ["keyword", "require"], + ["keyword", "rescue"], + ["keyword", "return"], + ["keyword", "select"], + ["keyword", "self"], + ["keyword", "sizeof"], + ["keyword", "struct"], + ["keyword", "super"], + ["keyword", "then"], + ["keyword", "type"], + ["keyword", "typeof"], + ["keyword", "uninitialized"], + ["keyword", "union"], + ["keyword", "unless"], + ["keyword", "until"], + ["keyword", "when"], + ["keyword", "while"], + ["keyword", "with"], + ["keyword", "yield"], + + ["keyword", "__DIR__"], + ["keyword", "__END_LINE__"], + ["keyword", "__FILE__"], + ["keyword", "__LINE__"] ] ---------------------------------------------------- diff --git a/tests/languages/erb+haml/erb_inclusion.test b/tests/languages/erb+haml/erb_inclusion.test index be2a7b1a12..b070d61f81 100644 --- a/tests/languages/erb+haml/erb_inclusion.test +++ b/tests/languages/erb+haml/erb_inclusion.test @@ -16,7 +16,9 @@ " render ", ["variable", "@products"], ["operator", "||"], - ["string", ["\"empty_list\""]] + ["string-literal", [ + ["string", "\"empty_list\""] + ]] ]], ["delimiter", "%>"] ]] @@ -30,7 +32,9 @@ " render ", ["variable", "@products"], ["operator", "||"], - ["string", ["\"empty_list\""]] + ["string-literal", [ + ["string", "\"empty_list\""] + ]] ]], ["delimiter", "%>"] ]] diff --git a/tests/languages/erb/erb_feature.test b/tests/languages/erb/erb_feature.test index 309680bb9a..698383487a 100644 --- a/tests/languages/erb/erb_feature.test +++ b/tests/languages/erb/erb_feature.test @@ -18,7 +18,9 @@ " render ", ["variable", "@products"], ["operator", "||"], - ["string", ["\"empty_list\""]] + ["string-literal", [ + ["string", "\"empty_list\""] + ]] ]], ["delimiter", "%>"] ]], diff --git a/tests/languages/erb/erb_in_markup_feature.test b/tests/languages/erb/erb_in_markup_feature.test index ca0793ad51..abc557e686 100644 --- a/tests/languages/erb/erb_in_markup_feature.test +++ b/tests/languages/erb/erb_in_markup_feature.test @@ -22,7 +22,9 @@ ___ERB1___<%= 1 %>___ERB2___<%= 2 %> ["punctuation", "."], "strftime", ["punctuation", "("], - ["string", ["'%A'"]], + ["string-literal", [ + ["string", "'%A'"] + ]], ["punctuation", ")"] ]], ["delimiter", "%>"] diff --git a/tests/languages/haml/interpolation_feature.test b/tests/languages/haml/interpolation_feature.test index 17d00e6924..58cb497d07 100644 --- a/tests/languages/haml/interpolation_feature.test +++ b/tests/languages/haml/interpolation_feature.test @@ -14,7 +14,9 @@ ["interpolation", [ ["delimiter", "#{"], ["ruby", [ - ["string", ["\"foobar\""]] + ["string-literal", [ + ["string", "\"foobar\""] + ]] ]], ["delimiter", "}"] ]] diff --git a/tests/languages/haml/tag_feature.test b/tests/languages/haml/tag_feature.test index 09f967f5e7..9bd73a2a5c 100644 --- a/tests/languages/haml/tag_feature.test +++ b/tests/languages/haml/tag_feature.test @@ -42,22 +42,24 @@ ["attributes", [ ["punctuation", "{"], ["symbol", ":type"], - ["operator", "="], - ["operator", ">"], - ["string", ["\"text/javascript\""]], + ["operator", "=>"], + ["string-literal", [ + ["string", "\"text/javascript\""] + ]], ["punctuation", ","], ["symbol", ":src"], - ["operator", "="], - ["operator", ">"], - ["string", [ - "\"javascripts/script_", + ["operator", "=>"], + ["string-literal", [ + ["string", "\"javascripts/script_"], ["interpolation", [ ["delimiter", "#{"], - ["number", "42"], + ["content", [ + ["number", "42"] + ]], ["delimiter", "}"] ]], - "\"" + ["string", "\""] ]], ["punctuation", "}"] ]] @@ -68,8 +70,7 @@ ["attributes", [ ["punctuation", "{"], ["symbol", ":id"], - ["operator", "="], - ["operator", ">"], + ["operator", "=>"], ["punctuation", "["], ["variable", "@item"], ["punctuation", "."], @@ -111,7 +112,9 @@ ["punctuation", "{"], "html_attrs", ["punctuation", "("], - ["string", ["'fr-fr'"]], + ["string-literal", [ + ["string", "'fr-fr'"] + ]], ["punctuation", ")"], ["punctuation", "}"] ]] @@ -142,8 +145,7 @@ ["attributes", [ ["punctuation", "{"], ["symbol", ":id"], - ["operator", "="], - ["operator", ">"], + ["operator", "=>"], ["variable", "@article"], ["punctuation", "."], "number", diff --git a/tests/languages/ruby+haml/ruby_inclusion.test b/tests/languages/ruby+haml/ruby_inclusion.test index f1bbc0f006..79d02505e7 100644 --- a/tests/languages/ruby+haml/ruby_inclusion.test +++ b/tests/languages/ruby+haml/ruby_inclusion.test @@ -20,14 +20,12 @@ ["function", "circumference"] ]], - ["constant", "Math"], - ["punctuation", ":"], - ["punctuation", ":"], + "\r\n\t\tMath", + ["double-colon", "::"], ["constant", "PI"], ["operator", "*"], " radius ", - ["operator", "*"], - ["operator", "*"], + ["operator", "**"], ["number", "2"], ["keyword", "end"] @@ -44,14 +42,12 @@ ["function", "circumference"] ]], - ["constant", "Math"], - ["punctuation", ":"], - ["punctuation", ":"], + "\r\n\t\t\tMath", + ["double-colon", "::"], ["constant", "PI"], ["operator", "*"], " radius ", - ["operator", "*"], - ["operator", "*"], + ["operator", "**"], ["number", "2"], ["keyword", "end"] diff --git a/tests/languages/ruby/class-name_feature.test b/tests/languages/ruby/class-name_feature.test new file mode 100644 index 0000000000..8386b00992 --- /dev/null +++ b/tests/languages/ruby/class-name_feature.test @@ -0,0 +1,73 @@ +class Customer + @@no_of_customers = 0 +end + +cust1 = Customer. new +cust2 = Customer. new + +class Accounts + def reading_charge + end + def Accounts.return_date + end +end + +class Salad + def self.buy_olive_oil + end +end + +---------------------------------------------------- + +[ + ["keyword", "class"], ["class-name", ["Customer"]], + ["variable", "@@no_of_customers"], ["operator", "="], ["number", "0"], + ["keyword", "end"], + + "\r\n\r\ncust1 ", + ["operator", "="], + ["class-name", ["Customer"]], + ["punctuation", "."], + ["keyword", "new"], + + "\r\ncust2 ", + ["operator", "="], + ["class-name", ["Customer"]], + ["punctuation", "."], + ["keyword", "new"], + + ["keyword", "class"], + ["class-name", ["Accounts"]], + + ["keyword", "def"], + ["method-definition", [ + ["function", "reading_charge"] + ]], + + ["keyword", "end"], + + ["keyword", "def"], + ["method-definition", [ + ["class-name", "Accounts"], + ["punctuation", "."], + ["function", "return_date"] + ]], + + ["keyword", "end"], + + ["keyword", "end"], + + ["keyword", "class"], + ["class-name", ["Salad"]], + + ["keyword", "def"], + ["method-definition", [ + ["keyword", "self"], + ["punctuation", "."], + ["function", "buy_olive_oil"] + ]], + + ["keyword", "end"], + + ["keyword", "end"] +] diff --git a/tests/languages/ruby/command_feature.test b/tests/languages/ruby/command_feature.test new file mode 100644 index 0000000000..b65f8e87e3 --- /dev/null +++ b/tests/languages/ruby/command_feature.test @@ -0,0 +1,105 @@ +`echo foo` +`echo #{user_input}` +`grep hosts /private/etc/* 2>&1` + +%x[ ls ] +%x{ ls } +%x + +%x!foo #{ 42 }! +%x(foo #{ 42 }) +%x{foo #{ 42 }} +%x[foo #{ 42 }] +%x + +---------------------------------------------------- + +[ + ["command-literal", [ + ["command", "`echo foo`"] + ]], + ["command-literal", [ + ["command", "`echo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", ["user_input"]], + ["delimiter", "}"] + ]], + ["command", "`"] + ]], + ["command-literal", [ + ["command", "`grep hosts /private/etc/* 2>&1`"] + ]], + + ["command-literal", [ + ["command", "%x[ ls ]"] + ]], + ["command-literal", [ + ["command", "%x{ ls }"] + ]], + ["command-literal", [ + ["command", "%x"] + ]], + + ["command-literal", [ + ["command", "%x!foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["command", "!"] + ]], + ["command-literal", [ + ["command", "%x(foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["command", ")"] + ]], + ["command-literal", [ + ["command", "%x{foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["command", "}"] + ]], + ["command-literal", [ + ["command", "%x[foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["command", "]"] + ]], + ["command-literal", [ + ["command", "%x"] + ]] +] diff --git a/tests/languages/ruby/constant_feature.test b/tests/languages/ruby/constant_feature.test index 220cf7463c..bb21cd3125 100644 --- a/tests/languages/ruby/constant_feature.test +++ b/tests/languages/ruby/constant_feature.test @@ -1,4 +1,3 @@ -Foobar FOO_BAR_42 F FOO @@ -8,7 +7,6 @@ BAZ! ---------------------------------------------------- [ - ["constant", "Foobar"], ["constant", "FOO_BAR_42"], ["constant", "F"], ["constant", "FOO"], @@ -18,4 +16,4 @@ BAZ! ---------------------------------------------------- -Checks for constants. \ No newline at end of file +Checks for constants. diff --git a/tests/languages/ruby/issue1336.test b/tests/languages/ruby/issue1336.test index 431bdf2ee6..51e864f210 100644 --- a/tests/languages/ruby/issue1336.test +++ b/tests/languages/ruby/issue1336.test @@ -5,11 +5,9 @@ Foo::Bar [ ["symbol", ":Foo"], - ["constant", "Foo"], - ["punctuation", ":"], ["punctuation", ":"], - ["constant", "Bar"] + "\r\nFoo", ["double-colon", "::"], "Bar" ] ---------------------------------------------------- -Ensures module syntax is not confused with symbols. See #1336 \ No newline at end of file +Ensures module syntax is not confused with symbols. See #1336 diff --git a/tests/languages/ruby/keyword_feature.test b/tests/languages/ruby/keyword_feature.test index 5d367dc284..db00719ad0 100644 --- a/tests/languages/ruby/keyword_feature.test +++ b/tests/languages/ruby/keyword_feature.test @@ -20,7 +20,7 @@ for if in include -module +module; new; next nil @@ -72,7 +72,7 @@ yield ["keyword", "if"], ["keyword", "in"], ["keyword", "include"], - ["keyword", "module"], + ["keyword", "module"], ["punctuation", ";"], ["keyword", "new"], ["punctuation", ";"], ["keyword", "next"], ["keyword", "nil"], diff --git a/tests/languages/ruby/method_definition_feature.test b/tests/languages/ruby/method_definition_feature.test index c8ed510c0a..8325663176 100644 --- a/tests/languages/ruby/method_definition_feature.test +++ b/tests/languages/ruby/method_definition_feature.test @@ -59,14 +59,12 @@ end ["function", "circumference"] ]], - ["constant", "Math"], - ["punctuation", ":"], - ["punctuation", ":"], + "\r\n Math", + ["double-colon", "::"], ["constant", "PI"], ["operator", "*"], " radius ", - ["operator", "*"], - ["operator", "*"], + ["operator", "**"], ["number", "2"], ["keyword", "end"], @@ -78,7 +76,7 @@ end ["function", "grow_by"] ]], " factor", - ["punctuation", ":"], + ["operator", ":"], ["variable", "@radius"], ["operator", "="], diff --git a/tests/languages/ruby/operator_feature.test b/tests/languages/ruby/operator_feature.test new file mode 100644 index 0000000000..c2172901dc --- /dev/null +++ b/tests/languages/ruby/operator_feature.test @@ -0,0 +1,80 @@ ++ - * / % ** ++= -= *= /= %= **= + +== != < > <= >= <=> === +!~ =~ += +& | ^ ~ << >> +&= |= ^= <<= >>= +&& || ! +&&= ||= + +=> + +&. + +? : +.. ... + +and or not + +---------------------------------------------------- + +[ + ["operator", "+"], + ["operator", "-"], + ["operator", "*"], + ["operator", "/"], + ["operator", "%"], + ["operator", "**"], + + ["operator", "+="], + ["operator", "-="], + ["operator", "*="], + ["operator", "/="], + ["operator", "%="], + ["operator", "**="], + + ["operator", "=="], + ["operator", "!="], + ["operator", "<"], + ["operator", ">"], + ["operator", "<="], + ["operator", ">="], + ["operator", "<=>"], + ["operator", "==="], + + ["operator", "!~"], + ["operator", "=~"], + + ["operator", "="], + + ["operator", "&"], + ["operator", "|"], + ["operator", "^"], + ["operator", "~"], + ["operator", "<<"], + ["operator", ">>"], + + ["operator", "&="], + ["operator", "|="], + ["operator", "^="], + ["operator", "<<="], + ["operator", ">>="], + + ["operator", "&&"], + ["operator", "||"], + ["operator", "!"], + + ["operator", "&&="], + ["operator", "||="], + + ["operator", "=>"], + + ["operator", "&."], + + ["operator", "?"], ["operator", ":"], + ["operator", ".."], ["operator", "..."], + + ["keyword", "and"], ["keyword", "or"], ["keyword", "not"] +] diff --git a/tests/languages/ruby/punctuation_feature.test b/tests/languages/ruby/punctuation_feature.test new file mode 100644 index 0000000000..fc24af160c --- /dev/null +++ b/tests/languages/ruby/punctuation_feature.test @@ -0,0 +1,20 @@ +( ) { } [ ] +. , ; +:: + +---------------------------------------------------- + +[ + ["punctuation", "("], + ["punctuation", ")"], + ["punctuation", "{"], + ["punctuation", "}"], + ["punctuation", "["], + ["punctuation", "]"], + + ["punctuation", "."], + ["punctuation", ","], + ["punctuation", ";"], + + ["double-colon", "::"] +] diff --git a/tests/languages/ruby/regex_feature.test b/tests/languages/ruby/regex_feature.test index aa035ffdd6..4cb9198fa3 100644 --- a/tests/languages/ruby/regex_feature.test +++ b/tests/languages/ruby/regex_feature.test @@ -26,111 +26,149 @@ ---------------------------------------------------- [ - ["regex", ["/[foo]\\/bar/gim"]], + ["regex-literal", [ + ["regex", "/[foo]\\/bar/gim"] + ]], - ["regex", ["/[bar]/"]], + ["regex-literal", [ + ["regex", "/[bar]/"] + ]], ["punctuation", ","], - ["regex", ["/./i"]], + ["regex-literal", [ + ["regex", "/./i"] + ]], ["punctuation", ";"], - ["regex", [ - "/foo", + ["regex-literal", [ + ["regex", "/foo"], ["interpolation", [ ["delimiter", "#{"], - "bar", + ["content", ["bar"]], ["delimiter", "}"] ]], - "/" + ["regex", "/"] ]], ["punctuation", ";"], - ["regex", ["/ab+c/ix"]], + ["regex-literal", [ + ["regex", "/ab+c/ix"] + ]], - ["regex", [ - "%r!foo?bar", + ["regex-literal", [ + ["regex", "%r!foo?bar"], ["interpolation", [ ["delimiter", "#{"], - ["number", "39"], - ["operator", "+"], - ["number", "3"], + ["content", [ + ["number", "39"], + ["operator", "+"], + ["number", "3"] + ]], ["delimiter", "}"] ]], - "!" + ["regex", "!"] ]], - ["regex", [ - "%r(foo?bar", + ["regex-literal", [ + ["regex", "%r(foo?bar"], ["interpolation", [ ["delimiter", "#{"], - ["number", "39"], - ["operator", "+"], - ["number", "3"], + ["content", [ + ["number", "39"], + ["operator", "+"], + ["number", "3"] + ]], ["delimiter", "}"] ]], - ")" + ["regex", ")"] ]], - ["regex", [ - "%r{foo?bar", + ["regex-literal", [ + ["regex", "%r{foo?bar"], ["interpolation", [ ["delimiter", "#{"], - ["number", "39"], - ["operator", "+"], - ["number", "3"], + ["content", [ + ["number", "39"], + ["operator", "+"], + ["number", "3"] + ]], ["delimiter", "}"] ]], - "}" + ["regex", "}"] ]], - ["regex", [ - "%r[foo?bar", + ["regex-literal", [ + ["regex", "%r[foo?bar"], ["interpolation", [ ["delimiter", "#{"], - ["number", "39"], - ["operator", "+"], - ["number", "3"], + ["content", [ + ["number", "39"], + ["operator", "+"], + ["number", "3"] + ]], ["delimiter", "}"] ]], - "]" + ["regex", "]"] ]], - ["regex", [ - "%r" + ["regex", ">"] ]], - ["regex", ["/foo/"]], + ["regex-literal", [ + ["regex", "/foo/"] + ]], ["comment", "# comment"], - ["regex", [ - "/foo", + ["regex-literal", [ + ["regex", "/foo"], ["interpolation", [ ["delimiter", "#{"], - "bar", + ["content", ["bar"]], ["delimiter", "}"] ]], - "/" + ["regex", "/"] ]], ["comment", "# comment"], ["comment", "# flags"], - ["regex", ["/abc/e"]], - ["regex", ["/abc/g"]], - ["regex", ["/abc/i"]], - ["regex", ["/abc/m"]], - ["regex", ["/abc/n"]], - ["regex", ["/abc/o"]], - ["regex", ["/abc/s"]], - ["regex", ["/abc/u"]], - ["regex", ["/abc/x"]] + ["regex-literal", [ + ["regex", "/abc/e"] + ]], + ["regex-literal", [ + ["regex", "/abc/g"] + ]], + ["regex-literal", [ + ["regex", "/abc/i"] + ]], + ["regex-literal", [ + ["regex", "/abc/m"] + ]], + ["regex-literal", [ + ["regex", "/abc/n"] + ]], + ["regex-literal", [ + ["regex", "/abc/o"] + ]], + ["regex-literal", [ + ["regex", "/abc/s"] + ]], + ["regex-literal", [ + ["regex", "/abc/u"] + ]], + ["regex-literal", [ + ["regex", "/abc/x"] + ]] ] ---------------------------------------------------- diff --git a/tests/languages/ruby/string_feature.test b/tests/languages/ruby/string_feature.test index f8a6473725..c3f9f08dcc 100644 --- a/tests/languages/ruby/string_feature.test +++ b/tests/languages/ruby/string_feature.test @@ -9,6 +9,7 @@ bar" "foo #bar" "foo #{ 42 } bar" +"\#{a + b}" %!foo #{ 42 }! %(foo #{ 42 }) @@ -30,11 +31,6 @@ bar" %W{foo #{ 42 }} %W[foo #{ 42 }] %W -%x!foo #{ 42 }! -%x(foo #{ 42 }) -%x{foo #{ 42 }} -%x[foo #{ 42 }] -%x <" - ]], - ["string", [ - "%Q!foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "!" - ]], - ["string", [ - "%Q(foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - ")" - ]], - ["string", [ - "%Q{foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "}" - ]], - ["string", [ - "%Q[foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "]" - ]], - ["string", [ - "%Q" - ]], - ["string", [ - "%I!foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "!" - ]], - ["string", [ - "%I(foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - ")" - ]], - ["string", [ - "%I{foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "}" - ]], - ["string", [ - "%I[foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "]" - ]], - ["string", [ - "%I" - ]], - ["string", [ - "%W!foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "!" - ]], - ["string", [ - "%W(foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - ")" - ]], - ["string", [ - "%W{foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "}" - ]], - ["string", [ - "%W[foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "]" - ]], - ["string", [ - "%W" - ]], - ["string", [ - "%x!foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "!" - ]], - ["string", [ - "%x(foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - ")" - ]], - ["string", [ - "%x{foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "}" - ]], - ["string", [ - "%x[foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - "]" - ]], - ["string", [ - "%x" - ]], - ["string", [ - ["delimiter", [ - ["punctuation", "<<"], - "STRING" - ]], - "\r\n foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - " bar\r\n", - ["delimiter", ["STRING"]] - ]], - ["string", [ - ["delimiter", [ - ["punctuation", "<<-"], - "STRING" - ]], - "\r\n foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - " bar\r\n ", - ["delimiter", ["STRING"]] - ]], - ["string", [ - ["delimiter", [ - ["punctuation", "<<~"], - "STRING" - ]], - "\r\n foo ", - ["interpolation", [ - ["delimiter", "#{"], - ["number", "42"], - ["delimiter", "}"] - ]], - " bar\r\n ", - ["delimiter", ["STRING"]] - ]], - ["string", [ - ["delimiter", [ - ["punctuation", "<<'"], - "STRING", - ["punctuation", "'"] - ]], - "\r\n foo #{42} bar\r\n", - ["delimiter", ["STRING"]] - ]], - ["string", [ - ["delimiter", [ - ["punctuation", "<<-'"], - "STRING", - ["punctuation", "'"] - ]], - "\r\n foo #{42} bar\r\n ", - ["delimiter", ["STRING"]] - ]], - ["string", [ - ["delimiter", [ - ["punctuation", "<<~'"], - "STRING", - ["punctuation", "'"] - ]], - "\r\n foo #{42} bar\r\n ", - ["delimiter", ["STRING"]] - ]] + ["string-literal", [ + ["string", "''"] + ]], + ["string-literal", [ + ["string", "\"\""] + ]], + ["string-literal", [ + ["string", "'foo'"] + ]], + ["string-literal", [ + ["string", "\"foo\""] + ]], + ["string-literal", [ + ["string", "'foo\\\r\nbar'"] + ]], + ["string-literal", [ + ["string", "\"foo\\\r\nbar\""] + ]], + + ["string-literal", [ + ["string", "\"foo #bar\""] + ]], + ["string-literal", [ + ["string", "\"foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", " bar\""] + ]], + ["string-literal", [ + ["string", "\"\\#{a + b}\""] + ]], + + ["string-literal", [ + ["string", "%!foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "!"] + ]], + ["string-literal", [ + ["string", "%(foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", ")"] + ]], + ["string-literal", [ + ["string", "%{foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "}"] + ]], + ["string-literal", [ + ["string", "%[foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "]"] + ]], + ["string-literal", [ + ["string", "%"] + ]], + ["string-literal", [ + ["string", "%Q!foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "!"] + ]], + ["string-literal", [ + ["string", "%Q(foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", ")"] + ]], + ["string-literal", [ + ["string", "%Q{foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "}"] + ]], + ["string-literal", [ + ["string", "%Q[foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "]"] + ]], + ["string-literal", [ + ["string", "%Q"] + ]], + ["string-literal", [ + ["string", "%I!foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "!"] + ]], + ["string-literal", [ + ["string", "%I(foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", ")"] + ]], + ["string-literal", [ + ["string", "%I{foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "}"] + ]], + ["string-literal", [ + ["string", "%I[foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "]"] + ]], + ["string-literal", [ + ["string", "%I"] + ]], + ["string-literal", [ + ["string", "%W!foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "!"] + ]], + ["string-literal", [ + ["string", "%W(foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", ")"] + ]], + ["string-literal", [ + ["string", "%W{foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "}"] + ]], + ["string-literal", [ + ["string", "%W[foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", "]"] + ]], + ["string-literal", [ + ["string", "%W"] + ]], + + ["string-literal", [ + ["delimiter", [ + ["punctuation", "<<"], + ["symbol", "STRING"] + ]], + ["string", "\r\n foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", " bar\r\n"], + ["delimiter", [ + ["symbol", "STRING"] + ]] + ]], + ["string-literal", [ + ["delimiter", [ + ["punctuation", "<<-"], + ["symbol", "STRING"] + ]], + ["string", "\r\n foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", " bar\r\n "], + ["delimiter", [ + ["symbol", "STRING"] + ]] + ]], + ["string-literal", [ + ["delimiter", [ + ["punctuation", "<<~"], + ["symbol", "STRING"] + ]], + ["string", "\r\n foo "], + ["interpolation", [ + ["delimiter", "#{"], + ["content", [ + ["number", "42"] + ]], + ["delimiter", "}"] + ]], + ["string", " bar\r\n "], + ["delimiter", [ + ["symbol", "STRING"] + ]] + ]], + ["string-literal", [ + ["delimiter", [ + ["punctuation", "<<'"], + ["symbol", "STRING"], + ["punctuation", "'"] + ]], + ["string", "\r\n foo #{42} bar\r\n"], + ["delimiter", [ + ["symbol", "STRING"] + ]] + ]], + ["string-literal", [ + ["delimiter", [ + ["punctuation", "<<-'"], + ["symbol", "STRING"], + ["punctuation", "'"] + ]], + ["string", "\r\n foo #{42} bar\r\n "], + ["delimiter", [ + ["symbol", "STRING"] + ]] + ]], + ["string-literal", [ + ["delimiter", [ + ["punctuation", "<<~'"], + ["symbol", "STRING"], + ["punctuation", "'"] + ]], + ["string", "\r\n foo #{42} bar\r\n "], + ["delimiter", [ + ["symbol", "STRING"] + ]] + ]] ] ---------------------------------------------------- diff --git a/tests/languages/ruby/symbol_feature.test b/tests/languages/ruby/symbol_feature.test index 6a51758056..c60db306d4 100644 --- a/tests/languages/ruby/symbol_feature.test +++ b/tests/languages/ruby/symbol_feature.test @@ -2,6 +2,19 @@ :foo :BAR? :Baz_42! +:あ +:"name" +:"\u{c4 d6 dc}" +:question? +:exclamation! +:$; + +:foo.object_id + +# in hashes + +{ :one => "eins", :two => "zwei", :three => "drei" } +{ one: "eins", two: "zwei", three: "drei" } ---------------------------------------------------- @@ -9,9 +22,59 @@ ["symbol", ":_"], ["symbol", ":foo"], ["symbol", ":BAR?"], - ["symbol", ":Baz_42!"] + ["symbol", ":Baz_42!"], + ["symbol", ":あ"], + ["symbol", ":\"name\""], + ["symbol", ":\"\\u{c4 d6 dc}\""], + ["symbol", ":question?"], + ["symbol", ":exclamation!"], + ["symbol", ":$;"], + + ["symbol", ":foo"], ["punctuation", "."], "object_id\r\n\r\n", + + ["comment", "# in hashes"], + + ["punctuation", "{"], + ["symbol", ":one"], + ["operator", "=>"], + ["string-literal", [ + ["string", "\"eins\""] + ]], + ["punctuation", ","], + ["symbol", ":two"], + ["operator", "=>"], + ["string-literal", [ + ["string", "\"zwei\""] + ]], + ["punctuation", ","], + ["symbol", ":three"], + ["operator", "=>"], + ["string-literal", [ + ["string", "\"drei\""] + ]], + ["punctuation", "}"], + + ["punctuation", "{"], + ["symbol", "one"], + ["operator", ":"], + ["string-literal", [ + ["string", "\"eins\""] + ]], + ["punctuation", ","], + ["symbol", "two"], + ["operator", ":"], + ["string-literal", [ + ["string", "\"zwei\""] + ]], + ["punctuation", ","], + ["symbol", "three"], + ["operator", ":"], + ["string-literal", [ + ["string", "\"drei\""] + ]], + ["punctuation", "}"] ] ---------------------------------------------------- -Checks for symbols. \ No newline at end of file +Checks for symbols.