Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Crystal: Improved tokenization (#3194)
  • Loading branch information
RunDevelopment committed Nov 22, 2021
1 parent e137035 commit 51e3ecc
Show file tree
Hide file tree
Showing 11 changed files with 759 additions and 66 deletions.
68 changes: 37 additions & 31 deletions components/prism-crystal.js
@@ -1,51 +1,57 @@
(function (Prism) {
Prism.languages.crystal = Prism.languages.extend('ruby', {
keyword: [
/\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|uninitialized|union|unless|until|when|while|with|yield)\b/,
'keyword': [
/\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|annotation|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|ifdef|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|ptr|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|undef|uninitialized|union|unless|until|when|while|with|yield)\b/,
{
pattern: /(\.\s*)(?:is_a|responds_to)\?/,
lookbehind: true
}
],

number: /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/
'number': /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/,
'operator': [
/->/,
Prism.languages.ruby.operator,
],
'punctuation': /[(){}[\].,;\\]/,
});

Prism.languages.insertBefore('crystal', 'string-literal', {
attribute: {
pattern: /@\[.+?\]/,
alias: 'attr-name',
'attribute': {
pattern: /@\[.*?\]/,
inside: {
delimiter: {
'delimiter': {
pattern: /^@\[|\]$/,
alias: 'tag'
alias: 'punctuation'
},
'attribute': {
pattern: /^(\s*)\w+/,
lookbehind: true,
alias: 'class-name'
},
'args': {
pattern: /\S(?:[\s\S]*\S)?/,
inside: Prism.languages.crystal
},
rest: Prism.languages.crystal
}
},

expansion: [
{
pattern: /\{\{.+?\}\}/,
inside: {
delimiter: {
pattern: /^\{\{|\}\}$/,
alias: 'tag'
},
rest: Prism.languages.crystal
}
},
{
pattern: /\{%.+?%\}/,
inside: {
delimiter: {
pattern: /^\{%|%\}$/,
alias: 'tag'
},
rest: Prism.languages.crystal
'expansion': {
pattern: /\{(?:\{.*?\}|%.*?%)\}/,
inside: {
'content': {
pattern: /^(\{.)[\s\S]+(?=.\}$)/,
lookbehind: true,
inside: Prism.languages.crystal
},
'delimiter': {
pattern: /^\{[\{%]|[\}%]\}$/,
alias: 'operator'
}
}
]
},
'char': {
pattern: /'(?:[^\\\r\n]{1,2}|\\(?:.|u(?:[A-Fa-f0-9]{1,4}|\{[A-Fa-f0-9]{1,6}\})))'/,
greedy: true
}
});

}(Prism));
2 changes: 1 addition & 1 deletion components/prism-crystal.min.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

58 changes: 32 additions & 26 deletions tests/languages/crystal/attribute_feature.test
Expand Up @@ -8,47 +8,53 @@
[
["attribute", [
["delimiter", "@["],
"AlwaysInline",
["attribute", "AlwaysInline"],
["delimiter", "]"]
]],
["attribute", [
["delimiter", "@["],
"CallConvention",
["punctuation", "("],
["string-literal", [
["string", "\"X86_StdCall\""]
["attribute", "CallConvention"],
["args", [
["punctuation", "("],
["string-literal", [
["string", "\"X86_StdCall\""]
]],
["punctuation", ")"]
]],
["punctuation", ")"],
["delimiter", "]"]
]],
["attribute", [
["delimiter", "@["],
"MyAnnotation",
["punctuation", "("],
["symbol", "key"],
["operator", ":"],
["string-literal", [
["string", "\"value\""]
["attribute", "MyAnnotation"],
["args", [
["punctuation", "("],
["symbol", "key"],
["operator", ":"],
["string-literal", [
["string", "\"value\""]
]],
["punctuation", ","],
["symbol", "value"],
["operator", ":"],
["number", "123"],
["punctuation", ")"]
]],
["punctuation", ","],
["symbol", "value"],
["operator", ":"],
["number", "123"],
["punctuation", ")"],
["delimiter", "]"]
]],
["attribute", [
["delimiter", "@["],
"MyAnnotation",
["punctuation", "("],
["string-literal", [
["string", "\"foo\""]
["attribute", "MyAnnotation"],
["args", [
["punctuation", "("],
["string-literal", [
["string", "\"foo\""]
]],
["punctuation", ","],
["number", "123"],
["punctuation", ","],
["boolean", "false"],
["punctuation", ")"]
]],
["punctuation", ","],
["number", "123"],
["punctuation", ","],
["boolean", "false"],
["punctuation", ")"],
["delimiter", "]"]
]]
]
Expand Down
41 changes: 41 additions & 0 deletions tests/languages/crystal/char_feature.test
@@ -0,0 +1,41 @@
'a'
'z'
'0'
'_'
'あ'
'\'' # single quote
'\\' # backslash
'\a' # alert
'\b' # backspace
'\e' # escape
'\f' # form feed
'\n' # newline
'\r' # carriage return
'\t' # tab
'\v' # vertical tab
'\0' # null character
'\uFFFF' # hexadecimal unicode character
'\u{10FFFF}' # hexadecimal unicode character

----------------------------------------------------

[
["char", "'a'"],
["char", "'z'"],
["char", "'0'"],
["char", "'_'"],
["char", "'あ'"],
["char", "'\\''"], ["comment", "# single quote"],
["char", "'\\\\'"], ["comment", "# backslash"],
["char", "'\\a'"], ["comment", "# alert"],
["char", "'\\b'"], ["comment", "# backspace"],
["char", "'\\e'"], ["comment", "# escape"],
["char", "'\\f'"], ["comment", "# form feed"],
["char", "'\\n'"], ["comment", "# newline"],
["char", "'\\r'"], ["comment", "# carriage return"],
["char", "'\\t'"], ["comment", "# tab"],
["char", "'\\v'"], ["comment", "# vertical tab"],
["char", "'\\0'"], ["comment", "# null character"],
["char", "'\\uFFFF'"], ["comment", "# hexadecimal unicode character"],
["char", "'\\u{10FFFF}'"], ["comment", "# hexadecimal unicode character"]
]
24 changes: 16 additions & 8 deletions tests/languages/crystal/expansion_feature.test
Expand Up @@ -8,26 +8,34 @@
[
["expansion", [
["delimiter", "{{"],
["number", "1_u32"],
["content", [
["number", "1_u32"]
]],
["delimiter", "}}"]
]],
["expansion", [
["delimiter", "{%"],
["number", "2_u32"],
["content", [
["number", "2_u32"]
]],
["delimiter", "%}"]
]],
["expansion", [
["delimiter", "{{"],
["punctuation", "{"],
["number", "3_u32"],
["punctuation", "}"],
["content", [
["punctuation", "{"],
["number", "3_u32"],
["punctuation", "}"]
]],
["delimiter", "}}"]
]],
["expansion", [
["delimiter", "{%"],
["operator", "%"],
["number", "4_u32"],
["operator", "%"],
["content", [
["operator", "%"],
["number", "4_u32"],
["operator", "%"]
]],
["delimiter", "%}"]
]]
]
Expand Down
8 changes: 8 additions & 0 deletions tests/languages/crystal/keyword_feature.test
Expand Up @@ -2,6 +2,7 @@
.responds_to?
abstract
alias
annotation;
as
asm
begin
Expand All @@ -19,6 +20,7 @@ extend
for
fun
if
ifdef
include
instance_sizeof
lib
Expand All @@ -30,6 +32,7 @@ out
pointerof
private
protected
ptr
require
rescue
return
Expand All @@ -41,6 +44,7 @@ super
then
type
typeof
undef
uninitialized
union
unless
Expand All @@ -62,6 +66,7 @@ __LINE__
["punctuation", "."], ["keyword", "responds_to?"],
["keyword", "abstract"],
["keyword", "alias"],
["keyword", "annotation"], ["punctuation", ";"],
["keyword", "as"],
["keyword", "asm"],
["keyword", "begin"],
Expand All @@ -79,6 +84,7 @@ __LINE__
["keyword", "for"],
["keyword", "fun"],
["keyword", "if"],
["keyword", "ifdef"],
["keyword", "include"],
["keyword", "instance_sizeof"],
["keyword", "lib"],
Expand All @@ -90,6 +96,7 @@ __LINE__
["keyword", "pointerof"],
["keyword", "private"],
["keyword", "protected"],
["keyword", "ptr"],
["keyword", "require"],
["keyword", "rescue"],
["keyword", "return"],
Expand All @@ -101,6 +108,7 @@ __LINE__
["keyword", "then"],
["keyword", "type"],
["keyword", "typeof"],
["keyword", "undef"],
["keyword", "uninitialized"],
["keyword", "union"],
["keyword", "unless"],
Expand Down

0 comments on commit 51e3ecc

Please sign in to comment.