Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Concurnas: Improved tokenization (#3189)
  • Loading branch information
RunDevelopment committed Nov 22, 2021
1 parent 6af8a64 commit 7b34e65
Show file tree
Hide file tree
Showing 8 changed files with 107 additions and 38 deletions.
43 changes: 28 additions & 15 deletions components/prism-concurnas.js
@@ -1,19 +1,20 @@
Prism.languages.concurnas = {
'comment': [
{
pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,
lookbehind: true
},
{
pattern: /(^|[^\\:])\/\/.*/,
lookbehind: true,
greedy: true
}
],
'comment': {
pattern: /(^|[^\\])(?:\/\*[\s\S]*?(?:\*\/|$)|\/\/.*)/,
lookbehind: true,
greedy: true
},
'langext': {
pattern: /\b\w+\s*\|\|[\s\S]+?\|\|/,
greedy: true,
alias: 'string'
inside: {
'class-name': /^\w+/,
'string': {
pattern: /(^\s*\|\|)[\s\S]+(?=\|\|$)/,
lookbehind: true
},
'punctuation': /\|\|/
}
},
'function': {
pattern: /((?:^|\s)def[ \t]+)[a-zA-Z_]\w*(?=\s*\()/,
Expand All @@ -23,16 +24,28 @@ Prism.languages.concurnas = {
'boolean': /\b(?:false|true)\b/,
'number': /\b0b[01][01_]*L?\b|\b0x(?:[\da-f_]*\.)?[\da-f_p+-]+\b|(?:\b\d[\d_]*(?:\.[\d_]*)?|\B\.\d[\d_]*)(?:e[+-]?\d[\d_]*)?[dfls]?/i,
'punctuation': /[{}[\];(),.:]/,
'operator': /<==|>==|=>|->|<-|<>|\^|&==|&<>|!|\?:?|\.\?|\+\+|--|[-+*/=<>]=?|\b(?:and|as|band|bor|bxor|comp|is|isnot|mod|or)\b=?/,
'operator': /<==|>==|=>|->|<-|<>|&==|&<>|\?:?|\.\?|\+\+|--|[-+*/=<>]=?|[!^~]|\b(?:and|as|band|bor|bxor|comp|is|isnot|mod|or)\b=?/,
'annotation': {
pattern: /@(?:\w+:)?(?:\w+|\[[^\]]+\])?/,
alias: 'builtin'
}
};

Prism.languages.insertBefore('concurnas', 'langext', {
'string': {
pattern: /[rs]?("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,
'regex-literal': {
pattern: /\br("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,
greedy: true,
inside: {
'interpolation': {
pattern: /((?:^|[^\\])(?:\\{2})*)\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}/,
lookbehind: true,
inside: Prism.languages.concurnas
},
'regex': /[\s\S]+/
}
},
'string-literal': {
pattern: /(?:\B|\bs)("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,
greedy: true,
inside: {
'interpolation': {
Expand Down
2 changes: 1 addition & 1 deletion components/prism-concurnas.min.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions tests/languages/concurnas/comment_feature.test
@@ -0,0 +1,11 @@
// comment
/*
comment
*/

----------------------------------------------------

[
["comment", "// comment"],
["comment", "/*\r\ncomment\r\n*/"]
]
14 changes: 11 additions & 3 deletions tests/languages/concurnas/function_feature.test
Expand Up @@ -4,10 +4,18 @@ myfunc()
----------------------------------------------------

[
["keyword", "def"], ["function", "myfunc"], ["punctuation", "("], ["punctuation", ")"], ["operator", "=>"], ["number", "12"],
"\r\nmyfunc" , ["punctuation", "("], ["punctuation", ")"]
["keyword", "def"],
["function", "myfunc"],
["punctuation", "("],
["punctuation", ")"],
["operator", "=>"],
["number", "12"],

"\r\nmyfunc",
["punctuation", "("],
["punctuation", ")"]
]

----------------------------------------------------

Checks for functions.
Checks for functions.
23 changes: 23 additions & 0 deletions tests/languages/concurnas/langext_feature.test
@@ -0,0 +1,23 @@
myAPL || x[⍋x←6?40] ||
SimpleLisp||(+ 1 2 (* 3 3 ) )||

|| invalid ||

----------------------------------------------------

[
["langext", [
["class-name", "myAPL"],
["punctuation", "||"],
["string", " x[⍋x←6?40] "],
["punctuation", "||"]
]],
["langext", [
["class-name", "SimpleLisp"],
["punctuation", "||"],
["string", "(+ 1 2 (* 3 3 ) )"],
["punctuation", "||"]
]],

"\r\n\r\n || invalid ||"
]
17 changes: 15 additions & 2 deletions tests/languages/concurnas/operator_feature.test
Expand Up @@ -10,42 +10,55 @@ mod mod=
< <== > >==
and or
band bor bxor
^
^ ~

----------------------------------------------------

[
["operator", "+"],
["operator", "++"],
["operator", "+="],

["operator", "-"],
["operator", "--"],
["operator", "-="],

["operator", "="],
["operator", "=="],
["operator", "<>"],

["operator", "&=="],
["operator", "&<>"],

["operator", "isnot"],

["operator", "is"],
["operator", "as"],

["operator", "comp"],

["operator", "/"],
["operator", "/="],
["operator", "*"],
["operator", "*="],

["operator", "mod"],
["operator", "mod="],

["operator", "<"],
["operator", "<=="],
["operator", ">"],
["operator", ">=="],

["operator", "and"],
["operator", "or"],

["operator", "band"],
["operator", "bor"],
["operator", "bxor"],
["operator", "^"]

["operator", "^"],
["operator", "~"]
]

----------------------------------------------------
Expand Down
13 changes: 13 additions & 0 deletions tests/languages/concurnas/regex_feature.test
@@ -0,0 +1,13 @@
r'say'
r"hello"

----------------------------------------------------

[
["regex-literal", [
["regex", "r'say'"]
]],
["regex-literal", [
["regex", "r\"hello\""]
]]
]
22 changes: 5 additions & 17 deletions tests/languages/concurnas/string_feature.test
@@ -1,19 +1,15 @@
"hi"
"addition result: {1+2}"
'hi'
r'say'
r"hello"
'contains: "'
myAPL || x[⍋x←6?40] ||
|| invalid ||

----------------------------------------------------

[
["string", [
["string-literal", [
["string", "\"hi\""]
]],
["string", [
["string-literal", [
["string", "\"addition result: "],
["interpolation", [
["punctuation", "{"],
Expand All @@ -24,20 +20,12 @@ myAPL || x[⍋x←6?40] ||
]],
["string", "\""]
]],
["string", [
["string-literal", [
["string", "'hi'"]
]],
["string", [
["string", "r'say'"]
]],
["string", [
["string", "r\"hello\""]
]],
["string", [
["string-literal", [
["string", "'contains: \"'"]
]],
["langext", "myAPL || x[⍋x←6?40] ||"],
"\r\n || invalid ||"
]]
]

----------------------------------------------------
Expand Down

0 comments on commit 7b34e65

Please sign in to comment.