Skip to content

Commit dfbb202

Browse files
RunDevelopmentRunDevelopment
authoredDec 5, 2021
Jolie: Improved tokenization (#3221)
·
v1.30.0v1.26.0
1 parent 563cd73 commit dfbb202

9 files changed

+355
-210
lines changed
 

‎components/prism-jolie.js

+20-34
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,41 @@
11
Prism.languages.jolie = Prism.languages.extend('clike', {
22
'string': {
3-
pattern: /(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,
3+
pattern: /(^|[^\\])"(?:\\[\s\S]|[^"\\])*"/,
4+
lookbehind: true,
45
greedy: true
56
},
6-
'keyword': /\b(?:Aggregates|Interfaces|Java|Javascript|Jolie|Location|OneWay|Protocol|Redirects|RequestResponse|cH|comp|concurrent|constants|courier|cset|csets|default|define|else|embedded|execution|exit|extender|for|foreach|forward|global|if|in|include|init|inputPort|install|instanceof|interface|is_defined|linkIn|linkOut|main|new|nullProcess|outputPort|over|provide|scope|sequential|service|single|spawn|synchronized|this|throw|throws|type|undef|until|while|with)\b/,
7+
'class-name': {
8+
pattern: /((?:\b(?:as|courier|embed|in|inputPort|outputPort|service)\b|@)[ \t]*)\w+/,
9+
lookbehind: true
10+
},
11+
'keyword': /\b(?:as|cH|comp|concurrent|constants|courier|cset|csets|default|define|else|embed|embedded|execution|exit|extender|for|foreach|forward|from|global|if|import|in|include|init|inputPort|install|instanceof|interface|is_defined|linkIn|linkOut|main|new|nullProcess|outputPort|over|private|provide|public|scope|sequential|service|single|spawn|synchronized|this|throw|throws|type|undef|until|while|with)\b/,
12+
'function': /\b[a-z_]\w*(?=[ \t]*[@(])/i,
713
'number': /(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?l?/i,
8-
'operator': /-[-=>]?|\+[+=]?|<[<=]?|[>=*!]=?|&&|\|\||[:?\/%^]/,
9-
'punctuation': /[,.]/,
10-
'builtin': /\b(?:Byte|any|bool|char|double|float|int|long|string|undefined|void)\b/,
11-
'symbol': /[|;@]/
14+
'operator': /-[-=>]?|\+[+=]?|<[<=]?|[>=*!]=?|&&|\|\||[?\/%^@|]/,
15+
'punctuation': /[()[\]{},;.:]/,
16+
'builtin': /\b(?:Byte|any|bool|char|double|enum|float|int|length|long|ranges|regex|string|undefined|void)\b/
1217
});
1318

14-
delete Prism.languages.jolie['class-name'];
15-
1619
Prism.languages.insertBefore('jolie', 'keyword', {
17-
'function':
18-
{
19-
pattern: /((?:\b(?:courier|in|inputPort|outputPort|service)\b|@)\s*)\w+/,
20-
lookbehind: true
21-
},
2220
'aggregates': {
2321
pattern: /(\bAggregates\s*:\s*)(?:\w+(?:\s+with\s+\w+)?\s*,\s*)*\w+(?:\s+with\s+\w+)?/,
2422
lookbehind: true,
2523
inside: {
26-
'with-extension': {
27-
pattern: /\bwith\s+\w+/,
28-
inside: {
29-
'keyword': /\bwith\b/
30-
}
31-
},
32-
'function': {
33-
pattern: /\w+/
34-
},
35-
'punctuation': {
36-
pattern: /,/
37-
}
24+
'keyword': /\bwith\b/,
25+
'class-name': /\w+/,
26+
'punctuation': /,/
3827
}
3928
},
4029
'redirects': {
4130
pattern: /(\bRedirects\s*:\s*)(?:\w+\s*=>\s*\w+\s*,\s*)*(?:\w+\s*=>\s*\w+)/,
4231
lookbehind: true,
4332
inside: {
44-
'punctuation': {
45-
pattern: /,/
46-
},
47-
'function': {
48-
pattern: /\w+/
49-
},
50-
'symbol': {
51-
pattern: /=>/
52-
}
33+
'punctuation': /,/,
34+
'class-name': /\w+/,
35+
'operator': /=>/
5336
}
37+
},
38+
'property': {
39+
pattern: /\b(?:Aggregates|[Ii]nterfaces|Java|Javascript|Jolie|[Ll]ocation|OneWay|[Pp]rotocol|Redirects|RequestResponse)\b(?=[ \t]*:)/
5440
}
5541
});

‎components/prism-jolie.min.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/languages/jolie/builtin_feature.test

+22-14
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,35 @@
1-
undefined
2-
string
3-
int
4-
void
5-
long
61
Byte
2+
any
73
bool
4+
char
85
double
6+
enum
97
float
10-
char
11-
any
8+
int
9+
length
10+
long
11+
ranges
12+
regex
13+
string
14+
undefined
15+
void
1216

1317
----------------------------------------------------
1418

1519
[
16-
["builtin", "undefined"],
17-
["builtin", "string"],
18-
["builtin", "int"],
19-
["builtin", "void"],
20-
["builtin", "long"],
2120
["builtin", "Byte"],
21+
["builtin", "any"],
2222
["builtin", "bool"],
23+
["builtin", "char"],
2324
["builtin", "double"],
25+
["builtin", "enum"],
2426
["builtin", "float"],
25-
["builtin", "char"],
26-
["builtin", "any"]
27+
["builtin", "int"],
28+
["builtin", "length"],
29+
["builtin", "long"],
30+
["builtin", "ranges"],
31+
["builtin", "regex"],
32+
["builtin", "string"],
33+
["builtin", "undefined"],
34+
["builtin", "void"]
2735
]

‎tests/languages/jolie/comment_feature.test

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// single line
2+
3+
/*
4+
multiple
5+
lines
6+
*/
7+
8+
----------------------------------------------------
9+
10+
[
11+
["comment", "// single line"],
12+
13+
["comment", "/*\r\nmultiple\r\nlines\r\n*/"]
14+
]

‎tests/languages/jolie/deployment_features.test

+126-30
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,152 @@
11
Aggregates: First, Second with Third
2+
23
Redirects: First => Second, Third => Fourth
4+
35
Jolie: "logger.ol" in LoggerService
6+
47
log@LoggerService( new )();
58
println @ Console( "none" )()
69

10+
outputPort OutputPort3 {
11+
location: "socket://localhost:9002/"
12+
protocol: sodep
13+
interfaces: Interface3
14+
}
15+
16+
interface MyInterface {
17+
OneWay:
18+
myOW( string )
19+
RequestResponse:
20+
myRR( string )( string )
21+
}
22+
23+
private service MainService {
24+
embed ConfigurationService( ) as Conf
25+
main {
26+
getDBConn@Conf( )( res )
27+
}
28+
}
29+
730
----------------------------------------------------
831

932
[
10-
["keyword", "Aggregates"],
11-
["operator", ":"],
33+
["property", "Aggregates"],
34+
["punctuation", ":"],
1235
["aggregates", [
13-
["function", "First"],
36+
["class-name", "First"],
1437
["punctuation", ","],
15-
["function", "Second"],
16-
["with-extension", [
17-
["keyword", "with"],
18-
" Third"
19-
]]
38+
["class-name", "Second"],
39+
["keyword", "with"],
40+
["class-name", "Third"]
2041
]],
2142

22-
["keyword", "Redirects"],
23-
["operator", ":"],
43+
["property", "Redirects"],
44+
["punctuation", ":"],
2445
["redirects", [
25-
["function", "First"],
26-
["symbol", "=>"],
27-
["function", "Second"],
46+
["class-name", "First"],
47+
["operator", "=>"],
48+
["class-name", "Second"],
2849
["punctuation", ","],
29-
["function", "Third"],
30-
["symbol", "=>"],
31-
["function", "Fourth"]
50+
["class-name", "Third"],
51+
["operator", "=>"],
52+
["class-name", "Fourth"]
3253
]],
3354

34-
["keyword", "Jolie"],
35-
["operator", ":"],
55+
["property", "Jolie"],
56+
["punctuation", ":"],
3657
["string", "\"logger.ol\""],
3758
["keyword", "in"],
38-
["function", "LoggerService"],
59+
["class-name", "LoggerService"],
3960

40-
"\r\nlog",
41-
["symbol", "@"],
42-
["function", "LoggerService"],
43-
"( ",
61+
["function", "log"],
62+
["operator", "@"],
63+
["class-name", "LoggerService"],
64+
["punctuation", "("],
4465
["keyword", "new"],
45-
" )()",
46-
["symbol", ";"],
66+
["punctuation", ")"],
67+
["punctuation", "("],
68+
["punctuation", ")"],
69+
["punctuation", ";"],
4770

48-
"\r\nprintln ",
49-
["symbol", "@"],
50-
["function", "Console"],
51-
"( ",
71+
["function", "println"],
72+
["operator", "@"],
73+
["class-name", "Console"],
74+
["punctuation", "("],
5275
["string", "\"none\""],
53-
" )()"
76+
["punctuation", ")"],
77+
["punctuation", "("],
78+
["punctuation", ")"],
79+
80+
["keyword", "outputPort"],
81+
["class-name", "OutputPort3"],
82+
["punctuation", "{"],
83+
84+
["property", "location"],
85+
["punctuation", ":"],
86+
["string", "\"socket://localhost:9002/\""],
87+
88+
["property", "protocol"],
89+
["punctuation", ":"],
90+
" sodep\r\n ",
91+
92+
["property", "interfaces"],
93+
["punctuation", ":"],
94+
" Interface3\r\n",
95+
96+
["punctuation", "}"],
97+
98+
["keyword", "interface"],
99+
" MyInterface ",
100+
["punctuation", "{"],
101+
102+
["property", "OneWay"],
103+
["punctuation", ":"],
104+
105+
["function", "myOW"],
106+
["punctuation", "("],
107+
["builtin", "string"],
108+
["punctuation", ")"],
109+
110+
["property", "RequestResponse"],
111+
["punctuation", ":"],
112+
113+
["function", "myRR"],
114+
["punctuation", "("],
115+
["builtin", "string"],
116+
["punctuation", ")"],
117+
["punctuation", "("],
118+
["builtin", "string"],
119+
["punctuation", ")"],
120+
121+
["punctuation", "}"],
122+
123+
["keyword", "private"],
124+
["keyword", "service"],
125+
["class-name", "MainService"],
126+
["punctuation", "{"],
127+
128+
["keyword", "embed"],
129+
["class-name", "ConfigurationService"],
130+
["punctuation", "("],
131+
["punctuation", ")"],
132+
["keyword", "as"],
133+
["class-name", "Conf"],
134+
135+
["keyword", "main"],
136+
["punctuation", "{"],
137+
138+
["function", "getDBConn"],
139+
["operator", "@"],
140+
["class-name", "Conf"],
141+
["punctuation", "("],
142+
["punctuation", ")"],
143+
["punctuation", "("],
144+
" res ",
145+
["punctuation", ")"],
146+
147+
["punctuation", "}"],
148+
149+
["punctuation", "}"]
54150
]
55151

56152
----------------------------------------------------

‎tests/languages/jolie/keyword_feature.test

+111-124
Original file line numberDiff line numberDiff line change
@@ -1,130 +1,117 @@
1-
include
2-
define
3-
is_defined
4-
undef
5-
main
6-
init
7-
outputPort ;
8-
inputPort ;
9-
Location
10-
Protocol
11-
Interfaces
12-
RequestResponse
13-
OneWay
14-
type
15-
interface
16-
extender
17-
throws
18-
cset
19-
csets
20-
forward
21-
courier ;
22-
Aggregates
23-
Redirects
24-
embedded
25-
extender
26-
execution
27-
sequential
28-
concurrent
29-
single
30-
scope
31-
install
32-
throw
33-
comp
34-
cH
35-
default
36-
global
37-
linkIn
38-
linkOut
39-
synchronized
40-
this
41-
new
42-
for
43-
if
44-
else
45-
while
46-
in ;
47-
Jolie
48-
Java
49-
Javascript
50-
nullProcess
51-
spawn
52-
constants
53-
with
54-
provide
55-
until
56-
exit
57-
foreach
58-
instanceof
59-
over
60-
service
1+
as;
2+
cH;
3+
comp;
4+
concurrent;
5+
constants;
6+
courier;
7+
cset;
8+
csets;
9+
default;
10+
define;
11+
else;
12+
embed;
13+
embedded;
14+
execution;
15+
exit;
16+
extender;
17+
for;
18+
foreach;
19+
forward;
20+
from;
21+
global;
22+
if;
23+
import;
24+
in;
25+
include;
26+
init;
27+
inputPort;
28+
install;
29+
instanceof;
30+
interface;
31+
is_defined;
32+
linkIn;
33+
linkOut;
34+
main;
35+
new;
36+
nullProcess;
37+
outputPort;
38+
over;
39+
private;
40+
provide;
41+
public;
42+
scope;
43+
sequential;
44+
service;
45+
single;
46+
spawn;
47+
synchronized;
48+
this;
49+
throw;
50+
throws;
51+
type;
52+
undef;
53+
until;
54+
while;
55+
with;
6156

6257
----------------------------------------------------
58+
6359
[
64-
["keyword", "include"],
65-
["keyword", "define"],
66-
["keyword", "is_defined"],
67-
["keyword", "undef"],
68-
["keyword", "main"],
69-
["keyword", "init"],
70-
["keyword", "outputPort"],
71-
["symbol", ";"],
72-
["keyword", "inputPort"],
73-
["symbol", ";"],
74-
["keyword", "Location"],
75-
["keyword", "Protocol"],
76-
["keyword", "Interfaces"],
77-
["keyword", "RequestResponse"],
78-
["keyword", "OneWay"],
79-
["keyword", "type"],
80-
["keyword", "interface"],
81-
["keyword", "extender"],
82-
["keyword", "throws"],
83-
["keyword", "cset"],
84-
["keyword", "csets"],
85-
["keyword", "forward"],
86-
["keyword", "courier"],
87-
["symbol", ";"],
88-
["keyword", "Aggregates"],
89-
["keyword", "Redirects"],
90-
["keyword", "embedded"],
91-
["keyword", "extender"],
92-
["keyword", "execution"],
93-
["keyword", "sequential"],
94-
["keyword", "concurrent"],
95-
["keyword", "single"],
96-
["keyword", "scope"],
97-
["keyword", "install"],
98-
["keyword", "throw"],
99-
["keyword", "comp"],
100-
["keyword", "cH"],
101-
["keyword", "default"],
102-
["keyword", "global"],
103-
["keyword", "linkIn"],
104-
["keyword", "linkOut"],
105-
["keyword", "synchronized"],
106-
["keyword", "this"],
107-
["keyword", "new"],
108-
["keyword", "for"],
109-
["keyword", "if"],
110-
["keyword", "else"],
111-
["keyword", "while"],
112-
["keyword", "in"],
113-
["symbol", ";"],
114-
["keyword", "Jolie"],
115-
["keyword", "Java"],
116-
["keyword", "Javascript"],
117-
["keyword", "nullProcess"],
118-
["keyword", "spawn"],
119-
["keyword", "constants"],
120-
["keyword", "with"],
121-
["keyword", "provide"],
122-
["keyword", "until"],
123-
["keyword", "exit"],
124-
["keyword", "foreach"],
125-
["keyword", "instanceof"],
126-
["keyword", "over"],
127-
["keyword", "service"]
60+
["keyword", "as"], ["punctuation", ";"],
61+
["keyword", "cH"], ["punctuation", ";"],
62+
["keyword", "comp"], ["punctuation", ";"],
63+
["keyword", "concurrent"], ["punctuation", ";"],
64+
["keyword", "constants"], ["punctuation", ";"],
65+
["keyword", "courier"], ["punctuation", ";"],
66+
["keyword", "cset"], ["punctuation", ";"],
67+
["keyword", "csets"], ["punctuation", ";"],
68+
["keyword", "default"], ["punctuation", ";"],
69+
["keyword", "define"], ["punctuation", ";"],
70+
["keyword", "else"], ["punctuation", ";"],
71+
["keyword", "embed"], ["punctuation", ";"],
72+
["keyword", "embedded"], ["punctuation", ";"],
73+
["keyword", "execution"], ["punctuation", ";"],
74+
["keyword", "exit"], ["punctuation", ";"],
75+
["keyword", "extender"], ["punctuation", ";"],
76+
["keyword", "for"], ["punctuation", ";"],
77+
["keyword", "foreach"], ["punctuation", ";"],
78+
["keyword", "forward"], ["punctuation", ";"],
79+
["keyword", "from"], ["punctuation", ";"],
80+
["keyword", "global"], ["punctuation", ";"],
81+
["keyword", "if"], ["punctuation", ";"],
82+
["keyword", "import"], ["punctuation", ";"],
83+
["keyword", "in"], ["punctuation", ";"],
84+
["keyword", "include"], ["punctuation", ";"],
85+
["keyword", "init"], ["punctuation", ";"],
86+
["keyword", "inputPort"], ["punctuation", ";"],
87+
["keyword", "install"], ["punctuation", ";"],
88+
["keyword", "instanceof"], ["punctuation", ";"],
89+
["keyword", "interface"], ["punctuation", ";"],
90+
["keyword", "is_defined"], ["punctuation", ";"],
91+
["keyword", "linkIn"], ["punctuation", ";"],
92+
["keyword", "linkOut"], ["punctuation", ";"],
93+
["keyword", "main"], ["punctuation", ";"],
94+
["keyword", "new"], ["punctuation", ";"],
95+
["keyword", "nullProcess"], ["punctuation", ";"],
96+
["keyword", "outputPort"], ["punctuation", ";"],
97+
["keyword", "over"], ["punctuation", ";"],
98+
["keyword", "private"], ["punctuation", ";"],
99+
["keyword", "provide"], ["punctuation", ";"],
100+
["keyword", "public"], ["punctuation", ";"],
101+
["keyword", "scope"], ["punctuation", ";"],
102+
["keyword", "sequential"], ["punctuation", ";"],
103+
["keyword", "service"], ["punctuation", ";"],
104+
["keyword", "single"], ["punctuation", ";"],
105+
["keyword", "spawn"], ["punctuation", ";"],
106+
["keyword", "synchronized"], ["punctuation", ";"],
107+
["keyword", "this"], ["punctuation", ";"],
108+
["keyword", "throw"], ["punctuation", ";"],
109+
["keyword", "throws"], ["punctuation", ";"],
110+
["keyword", "type"], ["punctuation", ";"],
111+
["keyword", "undef"], ["punctuation", ";"],
112+
["keyword", "until"], ["punctuation", ";"],
113+
["keyword", "while"], ["punctuation", ";"],
114+
["keyword", "with"], ["punctuation", ";"]
128115
]
129116

130117
----------------------------------------------------

‎tests/languages/jolie/operator_feature.test

+9-5
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,40 @@
44
< <= > >= -> <<
55
= ==
66
&&
7-
? * / % ; : | @
7+
? * / % | @
88

99
----------------------------------------------------
1010

1111
[
1212
["operator", "+"],
1313
["operator", "++"],
1414
["operator", "+="],
15+
1516
["operator", "-"],
1617
["operator", "--"],
1718
["operator", "-="],
19+
1820
["operator", "!"],
1921
["operator", "!="],
22+
2023
["operator", "<"],
2124
["operator", "<="],
2225
["operator", ">"],
2326
["operator", ">="],
2427
["operator", "->"],
2528
["operator", "<<"],
29+
2630
["operator", "="],
2731
["operator", "=="],
32+
2833
["operator", "&&"],
34+
2935
["operator", "?"],
3036
["operator", "*"],
3137
["operator", "/"],
3238
["operator", "%"],
33-
["symbol", ";"],
34-
["operator", ":"],
35-
["symbol", "|"],
36-
["symbol", "@"]
39+
["operator", "|"],
40+
["operator", "@"]
3741
]
3842

3943
----------------------------------------------------

‎tests/languages/jolie/punctuation_feature.test

+12-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,18 @@
1-
, .
1+
( ) [ ] { }
2+
, ; . :
23

34
----------------------------------------------------
45

56
[
7+
["punctuation", "("],
8+
["punctuation", ")"],
9+
["punctuation", "["],
10+
["punctuation", "]"],
11+
["punctuation", "{"],
12+
["punctuation", "}"],
13+
614
["punctuation", ","],
7-
["punctuation", "."]
15+
["punctuation", ";"],
16+
["punctuation", "."],
17+
["punctuation", ":"]
818
]

‎tests/languages/jolie/string_feature.test

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
""
2+
"10"
3+
"
4+
JOLIE preserves formatting.
5+
This line will be indented.
6+
This line too.
7+
"
8+
9+
jsonValue = "{
10+
\"int\": 123,
11+
\"bool\": true,
12+
\"long\": 124,
13+
\"double\": 123.4,
14+
\"string\": \"string\",
15+
\"void\": {},
16+
\"array\": [123, true,\"ciccio\",124,{}],
17+
\"obj\" : {
18+
\"int\": 1243,
19+
\"bool\": true,
20+
\"long\": 1234,
21+
\"double\": 1234.4,
22+
\"string\": \"string\",
23+
\"void\": {}
24+
}
25+
}"
26+
;
27+
28+
----------------------------------------------------
29+
30+
[
31+
["string", "\"\""],
32+
["string", "\"10\""],
33+
["string", "\"\r\nJOLIE preserves formatting.\r\n This line will be indented.\r\n This line too.\r\n\""],
34+
35+
"\r\n\r\njsonValue ",
36+
["operator", "="],
37+
["string", "\"{\r\n \\\"int\\\": 123,\r\n \\\"bool\\\": true,\r\n \\\"long\\\": 124,\r\n \\\"double\\\": 123.4,\r\n \\\"string\\\": \\\"string\\\",\r\n \\\"void\\\": {},\r\n \\\"array\\\": [123, true,\\\"ciccio\\\",124,{}],\r\n \\\"obj\\\" : {\r\n \\\"int\\\": 1243,\r\n \\\"bool\\\": true,\r\n \\\"long\\\": 1234,\r\n \\\"double\\\": 1234.4,\r\n \\\"string\\\": \\\"string\\\",\r\n \\\"void\\\": {}\r\n }\r\n }\""],
38+
39+
["punctuation", ";"]
40+
]

0 commit comments

Comments
 (0)
Please sign in to comment.