Skip to content

Commit 55526bb

Browse files
antfuantfu
committedAug 30, 2024
fix(engine): regex flags propagation
·
v3.2.1v1.15.1
1 parent 61a6bf2 commit 55526bb

File tree

11 files changed

+2706
-83
lines changed

11 files changed

+2706
-83
lines changed
 

‎packages/core/src/engines/javascript.ts

+4-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,10 @@ export class JavaScriptScanner implements PatternScanner {
2727
p
2828
// YAML specific handling; TODO: move to tm-grammars
2929
.replaceAll('[^\\s[-?:,\\[\\]{}#&*!|>\'"%@`]]', '[^\\s\\-?:,\\[\\]{}#&*!|>\'"%@`]'),
30-
{ flags: 'dgm' },
30+
{
31+
flags: 'dgm',
32+
ignoreContiguousAnchors: true,
33+
},
3134
)
3235
cache?.set(p, regex)
3336
return regex

‎packages/shiki/test/engine-js/__records__/jsonc.json

+241
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
[
2+
{
3+
"constractor": [
4+
[
5+
"\\b(?:true|false|null)\\b",
6+
"-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?",
7+
"\"",
8+
"\\[",
9+
"\\{",
10+
"/\\*\\*(?!/)",
11+
"/\\*",
12+
"(//).*$\\n?"
13+
]
14+
],
15+
"executions": [
16+
{
17+
"args": [
18+
"// comment\n",
19+
0
20+
],
21+
"result": {
22+
"index": 7,
23+
"captureIndices": [
24+
{
25+
"start": 0,
26+
"end": 11,
27+
"length": 11
28+
},
29+
{
30+
"start": 0,
31+
"end": 2,
32+
"length": 2
33+
}
34+
]
35+
}
36+
},
37+
{
38+
"args": [
39+
"// comment\n",
40+
11
41+
],
42+
"result": null
43+
},
44+
{
45+
"args": [
46+
"{\"foo\":\"bar\"}\n",
47+
0
48+
],
49+
"result": {
50+
"index": 4,
51+
"captureIndices": [
52+
{
53+
"start": 0,
54+
"end": 1,
55+
"length": 1
56+
}
57+
]
58+
}
59+
},
60+
{
61+
"args": [
62+
"{\"foo\":\"bar\"}\n",
63+
13
64+
],
65+
"result": null
66+
}
67+
]
68+
},
69+
{
70+
"constractor": [
71+
[
72+
"\\}",
73+
"\"",
74+
"/\\*\\*(?!/)",
75+
"/\\*",
76+
"(//).*$\\n?",
77+
":",
78+
"[^\\s\\}]"
79+
]
80+
],
81+
"executions": [
82+
{
83+
"args": [
84+
"{\"foo\":\"bar\"}\n",
85+
1
86+
],
87+
"result": {
88+
"index": 1,
89+
"captureIndices": [
90+
{
91+
"start": 1,
92+
"end": 2,
93+
"length": 1
94+
}
95+
]
96+
}
97+
},
98+
{
99+
"args": [
100+
"{\"foo\":\"bar\"}\n",
101+
6
102+
],
103+
"result": {
104+
"index": 5,
105+
"captureIndices": [
106+
{
107+
"start": 6,
108+
"end": 7,
109+
"length": 1
110+
}
111+
]
112+
}
113+
},
114+
{
115+
"args": [
116+
"{\"foo\":\"bar\"}\n",
117+
12
118+
],
119+
"result": {
120+
"index": 0,
121+
"captureIndices": [
122+
{
123+
"start": 12,
124+
"end": 13,
125+
"length": 1
126+
}
127+
]
128+
}
129+
}
130+
]
131+
},
132+
{
133+
"constractor": [
134+
[
135+
"\"",
136+
"\\\\(?:[\"\\\\/bfnrt]|u[0-9a-fA-F]{4})",
137+
"\\\\."
138+
]
139+
],
140+
"executions": [
141+
{
142+
"args": [
143+
"{\"foo\":\"bar\"}\n",
144+
2
145+
],
146+
"result": {
147+
"index": 0,
148+
"captureIndices": [
149+
{
150+
"start": 5,
151+
"end": 6,
152+
"length": 1
153+
}
154+
]
155+
}
156+
}
157+
]
158+
},
159+
{
160+
"constractor": [
161+
[
162+
"(,)|(?=\\})",
163+
"\\b(?:true|false|null)\\b",
164+
"-?(?:0|[1-9]\\d*)(?:(?:\\.\\d+)?(?:[eE][+-]?\\d+)?)?",
165+
"\"",
166+
"\\[",
167+
"\\{",
168+
"/\\*\\*(?!/)",
169+
"/\\*",
170+
"(//).*$\\n?",
171+
"[^\\s,]"
172+
]
173+
],
174+
"executions": [
175+
{
176+
"args": [
177+
"{\"foo\":\"bar\"}\n",
178+
7
179+
],
180+
"result": {
181+
"index": 3,
182+
"captureIndices": [
183+
{
184+
"start": 7,
185+
"end": 8,
186+
"length": 1
187+
}
188+
]
189+
}
190+
},
191+
{
192+
"args": [
193+
"{\"foo\":\"bar\"}\n",
194+
12
195+
],
196+
"result": {
197+
"index": 0,
198+
"captureIndices": [
199+
{
200+
"start": 12,
201+
"end": 12,
202+
"length": 0
203+
},
204+
{
205+
"start": 4294967295,
206+
"end": 4294967295,
207+
"length": 0
208+
}
209+
]
210+
}
211+
}
212+
]
213+
},
214+
{
215+
"constractor": [
216+
[
217+
"\"",
218+
"\\\\(?:[\"\\\\/bfnrt]|u[0-9a-fA-F]{4})",
219+
"\\\\."
220+
]
221+
],
222+
"executions": [
223+
{
224+
"args": [
225+
"{\"foo\":\"bar\"}\n",
226+
8
227+
],
228+
"result": {
229+
"index": 0,
230+
"captureIndices": [
231+
{
232+
"start": 11,
233+
"end": 12,
234+
"length": 1
235+
}
236+
]
237+
}
238+
}
239+
]
240+
}
241+
]

‎packages/shiki/test/engine-js/__records__/sql.json

+575
Large diffs are not rendered by default.

‎packages/shiki/test/engine-js/__records__/toml.json

+414
Large diffs are not rendered by default.

‎packages/shiki/test/engine-js/__records__/vue.json

+1,339
Large diffs are not rendered by default.

‎packages/shiki/test/engine-js/compare.test.ts

+47-7
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,44 @@ const cases: Cases[] = [
7777
'// comment\n{"foo":"bar"}',
7878
],
7979
},
80+
{
81+
name: 'vue',
82+
theme: () => import('../../src/assets/themes/vitesse-dark'),
83+
lang: () => import('../../src/assets/langs/vue'),
84+
cases: [
85+
`<script setup>\nimport { ref } from 'vue'\n</script>`,
86+
`<template>\n<div>{{ foo }}</div>\n</template>`,
87+
],
88+
},
89+
{
90+
name: 'toml',
91+
theme: () => import('../../src/assets/themes/nord'),
92+
lang: () => import('../../src/assets/langs/toml'),
93+
cases: [
94+
[
95+
`# This is a TOML document`,
96+
'',
97+
`title = "TOML Example"`,
98+
'',
99+
'[owner]',
100+
'name = "Tom Preston-Werner"',
101+
].join('\n'),
102+
],
103+
},
104+
{
105+
name: 'sql',
106+
theme: () => import('../../src/assets/themes/nord'),
107+
lang: () => import('../../src/assets/langs/sql'),
108+
cases: [
109+
'SELECT * FROM foo',
110+
[
111+
'USE AdventureWorks2022;',
112+
'GO',
113+
'IF OBJECT_ID(\'dbo.NewProducts\', \'U\') IS NOT NULL',
114+
'DROP TABLE dbo.NewProducts;',
115+
].join('\n'),
116+
],
117+
},
80118
]
81119

82120
describe('cases', async () => {
@@ -92,18 +130,18 @@ describe('cases', async () => {
92130

93131
for (const c of resolved) {
94132
it(c.c.name, async () => {
95-
const wasm = createWasmOnigLibWrapper()
96-
const native = createJavaScriptRegexEngine()
133+
const engineWasm = createWasmOnigLibWrapper()
134+
const engineJs = createJavaScriptRegexEngine()
97135

98136
const shiki1 = await createHighlighterCore({
99137
langs: c.lang,
100138
themes: [c.theme],
101-
engine: wasm,
139+
engine: engineWasm,
102140
})
103141
const shiki2 = await createHighlighterCore({
104142
langs: c.lang,
105143
themes: [c.theme],
106-
engine: native,
144+
engine: engineJs,
107145
})
108146

109147
const lang = c.lang[0].name
@@ -118,12 +156,14 @@ describe('cases', async () => {
118156
])
119157
}
120158

121-
await expect(JSON.stringify(wasm.instances, null, 2))
159+
await expect.soft(JSON.stringify(engineWasm.instances, null, 2))
122160
.toMatchFileSnapshot(`./__records__/${c.c.name}.json`)
123161

124-
for (const [a, b] of compare) {
162+
compare.forEach(([a, b]) => {
125163
expect.soft(a).toEqual(b)
126-
}
164+
// await expect.soft(a)
165+
// .toMatchFileSnapshot(`./__records__/tokens/${c.c.name}-${i}.json`)
166+
})
127167
})
128168
}
129169
})

‎packages/shiki/test/engine-js/general.test.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,6 @@ describe('should', () => {
4242
`)
4343

4444
expect(shiki.codeToHtml('@media foo { .bar { padding-left: 1px; }}', { lang: 'css', theme: 'min-dark' }))
45-
.toMatchInlineSnapshot(`"<pre class="shiki min-dark" style="background-color:#1f1f1f;color:#b392f0" tabindex="0"><code><span class="line"><span style="color:#B392F0">@media foo { .bar { </span><span style="color:#79B8FF">padding-left</span><span style="color:#F97583">:</span><span style="color:#F8F8F8"> 1px</span><span style="color:#B392F0">; }}</span></span></code></pre>"`)
45+
.toMatchInlineSnapshot(`"<pre class="shiki min-dark" style="background-color:#1f1f1f;color:#b392f0" tabindex="0"><code><span class="line"><span style="color:#F97583">@media</span><span style="color:#B392F0"> foo { .bar { </span><span style="color:#79B8FF">padding-left</span><span style="color:#F97583">:</span><span style="color:#F8F8F8"> 1px</span><span style="color:#B392F0">; }}</span></span></code></pre>"`)
4646
})
4747
})

‎packages/shiki/test/engine-js/verify.test.ts

+17-6
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,27 @@ for (const file of files) {
2424
const instances = JSON.parse(await fs.readFile(file, 'utf-8')) as Instance[]
2525
let i = 0
2626
for (const instance of instances) {
27-
describe(`instances ${i++}`, () => {
27+
i += 1
28+
describe(`instances ${i}`, () => {
2829
const scanner = new JavaScriptScanner(instance.constractor[0], cache, false)
2930
let j = 0
3031
for (const execution of instance.executions) {
31-
it(`case ${j++}`, () => {
32+
j += 1
33+
it(`case ${j}`, () => {
3234
onTestFailed(() => {
33-
console.error({
34-
patterns: scanner.patterns,
35-
regexps: scanner.regexps,
36-
})
35+
console.error(execution.result?.index != null
36+
? {
37+
args: execution.args,
38+
expected: {
39+
pattern: scanner.patterns[execution.result.index],
40+
regexp: scanner.regexps[execution.result.index],
41+
},
42+
}
43+
: {
44+
args: execution.args,
45+
patterns: scanner.patterns,
46+
regexps: scanner.regexps,
47+
})
3748
})
3849
const result = scanner.findNextMatchSync(...execution.args)
3950
expect(result).toEqual(execution.result)

‎pnpm-lock.yaml

+6-6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pnpm-workspace.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ catalog:
5151
minimist: ^1.2.8
5252
monaco-editor-core: ^0.51.0
5353
ofetch: ^1.3.4
54-
oniguruma-to-js: ^0.2.3
54+
oniguruma-to-js: ^0.2.4
5555
picocolors: ^1.0.1
5656
pinia: ^2.2.2
5757
pnpm: ^9.9.0

‎scripts/report-engine-js-compat.md

+61-61
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)
Please sign in to comment.