-
-
Notifications
You must be signed in to change notification settings - Fork 59
/
tokenize.js
56 lines (51 loc) · 1.21 KB
/
tokenize.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
module.exports = function (sch) {
var noComments = function (line) {
var i = line.indexOf('//')
return i > -1 ? line.slice(0, i) : line
}
var noMultilineComments = function () {
var inside = false
return function (token) {
if (token === '/*') {
inside = true
return false
}
if (token === '*/') {
inside = false
return false
}
return !inside
}
}
var trim = function (line) {
return line.trim()
}
var removeQuotedLines = function (list) {
return function (str) {
var s = '$' + list.length + '$'
list.push(str)
return s
}
}
var restoreQuotedLines = function (list) {
var re = /^\$(\d+)\$$/
return function (line) {
var m = line.match(re)
return m ? list[+m[1]] : line
}
}
var replacements = []
return sch
.replace(/"(\\"|[^"\n])*?"|'(\\'|[^'\n])*?'/gm, removeQuotedLines(replacements))
.replace(/([;,{}()=:[\]<>]|\/\*|\*\/)/g, ' $1 ')
.split(/\n/)
.map(trim)
.filter(Boolean)
.map(noComments)
.map(trim)
.filter(Boolean)
.join('\n')
.split(/\s+|\n+/gm)
.filter(noMultilineComments())
.map(restoreQuotedLines(replacements))
}