feat: introduce createIsLiteralPositionAcorn function (#2)

README.md

@@ -42,6 +42,13 @@ Try to use `stripLiteralAcorn` first, and fallback to `stripLiteralRegex` if Aco
 
 [Source](./src/index.ts)
 
+### `createIsLiteralPositionAcorn`
+Returns a function that returns whether the position is in a literal using [Acorn](https://github.com/acornjs/acorn)'s tokenizer.
+
+Will throw error if the input is not valid JavaScript.
+
+[Source](./src/acorn.ts)
+
 ## Sponsors
 
 <p align="center">

bench/index.bench.ts

@@ -1,6 +1,6 @@
 import { readFile } from 'fs/promises'
 import { bench, describe } from 'vitest'
-import { stripLiteralAcorn, stripLiteralRegex } from '../src'
+import { createIsLiteralPositionAcorn, stripLiteralAcorn, stripLiteralRegex } from '../src'
 
 const modules = {
   'vue-global': './node_modules/vue/dist/vue.runtime.global.js',
@@ -10,11 +10,14 @@ const modules = {
 Object.entries(modules).forEach(([name, path]) => {
   describe(`bench ${name}`, async () => {
     const code = await readFile(path, 'utf-8')
-    bench('regex', () => {
+    bench('stripLiteral (regex)', () => {
       stripLiteralRegex(code)
     })
-    bench('acorn', () => {
+    bench('stripLiteral (acorn)', () => {
       stripLiteralAcorn(code)
     })
+    bench('createIsLiteralPositionAcorn (acorn)', () => {
+      createIsLiteralPositionAcorn(code)
+    })
   })
 })

src/acorn.ts

@@ -40,3 +40,59 @@ export function stripLiteralAcorn(code: string) {
 
   return result
 }
+
+/**
+ * Returns a function that returns whether the position is
+ * in a literal using Acorn's tokenizer.
+ *
+ * Will throw error if the input is not valid JavaScript.
+ */
+export function createIsLiteralPositionAcorn(code: string) {
+  // literal start position, non-literal start position, literal start position, ...
+  const positionList: number[] = []
+
+  const tokens = tokenizer(code, {
+    ecmaVersion: 'latest',
+    sourceType: 'module',
+    allowHashBang: true,
+    allowAwaitOutsideFunction: true,
+    allowImportExportEverywhere: true,
+    onComment(_isBlock, _text, start, end) {
+      positionList.push(start)
+      positionList.push(end)
+    },
+  })
+  const inter = tokens[Symbol.iterator]()
+
+  while (true) {
+    const { done, value: token } = inter.next()
+    if (done)
+      break
+    if (token.type.label === 'string') {
+      positionList.push(token.start + 1)
+      positionList.push(token.end - 1)
+    }
+    else if (token.type.label === 'template') {
+      positionList.push(token.start)
+      positionList.push(token.end)
+    }
+  }
+
+  return (position: number) => {
+    const i = binarySearch(positionList, v => position < v)
+    return (i - 1) % 2 === 0
+  }
+}
+
+function binarySearch(array: ArrayLike<number>, pred: (v: number) => boolean) {
+  let low = -1
+  let high = array.length
+  while (1 + low < high) {
+    const mid = low + ((high - low) >> 1)
+    if (pred(array[mid]))
+      high = mid
+    else
+      low = mid
+  }
+  return high
+}

src/index.ts

@@ -1,7 +1,7 @@
 import { stripLiteralAcorn } from './acorn'
 import { stripLiteralRegex } from './regex'
 
-export { stripLiteralAcorn } from './acorn'
+export { stripLiteralAcorn, createIsLiteralPositionAcorn } from './acorn'
 export { stripLiteralRegex } from './regex'
 
 /**
@@ -17,4 +17,3 @@ export function stripLiteral(code: string) {
     return stripLiteralRegex(code)
   }
 }
-

test/__snapshots__/createIsLiteralPosition.test.ts.snap

@@ -0,0 +1,50 @@
+// Vitest Snapshot v1
+
+exports[`template string nested 1`] = `"\`**\${a + \`*\`}**\`"`;
+
+exports[`works 1`] = `
+"
+const a = 0
+  "
+`;
+
+exports[`works 2`] = `
+"
+****
+const a = 0
+  "
+`;
+
+exports[`works 3`] = `
+"
+*******
+const a = 0
+  "
+`;
+
+exports[`works 4`] = `
+"
+**+
+***+
+**
+const a = 0
+  "
+`;
+
+exports[`works 5`] = `
+"
+const a = '*'
+  "
+`;
+
+exports[`works 6`] = `
+"
+const a = \\"*\\"
+  "
+`;
+
+exports[`works 7`] = `
+"
+const a = \`*\${b}\`
+  "
+`;

test/createIsLiteralPosition.test.ts

@@ -0,0 +1,55 @@
+/* eslint-disable no-template-curly-in-string */
+import { expect, test } from 'vitest'
+import { createIsLiteralPositionAcorn } from '../src'
+
+function execute(code: string) {
+  const isLiteralPosition = createIsLiteralPositionAcorn(code)
+
+  const positions = new Array(code.length)
+    .fill(0)
+    .map((_, i) => i)
+  const result = positions
+    .map((pos) => {
+      if (code[pos] === '\n')
+        return isLiteralPosition(pos) ? '+\n' : '\n'
+      return isLiteralPosition(pos) ? '*' : code[pos]
+    })
+    .join('')
+
+  return result
+}
+
+test('works', () => {
+  expect(execute(`
+const a = 0
+  `)).toMatchSnapshot()
+  expect(execute(`
+// a
+const a = 0
+  `)).toMatchSnapshot()
+  expect(execute(`
+/* a */
+const a = 0
+  `)).toMatchSnapshot()
+  expect(execute(`
+/*
+  a
+*/
+const a = 0
+  `)).toMatchSnapshot()
+  expect(execute(`
+const a = 'a'
+  `)).toMatchSnapshot()
+  expect(execute(`
+const a = "a"
+  `)).toMatchSnapshot()
+  expect(execute(`
+const a = \`c\${b}\`
+  `)).toMatchSnapshot()
+})
+
+test('template string nested', () => {
+  expect(execute(
+    '`aa${a + `a`}aa`',
+  )).toMatchSnapshot()
+})