From d4af0b52b0b22f376d2a36605868794d96320e65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BF=A0=20/=20green?= Date: Tue, 13 Sep 2022 05:55:00 +0900 Subject: [PATCH] feat: introduce `createIsLiteralPositionAcorn` function (#2) --- README.md | 7 +++ bench/index.bench.ts | 9 ++- src/acorn.ts | 56 +++++++++++++++++++ src/index.ts | 3 +- .../createIsLiteralPosition.test.ts.snap | 50 +++++++++++++++++ test/createIsLiteralPosition.test.ts | 55 ++++++++++++++++++ 6 files changed, 175 insertions(+), 5 deletions(-) create mode 100644 test/__snapshots__/createIsLiteralPosition.test.ts.snap create mode 100644 test/createIsLiteralPosition.test.ts diff --git a/README.md b/README.md index 2b252f9..b2b2208 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,13 @@ Try to use `stripLiteralAcorn` first, and fallback to `stripLiteralRegex` if Aco [Source](./src/index.ts) +### `createIsLiteralPositionAcorn` +Returns a function that returns whether the position is in a literal using [Acorn](https://github.com/acornjs/acorn)'s tokenizer. + +Will throw error if the input is not valid JavaScript. + +[Source](./src/acorn.ts) + ## Sponsors

diff --git a/bench/index.bench.ts b/bench/index.bench.ts index b8e281c..18013eb 100644 --- a/bench/index.bench.ts +++ b/bench/index.bench.ts @@ -1,6 +1,6 @@ import { readFile } from 'fs/promises' import { bench, describe } from 'vitest' -import { stripLiteralAcorn, stripLiteralRegex } from '../src' +import { createIsLiteralPositionAcorn, stripLiteralAcorn, stripLiteralRegex } from '../src' const modules = { 'vue-global': './node_modules/vue/dist/vue.runtime.global.js', @@ -10,11 +10,14 @@ const modules = { Object.entries(modules).forEach(([name, path]) => { describe(`bench ${name}`, async () => { const code = await readFile(path, 'utf-8') - bench('regex', () => { + bench('stripLiteral (regex)', () => { stripLiteralRegex(code) }) - bench('acorn', () => { + bench('stripLiteral (acorn)', () => { stripLiteralAcorn(code) }) + bench('createIsLiteralPositionAcorn (acorn)', () => { + createIsLiteralPositionAcorn(code) + }) }) }) diff --git a/src/acorn.ts b/src/acorn.ts index f247732..1bc4489 100644 --- a/src/acorn.ts +++ b/src/acorn.ts @@ -40,3 +40,59 @@ export function stripLiteralAcorn(code: string) { return result } + +/** + * Returns a function that returns whether the position is + * in a literal using Acorn's tokenizer. + * + * Will throw error if the input is not valid JavaScript. + */ +export function createIsLiteralPositionAcorn(code: string) { + // literal start position, non-literal start position, literal start position, ... + const positionList: number[] = [] + + const tokens = tokenizer(code, { + ecmaVersion: 'latest', + sourceType: 'module', + allowHashBang: true, + allowAwaitOutsideFunction: true, + allowImportExportEverywhere: true, + onComment(_isBlock, _text, start, end) { + positionList.push(start) + positionList.push(end) + }, + }) + const inter = tokens[Symbol.iterator]() + + while (true) { + const { done, value: token } = inter.next() + if (done) + break + if (token.type.label === 'string') { + positionList.push(token.start + 1) + positionList.push(token.end - 1) + } + else if (token.type.label === 'template') { + positionList.push(token.start) + positionList.push(token.end) + } + } + + return (position: number) => { + const i = binarySearch(positionList, v => position < v) + return (i - 1) % 2 === 0 + } +} + +function binarySearch(array: ArrayLike, pred: (v: number) => boolean) { + let low = -1 + let high = array.length + while (1 + low < high) { + const mid = low + ((high - low) >> 1) + if (pred(array[mid])) + high = mid + else + low = mid + } + return high +} diff --git a/src/index.ts b/src/index.ts index 87659be..cbf7171 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ import { stripLiteralAcorn } from './acorn' import { stripLiteralRegex } from './regex' -export { stripLiteralAcorn } from './acorn' +export { stripLiteralAcorn, createIsLiteralPositionAcorn } from './acorn' export { stripLiteralRegex } from './regex' /** @@ -17,4 +17,3 @@ export function stripLiteral(code: string) { return stripLiteralRegex(code) } } - diff --git a/test/__snapshots__/createIsLiteralPosition.test.ts.snap b/test/__snapshots__/createIsLiteralPosition.test.ts.snap new file mode 100644 index 0000000..45d3521 --- /dev/null +++ b/test/__snapshots__/createIsLiteralPosition.test.ts.snap @@ -0,0 +1,50 @@ +// Vitest Snapshot v1 + +exports[`template string nested 1`] = `"\`**\${a + \`*\`}**\`"`; + +exports[`works 1`] = ` +" +const a = 0 + " +`; + +exports[`works 2`] = ` +" +**** +const a = 0 + " +`; + +exports[`works 3`] = ` +" +******* +const a = 0 + " +`; + +exports[`works 4`] = ` +" +**+ +***+ +** +const a = 0 + " +`; + +exports[`works 5`] = ` +" +const a = '*' + " +`; + +exports[`works 6`] = ` +" +const a = \\"*\\" + " +`; + +exports[`works 7`] = ` +" +const a = \`*\${b}\` + " +`; diff --git a/test/createIsLiteralPosition.test.ts b/test/createIsLiteralPosition.test.ts new file mode 100644 index 0000000..ecbfbcf --- /dev/null +++ b/test/createIsLiteralPosition.test.ts @@ -0,0 +1,55 @@ +/* eslint-disable no-template-curly-in-string */ +import { expect, test } from 'vitest' +import { createIsLiteralPositionAcorn } from '../src' + +function execute(code: string) { + const isLiteralPosition = createIsLiteralPositionAcorn(code) + + const positions = new Array(code.length) + .fill(0) + .map((_, i) => i) + const result = positions + .map((pos) => { + if (code[pos] === '\n') + return isLiteralPosition(pos) ? '+\n' : '\n' + return isLiteralPosition(pos) ? '*' : code[pos] + }) + .join('') + + return result +} + +test('works', () => { + expect(execute(` +const a = 0 + `)).toMatchSnapshot() + expect(execute(` +// a +const a = 0 + `)).toMatchSnapshot() + expect(execute(` +/* a */ +const a = 0 + `)).toMatchSnapshot() + expect(execute(` +/* + a +*/ +const a = 0 + `)).toMatchSnapshot() + expect(execute(` +const a = 'a' + `)).toMatchSnapshot() + expect(execute(` +const a = "a" + `)).toMatchSnapshot() + expect(execute(` +const a = \`c\${b}\` + `)).toMatchSnapshot() +}) + +test('template string nested', () => { + expect(execute( + '`aa${a + `a`}aa`', + )).toMatchSnapshot() +})