Update to use ES Modules (#64)

* take a crack at a cache * write cache key to cache file * remove identifiers from cache key * Normalize paths across runs * cleanup * Record failures in cache * modulo more whitespace * Fix reporting issue with replacements, add --nocache flag * Improve error reporting for replacements * normalize union order for type comparisons * just normalize the type part of the quickinfo * todo * Make it ESM * Fix for __dirname * update jest config, utils test snapshot * tests passing w/ ESM * add verifier:done-with-file directive * CJS for eslintrc * unit test for done-with-file * header reset with done-with-file
danvk · Aug 4, 2023 · 8ba532d · 8ba532d
1 parent 1774a2f
commit 8ba532d
Show file tree

Hide file tree

Showing 22 changed files with 356 additions and 71 deletions.
diff --git a/.eslintrc.js → .eslintrc.cjs b/.eslintrc.js → .eslintrc.cjs
@@ -12,5 +12,7 @@ module.exports =  {
     '@typescript-eslint/explicit-function-return-type': 'off',
     '@typescript-eslint/no-non-null-assertion': 'off',
     '@typescript-eslint/no-use-before-define': 'off',
+    // Let tsc handle these
+    '@typescript-eslint/no-unused-vars': 'off',
   }
 };
diff --git a/README.md b/README.md
@@ -221,6 +221,10 @@ See above for how to give directive to literate-ts in your source format.
     `node_modules` directory. Particularly useful with `@types`, e.g.
     `verifier:include-node-module:@types/lodash`.
   </dd>
+  <dt>verifier:done-with-file</dt>
+  <dd>
+    Stop verifying any code samples until the end of this file. Useful if you have some unstructured comments at the end of a file that happen to contain code samples.
+  </dd>
 </dl>
 
 ### Replacements
@@ -265,6 +269,7 @@ be careful not to mislead the reader when you do this.
 - `-f`/`--filter`: Only check IDs with the given prefix.
 - `-r`/`--replacements`: If specified, load `**/*.{ts,js,txt}` under this directory as additional sources.
 - `--alsologtostderr`: Log to stderr in addition to a log file.
+- `--nocache`: Disable reading and writing from on-disk cache. If this results in different behavior, please file an issue. The cache is in `node_modules/.cache/literate-ts`. Delete this directory to clear the cache.
 
 ## Development
 

diff --git a/index.ts b/index.ts
@@ -1,3 +1,3 @@
-import {main} from './src/main';
+import {main} from './src/main.js';
 
 main();
diff --git a/jest.config.js b/jest.config.js
@@ -1,9 +1,28 @@
-module.exports = {
+export default {
+  preset: "ts-jest/presets/default-esm",
+  testEnvironment: "node",
   transform: {
     '^.+\\.tsx?$': 'ts-jest',
   },
-  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  extensionsToTreatAsEsm: ['.ts'],
+  moduleNameMapper: {
+    '^(\\.{1,2}/.*)\\.js$': '$1',
+  },
+  transform: {
+    // '^.+\\.[tj]sx?$' to process js/ts with `ts-jest`
+    // '^.+\\.m?[tj]sx?$' to process js/ts/mjs/mts with `ts-jest`
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        useESM: true,
+      },
+    ],
+  },
+  testPathIgnorePatterns: [
+    "dist",
+  ],
+  // moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
   globals: {
-    __TEST__: true
+    __TEST__: true,
   }
-};
+}
diff --git a/package.json b/package.json
@@ -2,7 +2,8 @@
   "name": "literate-ts",
   "version": "1.3.0",
   "description": "Code samples that scale",
-  "main": "dist/index.js",
+  "exports": "./dist/index.js",
+  "type": "module",
   "repository": "https://github.com/danvk/literate-ts.git",
   "author": "Dan Vanderkam <danvdk@gmail.com>",
   "license": "MIT",
@@ -11,30 +12,33 @@
     "format": "prettier --write 'src/**/*.ts'",
     "format:check": "prettier --check 'src/**/*.ts'",
     "lint": "eslint 'src/**/*.ts'",
-    "test": "jest"
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest"
   },
   "bin": {
     "literate-ts": "./bin/literate-ts"
   },
   "dependencies": {
-    "chalk": "^2.4.2",
+    "chalk": "^5.3.0",
     "fast-json-stable-stringify": "^2.1.0",
+    "find-cache-dir": "^4.0.0",
     "fs-extra": "^8.1.0",
     "glob": "^7.1.4",
     "lodash": "^4.17.15",
     "ora": "^3.4.0",
+    "read-pkg-up": "^10.0.0",
     "tmp": "^0.1.0",
     "yargs": "^14.0.0"
   },
   "peerDependencies": {
     "typescript": ">4.1"
   },
   "devDependencies": {
+    "@types/find-cache-dir": "^3.2.1",
     "@types/fs-extra": "^8.0.0",
     "@types/glob": "^7.1.1",
     "@types/jest": "^24.0.18",
     "@types/lodash": "^4.14.138",
-    "@types/node": "^12.7.4",
+    "@types/node": "16",
     "@types/ora": "^3.2.0",
     "@types/tmp": "^0.1.0",
     "@types/yargs": "^13.0.2",

diff --git a/src/asciidoc.ts b/src/asciidoc.ts
@@ -1,6 +1,6 @@
-import {matchAndExtract} from './utils';
-import {Processor} from './code-sample';
-import {generateIdMetadata} from './metadata';
+import {matchAndExtract} from './utils.js';
+import {Processor} from './code-sample.js';
+import {generateIdMetadata} from './metadata.js';
 
 const EXTRACT_BACKTICKS = /```(tsx|ts)/;
 const EXTRACT_ID = /\[\[([^\]]*)\]\]/;

diff --git a/src/code-sample.ts b/src/code-sample.ts
@@ -1,12 +1,10 @@
 import _ from 'lodash';
 
-import {CodeSample, PrefixedCodeSample, Prefix, IdMetadata} from './types';
-import {log} from './logger';
-import {fail} from './test-tracker';
-import {extractAsciidocSamples} from './asciidoc';
-import {extractMarkdownSamples} from './markdown';
-import {generateIdMetadata} from './metadata';
-import {dedent} from './utils';
+import {CodeSample, PrefixedCodeSample, Prefix, IdMetadata} from './types.js';
+import {fail} from './test-tracker.js';
+import {extractAsciidocSamples} from './asciidoc.js';
+import {extractMarkdownSamples} from './markdown.js';
+import {generateIdMetadata} from './metadata.js';
 
 export interface Processor {
   setLineNum(line: number): void;
@@ -32,6 +30,7 @@ function process(
   let lastLanguage: string | null = null;
   let prefixes: readonly Prefix[] = [];
   let skipNext = false;
+  let skipRemaining = false;
   let prependNext = false;
   let prependLines: number[] | null = null;
   let nodeModules: readonly string[] = [];
@@ -53,6 +52,7 @@ function process(
         prefixes = [];
         prependNext = false;
         skipNext = false;
+        skipRemaining = false;
         tsOptions = {};
         nodeModules = [];
         nextIsTSX = false;
@@ -76,6 +76,8 @@ function process(
         ]);
       } else if (directive.startsWith('skip')) {
         skipNext = true;
+      } else if (directive.startsWith('done-with-file')) {
+        skipRemaining = true;
       } else if (directive.startsWith('tsconfig:')) {
         const [key, value] = directive.split(':', 2)[1].split('=', 2);
         tsOptions[key] = value === 'true' ? true : value === 'false' ? false : value;
@@ -107,7 +109,7 @@ function process(
         lastMetadata = generateIdMetadata(slug + '-' + lineNum, sourceFile, lineNum);
       }
       if (lastMetadata) {
-        if (!skipNext) {
+        if (!skipNext && !skipRemaining) {
           samples.push({
             ...lastMetadata,
             sectionHeader: lastSectionHeader,

diff --git a/src/logger.ts b/src/logger.ts
@@ -1,6 +1,6 @@
 import fs from 'fs';
 
-import {getTempDir} from './utils';
+import {getTempDir} from './utils.js';
 
 let logHandle: number | null = null;
 let _alsoStderr = false;

diff --git a/src/main.ts b/src/main.ts
@@ -8,21 +8,21 @@ import ora from 'ora';
 import ts from 'typescript';
 import yargs from 'yargs';
 
-import {checkSource, applyPrefixes, extractSamples} from './code-sample';
-import {startLog, log, flushLog, logFile} from './logger';
-import {runNode} from './node-runner';
+import {checkSource, applyPrefixes, extractSamples} from './code-sample.js';
+import {startLog, log, flushLog, logFile} from './logger.js';
+import {runNode} from './node-runner.js';
 import {
   getTestResults,
   startFile,
   fail,
   finishFile,
   finishSample,
   startSample,
-} from './test-tracker';
-import {checkTs, ConfigBundle} from './ts-checker';
-import {CodeSample} from './types';
-import {writeTempFile, fileSlug} from './utils';
-import {VERSION} from './version';
+} from './test-tracker.js';
+import {CACHE_DIR, checkTs, ConfigBundle} from './ts-checker.js';
+import {CodeSample} from './types.js';
+import {writeTempFile, fileSlug} from './utils.js';
+import {VERSION} from './version.js';
 
 const argv = yargs
   .strict()
@@ -74,6 +74,11 @@ const unParsedConfig = ts.readConfigFile('tsconfig.json', ts.sys.readFile).confi
 const {options: tsOptions} = ts.parseJsonConfigFileContent(unParsedConfig, ts.sys, process.cwd());
 
 console.log('Verifying with TypeScript', ts.version);
+if (!argv.nocache) {
+  console.log('Cache dir:', CACHE_DIR);
+} else {
+  console.log(chalk.yellow('Skipping cache (--nocache specified)'));
+}
 const spinner = argv.alsologtostderr ? null : ora('Initializing').start();
 
 const typeScriptBundle: ConfigBundle = {

diff --git a/src/markdown.ts b/src/markdown.ts
@@ -1,6 +1,6 @@
-import {matchAndExtract} from './utils';
-import {Processor} from './code-sample';
-import {generateIdMetadata} from './metadata';
+import {matchAndExtract} from './utils.js';
+import {Processor} from './code-sample.js';
+import {generateIdMetadata} from './metadata.js';
 
 const EXTRACT_ID = /<!-- #([^ ]+) -->/;
 const EXTRACT_DIRECTIVE = /<!-- verifier:(.*) -->/;

diff --git a/src/metadata.ts b/src/metadata.ts
@@ -1,4 +1,4 @@
-import {IdMetadata} from './types';
+import {IdMetadata} from './types.js';
 
 export function generateIdMetadata(id: string, sourceFile: string, line: number): IdMetadata {
   return {

diff --git a/src/node-runner.ts b/src/node-runner.ts
@@ -1,7 +1,7 @@
 import {exec} from 'child_process';
 import util from 'util';
 
-import {log} from './logger';
+import {log} from './logger.js';
 
 export interface ExecErrorType {
   code: number;

diff --git a/src/test-tracker.ts b/src/test-tracker.ts
@@ -1,5 +1,5 @@
-import {log, isLoggingToStderr} from './logger';
-import {CodeSample} from './types';
+import {log, isLoggingToStderr} from './logger.js';
+import {CodeSample} from './types.js';
 
 let currentFile: string;
 let currentSample: CodeSample | undefined;

diff --git a/src/test/asciidoc.test.ts b/src/test/asciidoc.test.ts
@@ -2,9 +2,9 @@ import fs from 'fs';
 import glob from 'glob';
 import path from 'path';
 
-import {dedent} from '../utils';
-import {extractSamples} from '../code-sample';
-import {baseExtract} from './common';
+import {dedent} from '../utils.js';
+import {extractSamples} from '../code-sample.js';
+import {baseExtract} from './common.js';
 
 const ASCII_DOC1 = `
 
@@ -226,6 +226,43 @@ describe('extractSamples', () => {
     ]);
   });
 
+  test('done-with-file', () => {
+    expect(
+      extractSamples(
+        dedent`
+    // verifier:done-with-file
+    [source,ts]
+    ----
+    console.log(a);
+    ----
+
+    and:
+    [source,ts]
+    ----
+    const x = 12;
+    ----
+
+    // verifier:reset
+    [[back-in-business]]
+    [source,ts]
+    ----
+    const backInBusiness = 23;
+    ----
+    `,
+        'done-with-file',
+        'source.asciidoc',
+      ).slice(-1),
+    ).toEqual([
+      {
+        ...baseExtract,
+        language: 'ts',
+        descriptor: './source.asciidoc:13',
+        id: 'back-in-business',
+        content: `const backInBusiness = 23;`,
+      },
+    ]);
+  });
+
   test('header resets', () => {
     expect(
       extractSamples(
@@ -308,5 +345,36 @@ describe('extractSamples', () => {
         ],
       },
     ]);
+
+    expect(
+      extractSamples(
+        dedent`
+      == Chapter 1
+      // verifier:done-with-file
+      [source,ts]
+      ----
+      const x = 12;
+      ----
+
+      == Chapter 2
+      [source,ts]
+      ----
+      const x = 12;
+      ----
+    `,
+        'header-reset-done-with-file',
+        'source.asciidoc',
+      ),
+    ).toEqual([
+      {
+        ...baseExtract,
+        descriptor: './source.asciidoc:10',
+        id: 'header-reset-done-with-file-10',
+        sectionHeader: 'Chapter 2',
+        language: 'ts',
+        content: `const x = 12;`,
+        prefixes: [],
+      },
+    ]);
   });
 });
diff --git a/src/test/code-sample.test.ts b/src/test/code-sample.test.ts
@@ -1,6 +1,6 @@
-import {stripSource, applyPrefixes, extractSamples} from '../code-sample';
-import {dedent} from '../utils';
-import {baseSample} from './common';
+import {stripSource, applyPrefixes, extractSamples} from '../code-sample.js';
+import {dedent} from '../utils.js';
+import {baseSample} from './common.js';
 
 const ASCIIDOC_PREPEND = `
 

diff --git a/src/test/markdown.test.ts b/src/test/markdown.test.ts
@@ -2,7 +2,7 @@ import fs from 'fs';
 import glob from 'glob';
 import path from 'path';
 
-import {extractSamples} from '../code-sample';
+import {extractSamples} from '../code-sample.js';
 
 describe('markdown', () => {
   it('should match snapshots', () => {

diff --git a/src/test/ts-checker.test.ts b/src/test/ts-checker.test.ts
@@ -10,8 +10,8 @@ import {
   getLanguageServiceHost,
   matchModuloWhitespace,
   sortUnions,
-} from '../ts-checker';
-import {dedent} from '../utils';
+} from '../ts-checker.js';
+import {dedent} from '../utils.js';
 
 describe('ts-checker', () => {
   describe('extractExpectedErrors', () => {

diff --git a/src/test/utils.test.ts b/src/test/utils.test.ts
@@ -1,4 +1,4 @@
-import {matchAndExtract, sha256} from '../utils';
+import {matchAndExtract, sha256} from '../utils.js';
 
 describe('utils', () => {
   test('matchAndExtract', () => {