diff --git a/packages/babel-cli/src/babel/dir.ts b/packages/babel-cli/src/babel/dir.ts index 2ab0652afbb3..5bbbec2886e5 100644 --- a/packages/babel-cli/src/babel/dir.ts +++ b/packages/babel-cli/src/babel/dir.ts @@ -3,6 +3,7 @@ import path from "path"; import fs from "fs"; import * as util from "./util"; +import * as watcher from "./watcher"; import type { CmdOptions } from "./options"; const FILE_TYPE = Object.freeze({ @@ -21,8 +22,6 @@ export default async function ({ cliOptions, babelOptions, }: CmdOptions): Promise { - const filenames = cliOptions.filenames; - async function write( src: string, base: string, @@ -66,7 +65,7 @@ export default async function ({ util.chmod(src, dest); if (cliOptions.verbose) { - console.log(src + " -> " + dest); + console.log(path.relative(process.cwd(), src) + " -> " + dest); } return FILE_TYPE.COMPILED; @@ -150,6 +149,8 @@ export default async function ({ startTime = null; }, 100); + if (cliOptions.watch) watcher.enable({ enableGlobbing: true }); + if (!cliOptions.skipInitialBuild) { if (cliOptions.deleteDirOnStart) { util.deleteDir(cliOptions.outDir); @@ -173,43 +174,36 @@ export default async function ({ } if (cliOptions.watch) { - const chokidar = util.requireChokidar(); - - filenames.forEach(function (filenameOrDir: string): void { - const watcher = chokidar.watch(filenameOrDir, { - persistent: true, - ignoreInitial: true, - awaitWriteFinish: { - stabilityThreshold: 50, - pollInterval: 10, - }, - }); - - // This, alongside with debounce, allows us to only log - // when we are sure that all the files have been compiled. - let processing = 0; - - ["add", "change"].forEach(function (type: string): void { - watcher.on(type, async function (filename: string) { - processing++; - if (startTime === null) startTime = process.hrtime(); - - try { - await handleFile( - filename, - filename === filenameOrDir - ? path.dirname(filenameOrDir) - : filenameOrDir, - ); - - compiledFiles++; - } catch (err) { - console.error(err); - } - - processing--; - if (processing === 0 && !cliOptions.quiet) logSuccess(); - }); + // This, alongside with debounce, allows us to only log + // when we are sure that all the files have been compiled. + let processing = 0; + + cliOptions.filenames.forEach(filenameOrDir => { + watcher.watch(filenameOrDir); + + watcher.onFilesChange(async filenames => { + processing++; + if (startTime === null) startTime = process.hrtime(); + + try { + const written = await Promise.all( + filenames.map(filename => + handleFile( + filename, + filename === filenameOrDir + ? path.dirname(filenameOrDir) + : filenameOrDir, + ), + ), + ); + + compiledFiles += written.filter(Boolean).length; + } catch (err) { + console.error(err); + } + + processing--; + if (processing === 0 && !cliOptions.quiet) logSuccess(); }); }); } diff --git a/packages/babel-cli/src/babel/file.ts b/packages/babel-cli/src/babel/file.ts index e05f664979ad..24f6a77f7951 100644 --- a/packages/babel-cli/src/babel/file.ts +++ b/packages/babel-cli/src/babel/file.ts @@ -6,6 +6,7 @@ import fs from "fs"; import * as util from "./util"; import type { CmdOptions } from "./options"; +import * as watcher from "./watcher"; type CompilationOutput = { code: string; @@ -123,7 +124,7 @@ export default async function ({ async function stdin(): Promise { const code = await readStdin(); - const res = await util.transform(cliOptions.filename, code, { + const res = await util.transformRepl(cliOptions.filename, code, { ...babelOptions, sourceFileName: "stdin", }); @@ -193,40 +194,33 @@ export default async function ({ } async function files(filenames: Array): Promise { + if (cliOptions.watch) { + watcher.enable({ enableGlobbing: false }); + } + if (!cliOptions.skipInitialBuild) { await walk(filenames); } if (cliOptions.watch) { - const chokidar = util.requireChokidar(); - chokidar - .watch(filenames, { - disableGlobbing: true, - persistent: true, - ignoreInitial: true, - awaitWriteFinish: { - stabilityThreshold: 50, - pollInterval: 10, - }, - }) - .on("all", function (type: string, filename: string): void { - if ( - !util.isCompilableExtension(filename, cliOptions.extensions) && - !filenames.includes(filename) - ) { - return; - } - - if (type === "add" || type === "change") { - if (cliOptions.verbose) { - console.log(type + " " + filename); - } + filenames.forEach(watcher.watch); + + watcher.onFilesChange((changes, event, cause) => { + const actionableChange = changes.some( + filename => + util.isCompilableExtension(filename, cliOptions.extensions) || + filenames.includes(filename), + ); + if (!actionableChange) return; + + if (cliOptions.verbose) { + console.log(`${event} ${cause}`); + } - walk(filenames).catch(err => { - console.error(err); - }); - } + walk(filenames).catch(err => { + console.error(err); }); + }); } } diff --git a/packages/babel-cli/src/babel/util.ts b/packages/babel-cli/src/babel/util.ts index fe0c255b6e40..de2657ebce80 100644 --- a/packages/babel-cli/src/babel/util.ts +++ b/packages/babel-cli/src/babel/util.ts @@ -2,7 +2,8 @@ import readdirRecursive from "fs-readdir-recursive"; import * as babel from "@babel/core"; import path from "path"; import fs from "fs"; -import { createRequire } from "module"; + +import * as watcher from "./watcher"; export function chmod(src: string, dest: string): void { try { @@ -60,7 +61,7 @@ const CALLER = { name: "@babel/cli", }; -export function transform( +export function transformRepl( filename: string, code: string, opts: any, @@ -79,18 +80,31 @@ export function transform( }); } -export function compile(filename: string, opts: any | Function): Promise { +export async function compile( + filename: string, + opts: any | Function, +): Promise { opts = { ...opts, caller: CALLER, }; - return new Promise((resolve, reject) => { + // TODO (Babel 8): Use `babel.transformFileAsync` + const result: any = await new Promise((resolve, reject) => { babel.transformFile(filename, opts, (err, result) => { if (err) reject(err); else resolve(result); }); }); + + if (result) { + if (!process.env.BABEL_8_BREAKING) { + if (!result.externalDependencies) return result; + } + watcher.updateExternalDependencies(filename, result.externalDependencies); + } + + return result; } export function deleteDir(path: string): void { @@ -114,24 +128,6 @@ process.on("uncaughtException", function (err) { process.exitCode = 1; }); -export function requireChokidar(): any { - const require = createRequire(import.meta.url); - - try { - return process.env.BABEL_8_BREAKING - ? require("chokidar") - : parseInt(process.versions.node) >= 8 - ? require("chokidar") - : require("@nicolo-ribaudo/chokidar-2"); - } catch (err) { - console.error( - "The optional dependency chokidar failed to install and is required for " + - "--watch. Chokidar is likely not supported on your platform.", - ); - throw err; - } -} - export function withExtension(filename: string, ext: string = ".js") { const newBasename = path.basename(filename, path.extname(filename)) + ext; return path.join(path.dirname(filename), newBasename); diff --git a/packages/babel-cli/src/babel/watcher.ts b/packages/babel-cli/src/babel/watcher.ts new file mode 100644 index 000000000000..0c1d3756984e --- /dev/null +++ b/packages/babel-cli/src/babel/watcher.ts @@ -0,0 +1,133 @@ +import { createRequire } from "module"; +import path from "path"; + +const fileToDeps = new Map>(); +const depToFiles = new Map>(); + +let isWatchMode = false; +let watcher; + +export function enable({ enableGlobbing }: { enableGlobbing: boolean }) { + isWatchMode = true; + + const { FSWatcher } = requireChokidar(); + + watcher = new FSWatcher({ + disableGlobbing: !enableGlobbing, + persistent: true, + ignoreInitial: true, + awaitWriteFinish: { + stabilityThreshold: 50, + pollInterval: 10, + }, + }); + + watcher.on("unlink", unwatchFile); +} + +export function watch(filename: string): void { + if (!isWatchMode) { + throw new Error( + "Internal Babel error: .watch called when not in watch mode.", + ); + } + + watcher.add(path.resolve(filename)); +} + +/** + * Call @param callback whenever a dependency (source file)/ + * external dependency (non-source file) changes. + * + * Handles mapping external dependencies to their corresponding + * dependencies. + */ +export function onFilesChange( + callback: (filenames: string[], event: string, cause: string) => void, +): void { + if (!isWatchMode) { + throw new Error( + "Internal Babel error: .onFilesChange called when not in watch mode.", + ); + } + + watcher.on("all", (event, filename) => { + if (event !== "change" && event !== "add") return; + + const absoluteFile = path.resolve(filename); + callback( + [absoluteFile, ...(depToFiles.get(absoluteFile) ?? [])], + event, + absoluteFile, + ); + }); +} + +export function updateExternalDependencies( + filename: string, + dependencies: Set, +) { + if (!isWatchMode) return; + + // Use absolute paths + const absFilename = path.resolve(filename); + const absDependencies = new Set( + Array.from(dependencies, dep => path.resolve(dep)), + ); + + if (fileToDeps.has(absFilename)) { + for (const dep of fileToDeps.get(absFilename)) { + if (!absDependencies.has(dep)) { + removeFileDependency(absFilename, dep); + } + } + } + for (const dep of absDependencies) { + if (!depToFiles.has(dep)) { + depToFiles.set(dep, new Set()); + + watcher.add(dep); + } + depToFiles.get(dep).add(absFilename); + } + + fileToDeps.set(absFilename, absDependencies); +} + +function removeFileDependency(filename: string, dep: string) { + depToFiles.get(dep).delete(filename); + + if (depToFiles.get(dep).size === 0) { + depToFiles.delete(dep); + + watcher.unwatch(dep); + } +} + +function unwatchFile(filename: string) { + if (!fileToDeps.has(filename)) return; + + for (const dep of fileToDeps.get(filename)) { + removeFileDependency(filename, dep); + } + fileToDeps.delete(filename); +} + +function requireChokidar(): any { + // @ts-expect-error - TS is not configured to support import.meta. + const require = createRequire(import.meta.url); + + try { + return process.env.BABEL_8_BREAKING + ? require("chokidar") + : parseInt(process.versions.node) >= 8 + ? require("chokidar") + : require("@nicolo-ribaudo/chokidar-2"); + } catch (err) { + console.error( + "The optional dependency chokidar failed to install and is required for " + + "--watch. Chokidar is likely not supported on your platform.", + ); + throw err; + } +} diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/executor.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/executor.js new file mode 100644 index 000000000000..ec654248d223 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/executor.js @@ -0,0 +1,47 @@ +const fs = require("fs"); +const assert = require("assert"); + +// For Node.js <= 10 +if (!assert.match) assert.match = (val, re) => assert(re.test(val)); + +const run = (function* () { + let files = [yield, yield].sort(); + assert.match(files[0], /src[\\/]index.js -> lib[\\/]index.js/); + assert.match(files[1], /src[\\/]main.js -> lib[\\/]main.js/); + assert.match(yield, /Successfully compiled 2 files with Babel \(\d+ms\)\./); + + logFile("lib/index.js"); + logFile("lib/main.js"); + + fs.writeFileSync("./file.txt", "Updated!"); + + files = [yield, yield].sort(); + assert.match(files[0], /src[\\/]index.js -> lib[\\/]index.js/); + assert.match(files[1], /src[\\/]main.js -> lib[\\/]main.js/); + assert.match(yield, /Successfully compiled 2 files with Babel \(\d+ms\)\./); + + logFile("lib/index.js"); + logFile("lib/main.js"); +})(); + +run.next(); + +process.stdin.on("data", function listener(chunk) { + const str = String(chunk).trim(); + if (!str) return; + + console.log(str); + + if (run.next(str).done) { + process.exit(0); + } +}); + +function logFile(file) { + console.log("EXECUTOR", file, JSON.stringify(fs.readFileSync(file, "utf8"))); +} + +setTimeout(() => { + console.error("EXECUTOR TIMEOUT"); + process.exit(1); +}, 5000); diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/babel.config.cjs b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/babel.config.cjs new file mode 100644 index 000000000000..82a7bc963ce6 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/babel.config.cjs @@ -0,0 +1,25 @@ +const fs = require("fs"); +const path = require("path"); + +function inlinePlugin(api, { filename }) { + const { types: t } = api; + + const contents = api.cache.using(() => fs.readFileSync(filename, "utf8")); + api.addExternalDependency(filename); + + return { + visitor: { + Identifier(path) { + if (path.node.name === "REPLACE_ME") { + path.replaceWith(t.stringLiteral(contents)); + } + }, + }, + }; +} + +module.exports = { + plugins: [ + [inlinePlugin, { filename: path.resolve(__dirname, "./file.txt") }], + ], +}; diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/file.txt b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/file.txt new file mode 100644 index 000000000000..50f9dc495c1e --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/file.txt @@ -0,0 +1 @@ +Hi :) diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/src/index.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/src/index.js new file mode 100644 index 000000000000..32fcf9431c12 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/src/index.js @@ -0,0 +1 @@ +let str = REPLACE_ME; diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/src/main.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/src/main.js new file mode 100644 index 000000000000..97ecfbae3a71 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/in-files/src/main.js @@ -0,0 +1 @@ +console.log(REPLACE_ME); diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/options.json b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/options.json new file mode 100644 index 000000000000..6fc02e90636e --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/options.json @@ -0,0 +1,6 @@ +{ + "args": ["src", "--out-dir", "lib", "--watch", "--verbose"], + "noBabelrc": true, + "noDefaultPlugins": true, + "minNodeVersion": 8 +} diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/out-files/lib/index.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/out-files/lib/index.js new file mode 100644 index 000000000000..1b260711254f --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/out-files/lib/index.js @@ -0,0 +1 @@ +let str = "Updated!"; diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/out-files/lib/main.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/out-files/lib/main.js new file mode 100644 index 000000000000..3ffbe342fad3 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/out-files/lib/main.js @@ -0,0 +1 @@ +console.log("Updated!"); diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/stdout.txt b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/stdout.txt new file mode 100644 index 000000000000..69de44066ee9 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch --verbose with external dependencies/stdout.txt @@ -0,0 +1,10 @@ +src/index.js -> lib/index.js +src/main.js -> lib/main.js +Successfully compiled 2 files with Babel (123ms). +EXECUTOR lib/index.js "let str = /"Hi :)/";" +EXECUTOR lib/main.js "console.log(/"Hi :)/");" +src/index.js -> lib/index.js +src/main.js -> lib/main.js +Successfully compiled 2 files with Babel (123ms). +EXECUTOR lib/index.js "let str = /"Updated!/";" +EXECUTOR lib/main.js "console.log(/"Updated!/");" diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/executor.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/executor.js new file mode 100644 index 000000000000..474cd214405c --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/executor.js @@ -0,0 +1,41 @@ +const fs = require("fs"); +const assert = require("assert"); + +// For Node.js <= 10 +if (!assert.match) assert.match = (val, re) => assert(re.test(val)); + +const run = function* () { + assert.match(yield, /Successfully compiled 2 files with Babel \(\d+ms\)\./); + + logFile("lib/index.js"); + logFile("lib/main.js"); + + fs.writeFileSync("./file.txt", "Updated!"); + + assert.match(yield, /Successfully compiled 2 files with Babel \(\d+ms\)\./); + + logFile("lib/index.js"); + logFile("lib/main.js"); +}(); + +run.next(); + +process.stdin.on("data", function listener(chunk) { + const str = String(chunk).trim(); + if (!str) return; + + console.log(str); + + if (run.next(str).done) { + process.exit(0); + } +}); + +function logFile(file) { + console.log("EXECUTOR", file, JSON.stringify(fs.readFileSync(file, "utf8"))); +} + +setTimeout(() => { + console.error("EXECUTOR TIMEOUT"); + process.exit(1); +}, 5000); diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/babel.config.cjs b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/babel.config.cjs new file mode 100644 index 000000000000..82a7bc963ce6 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/babel.config.cjs @@ -0,0 +1,25 @@ +const fs = require("fs"); +const path = require("path"); + +function inlinePlugin(api, { filename }) { + const { types: t } = api; + + const contents = api.cache.using(() => fs.readFileSync(filename, "utf8")); + api.addExternalDependency(filename); + + return { + visitor: { + Identifier(path) { + if (path.node.name === "REPLACE_ME") { + path.replaceWith(t.stringLiteral(contents)); + } + }, + }, + }; +} + +module.exports = { + plugins: [ + [inlinePlugin, { filename: path.resolve(__dirname, "./file.txt") }], + ], +}; diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/file.txt b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/file.txt new file mode 100644 index 000000000000..50f9dc495c1e --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/file.txt @@ -0,0 +1 @@ +Hi :) diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/src/index.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/src/index.js new file mode 100644 index 000000000000..32fcf9431c12 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/src/index.js @@ -0,0 +1 @@ +let str = REPLACE_ME; diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/src/main.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/src/main.js new file mode 100644 index 000000000000..97ecfbae3a71 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/in-files/src/main.js @@ -0,0 +1 @@ +console.log(REPLACE_ME); diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/options.json b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/options.json new file mode 100644 index 000000000000..290c3dcabb24 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/options.json @@ -0,0 +1,6 @@ +{ + "args": ["src", "--out-dir", "lib", "--watch"], + "noBabelrc": true, + "noDefaultPlugins": true, + "minNodeVersion": 8 +} diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/out-files/lib/index.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/out-files/lib/index.js new file mode 100644 index 000000000000..1b260711254f --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/out-files/lib/index.js @@ -0,0 +1 @@ +let str = "Updated!"; diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/out-files/lib/main.js b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/out-files/lib/main.js new file mode 100644 index 000000000000..3ffbe342fad3 --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/out-files/lib/main.js @@ -0,0 +1 @@ +console.log("Updated!"); diff --git a/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/stdout.txt b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/stdout.txt new file mode 100644 index 000000000000..eef2fd0d7c1c --- /dev/null +++ b/packages/babel-cli/test/fixtures/babel/dir --out-dir --watch with external dependencies/stdout.txt @@ -0,0 +1,6 @@ +Successfully compiled 2 files with Babel (123ms). +EXECUTOR lib/index.js "let str = /"Hi :)/";" +EXECUTOR lib/main.js "console.log(/"Hi :)/");" +Successfully compiled 2 files with Babel (123ms). +EXECUTOR lib/index.js "let str = /"Updated!/";" +EXECUTOR lib/main.js "console.log(/"Updated!/");" diff --git a/packages/babel-cli/test/index.js b/packages/babel-cli/test/index.js index 2de08e866d00..1941b84cf81d 100644 --- a/packages/babel-cli/test/index.js +++ b/packages/babel-cli/test/index.js @@ -44,11 +44,6 @@ const readDir = function (loc, filter) { }; const saveInFiles = function (files) { - // Place an empty .babelrc in each test so tests won't unexpectedly get to repo-level config. - if (!fs.existsSync(path.join(tmpLoc, ".babelrc"))) { - outputFileSync(path.join(tmpLoc, ".babelrc"), "{}"); - } - Object.keys(files).forEach(function (filename) { const content = files[filename]; outputFileSync(path.join(tmpLoc, filename), content); @@ -145,7 +140,7 @@ const buildTest = function (binName, testName, opts) { let args = [binLoc]; - if (binName !== "babel-external-helpers") { + if (binName !== "babel-external-helpers" && !opts.noDefaultPlugins) { args.push("--presets", presetLocs, "--plugins", pluginLocs); } @@ -157,14 +152,6 @@ const buildTest = function (binName, testName, opts) { let stderr = ""; let stdout = ""; - spawn.stderr.on("data", function (chunk) { - stderr += chunk; - }); - - spawn.stdout.on("data", function (chunk) { - stdout += chunk; - }); - spawn.on("close", function () { let err; @@ -186,6 +173,33 @@ const buildTest = function (binName, testName, opts) { spawn.stdin.write(opts.stdin); spawn.stdin.end(); } + + const captureOutput = proc => { + proc.stderr.on("data", function (chunk) { + stderr += chunk; + }); + + proc.stdout.on("data", function (chunk) { + stdout += chunk; + }); + }; + + if (opts.executor) { + const executor = child.spawn(process.execPath, [opts.executor], { + cwd: tmpLoc, + }); + + spawn.stdout.pipe(executor.stdin); + spawn.stderr.pipe(executor.stdin); + + executor.on("close", function () { + setTimeout(() => spawn.kill("SIGINT"), 250); + }); + + captureOutput(executor); + } else { + captureOutput(spawn); + } }; }; @@ -238,6 +252,11 @@ fs.readdirSync(fixtureLoc).forEach(function (binName) { opts = { args: [], ...taskOpts }; } + const executorLoc = path.join(testLoc, "executor.js"); + if (fs.existsSync(executorLoc)) { + opts.executor = executorLoc; + } + ["stdout", "stdin", "stderr"].forEach(function (key) { const loc = path.join(testLoc, key + ".txt"); opts[key + "Path"] = loc; @@ -256,14 +275,22 @@ fs.readdirSync(fixtureLoc).forEach(function (binName) { if (fs.existsSync(babelrcLoc)) { // copy .babelrc file to tmp directory opts.inFiles[".babelrc"] = helper.readFile(babelrcLoc); - opts.inFiles[".babelignore"] = helper.readFile(babelIgnoreLoc); + } else if (!opts.noBabelrc) { + opts.inFiles[".babelrc"] = "{}"; } if (fs.existsSync(babelIgnoreLoc)) { // copy .babelignore file to tmp directory opts.inFiles[".babelignore"] = helper.readFile(babelIgnoreLoc); } + + const skip = + opts.minNodeVersion && + parseInt(process.versions.node, 10) < opts.minNodeVersion; + // eslint-disable-next-line jest/valid-title - it(testName, buildTest(binName, testName, opts), 20000); + (skip + ? it.skip + : it)(testName, buildTest(binName, testName, opts), 20000); }); }); }); diff --git a/packages/babel-core/src/config/config-chain.ts b/packages/babel-core/src/config/config-chain.ts index ab93397c75ca..059d38034b52 100644 --- a/packages/babel-core/src/config/config-chain.ts +++ b/packages/babel-core/src/config/config-chain.ts @@ -12,6 +12,7 @@ import type { } from "./validation/options"; import pathPatternToRegex from "./pattern-to-regex"; import { ConfigPrinter, ChainFormatter } from "./printer"; +import type { ReadonlyDeepArray } from "./helpers/deep-array"; const debug = buildDebug("babel:config:config-chain"); @@ -46,6 +47,7 @@ export type PresetInstance = { options: ValidatedOptions; alias: string; dirname: string; + externalDependencies: ReadonlyDeepArray; }; export type ConfigContext = { diff --git a/packages/babel-core/src/config/full.ts b/packages/babel-core/src/config/full.ts index 64ecd6e75fe2..fb016f79734e 100644 --- a/packages/babel-core/src/config/full.ts +++ b/packages/babel-core/src/config/full.ts @@ -7,6 +7,8 @@ import * as context from "../index"; import Plugin from "./plugin"; import { getItemDescriptor } from "./item"; import { buildPresetChain } from "./config-chain"; +import { finalize as freezeDeepArray } from "./helpers/deep-array"; +import type { DeepArray, ReadonlyDeepArray } from "./helpers/deep-array"; import type { ConfigContext, ConfigChain, @@ -35,6 +37,7 @@ type LoadedDescriptor = { options: {}; dirname: string; alias: string; + externalDependencies: DeepArray; }; export type { InputOptions } from "./validation/options"; @@ -42,6 +45,7 @@ export type { InputOptions } from "./validation/options"; export type ResolvedConfig = { options: any; passes: PluginPasses; + externalDependencies: ReadonlyDeepArray; }; export type { Plugin }; @@ -87,6 +91,8 @@ export default gensync<(inputOpts: unknown) => ResolvedConfig | null>( const pluginDescriptorsByPass: Array> = [[]]; const passes: Array> = []; + const externalDependencies: DeepArray = []; + const ignored = yield* enhanceError( context, function* recursePresetDescriptors( @@ -102,32 +108,31 @@ export default gensync<(inputOpts: unknown) => ResolvedConfig | null>( const descriptor = rawPresets[i]; if (descriptor.options !== false) { try { - // Presets normally run in reverse order, but if they - // have their own pass they run after the presets - // in the previous pass. - if (descriptor.ownPass) { - presets.push({ - preset: yield* loadPresetDescriptor( - descriptor, - presetContext, - ), - pass: [], - }); - } else { - presets.unshift({ - preset: yield* loadPresetDescriptor( - descriptor, - presetContext, - ), - pass: pluginDescriptorsPass, - }); - } + // eslint-disable-next-line no-var + var preset = yield* loadPresetDescriptor( + descriptor, + presetContext, + ); } catch (e) { if (e.code === "BABEL_UNKNOWN_OPTION") { checkNoUnwrappedItemOptionPairs(rawPresets, i, "preset", e); } throw e; } + + externalDependencies.push(preset.externalDependencies); + + // Presets normally run in reverse order, but if they + // have their own pass they run after the presets + // in the previous pass. + if (descriptor.ownPass) { + presets.push({ preset: preset.chain, pass: [] }); + } else { + presets.unshift({ + preset: preset.chain, + pass: pluginDescriptorsPass, + }); + } } } @@ -183,7 +188,11 @@ export default gensync<(inputOpts: unknown) => ResolvedConfig | null>( const descriptor: UnloadedDescriptor = descs[i]; if (descriptor.options !== false) { try { - pass.push(yield* loadPluginDescriptor(descriptor, pluginContext)); + // eslint-disable-next-line no-var + var plugin = yield* loadPluginDescriptor( + descriptor, + pluginContext, + ); } catch (e) { if (e.code === "BABEL_UNKNOWN_PLUGIN_PROPERTY") { // print special message for `plugins: ["@babel/foo", { foo: "option" }]` @@ -191,6 +200,9 @@ export default gensync<(inputOpts: unknown) => ResolvedConfig | null>( } throw e; } + pass.push(plugin); + + externalDependencies.push(plugin.externalDependencies); } } } @@ -206,6 +218,7 @@ export default gensync<(inputOpts: unknown) => ResolvedConfig | null>( return { options: opts, passes: passes, + externalDependencies: freezeDeepArray(externalDependencies), }; }, ); @@ -230,8 +243,11 @@ function enhanceError(context, fn: T): T { * Load a generic plugin/preset from the given descriptor loaded from the config object. */ const makeDescriptorLoader = ( - apiFactory: (cache: CacheConfigurator) => API, -): ((d: UnloadedDescriptor, c: Context) => Handler) => + apiFactory: ( + cache: CacheConfigurator, + externalDependencies: Array, + ) => API, +) => makeWeakCache(function* ( { value, options, dirname, alias }: UnloadedDescriptor, cache: CacheConfigurator, @@ -241,6 +257,8 @@ const makeDescriptorLoader = ( options = options || {}; + const externalDependencies: Array = []; + let item = value; if (typeof value === "function") { const factory = maybeAsync( @@ -250,7 +268,7 @@ const makeDescriptorLoader = ( const api = { ...context, - ...apiFactory(cache), + ...apiFactory(cache, externalDependencies), }; try { item = yield* factory(api, options, dirname); @@ -280,7 +298,28 @@ const makeDescriptorLoader = ( ); } - return { value: item, options, dirname, alias }; + if ( + externalDependencies.length > 0 && + (!cache.configured() || cache.mode() === "forever") + ) { + let error = + `A plugin/preset has external untracked dependencies ` + + `(${externalDependencies[0]}), but the cache `; + if (!cache.configured()) { + error += `has not been configured to be invalidated when the external dependencies change. `; + } else { + error += ` has been configured to never be invalidated. `; + } + error += + `Plugins/presets should configure their cache to be invalidated when the external ` + + `dependencies change, for example using \`api.cache.invalidate(() => ` + + `statSync(filepath).mtimeMs)\` or \`api.cache.never()\`\n` + + `(While processing: ${JSON.stringify(alias)})`; + + throw new Error(error); + } + + return { value: item, options, dirname, alias, externalDependencies }; }); const pluginDescriptorLoader = makeDescriptorLoader< @@ -316,7 +355,7 @@ function* loadPluginDescriptor( } const instantiatePlugin = makeWeakCache(function* ( - { value, options, dirname, alias }: LoadedDescriptor, + { value, options, dirname, alias, externalDependencies }: LoadedDescriptor, cache: CacheConfigurator, ): Handler { const pluginObj = validatePluginObject(value); @@ -339,7 +378,10 @@ const instantiatePlugin = makeWeakCache(function* ( dirname, }; - const inherits = yield* forwardAsync(loadPluginDescriptor, run => { + const inherits = yield* forwardAsync< + (d: UnloadedDescriptor, c: Context.SimplePlugin) => Plugin, + Plugin + >(loadPluginDescriptor, run => { // If the inherited plugin changes, reinstantiate this plugin. return cache.invalidate(data => run(inheritsDescriptor, data)); }); @@ -354,9 +396,11 @@ const instantiatePlugin = makeWeakCache(function* ( inherits.visitor || {}, plugin.visitor || {}, ]); + + externalDependencies.push(inherits.externalDependencies); } - return new Plugin(plugin, options, alias); + return new Plugin(plugin, options, alias, externalDependencies); }); const validateIfOptionNeedsFilename = ( @@ -401,20 +445,32 @@ const validatePreset = ( function* loadPresetDescriptor( descriptor: UnloadedDescriptor, context: Context.FullPreset, -): Handler { +): Handler<{ + chain: ConfigChain | null; + externalDependencies: ReadonlyDeepArray; +}> { const preset = instantiatePreset( yield* presetDescriptorLoader(descriptor, context), ); validatePreset(preset, context, descriptor); - return yield* buildPresetChain(preset, context); + return { + chain: yield* buildPresetChain(preset, context), + externalDependencies: preset.externalDependencies, + }; } const instantiatePreset = makeWeakCacheSync( - ({ value, dirname, alias }: LoadedDescriptor): PresetInstance => { + ({ + value, + dirname, + alias, + externalDependencies, + }: LoadedDescriptor): PresetInstance => { return { options: validate("preset", value), alias, dirname, + externalDependencies: freezeDeepArray(externalDependencies), }; }, ); diff --git a/packages/babel-core/src/config/helpers/config-api.ts b/packages/babel-core/src/config/helpers/config-api.ts index 16fe7040a28e..3275c193da5f 100644 --- a/packages/babel-core/src/config/helpers/config-api.ts +++ b/packages/babel-core/src/config/helpers/config-api.ts @@ -37,6 +37,7 @@ export type ConfigAPI = { export type PresetAPI = { targets: TargetsFunction; + addExternalDependency: (ref: string) => void; } & ConfigAPI; export type PluginAPI = { @@ -77,6 +78,7 @@ export function makeConfigAPI( export function makePresetAPI( cache: CacheConfigurator, + externalDependencies: Array, ): PresetAPI { const targets = () => // We are using JSON.parse/JSON.stringify because it's only possible to cache @@ -84,15 +86,21 @@ export function makePresetAPI( // only contains strings as its properties. // Please make the Record and Tuple proposal happen! JSON.parse(cache.using(data => JSON.stringify(data.targets))); - return { ...makeConfigAPI(cache), targets }; + + const addExternalDependency = (ref: string) => { + externalDependencies.push(ref); + }; + + return { ...makeConfigAPI(cache), targets, addExternalDependency }; } export function makePluginAPI( cache: CacheConfigurator, + externalDependencies: Array, ): PluginAPI { const assumption = name => cache.using(data => data.assumptions[name]); - return { ...makePresetAPI(cache), assumption }; + return { ...makePresetAPI(cache, externalDependencies), assumption }; } function assertVersion(range: string | number): void { diff --git a/packages/babel-core/src/config/helpers/deep-array.ts b/packages/babel-core/src/config/helpers/deep-array.ts new file mode 100644 index 000000000000..02d73d3ba1ae --- /dev/null +++ b/packages/babel-core/src/config/helpers/deep-array.ts @@ -0,0 +1,23 @@ +export type DeepArray = Array>; + +// Just to make sure that DeepArray is not assignable to ReadonlyDeepArray +declare const __marker: unique symbol; +export type ReadonlyDeepArray = ReadonlyArray> & { + [__marker]: true; +}; + +export function finalize(deepArr: DeepArray): ReadonlyDeepArray { + return Object.freeze(deepArr) as ReadonlyDeepArray; +} + +export function flattenToSet(arr: ReadonlyDeepArray): Set { + const result = new Set(); + const stack = [arr]; + while (stack.length > 0) { + for (const el of stack.pop()) { + if (Array.isArray(el)) stack.push(el as ReadonlyDeepArray); + else result.add(el as T); + } + } + return result; +} diff --git a/packages/babel-core/src/config/plugin.ts b/packages/babel-core/src/config/plugin.ts index 1ac360fc8f99..bb3151e0c018 100644 --- a/packages/babel-core/src/config/plugin.ts +++ b/packages/babel-core/src/config/plugin.ts @@ -1,3 +1,5 @@ +import { finalize } from "./helpers/deep-array"; +import type { DeepArray, ReadonlyDeepArray } from "./helpers/deep-array"; import type { PluginObject } from "./validation/plugins"; export default class Plugin { @@ -12,7 +14,14 @@ export default class Plugin { options: {}; - constructor(plugin: PluginObject, options: {}, key?: string) { + externalDependencies: ReadonlyDeepArray; + + constructor( + plugin: PluginObject, + options: {}, + key?: string, + externalDependencies: DeepArray = [], + ) { this.key = plugin.name || key; this.manipulateOptions = plugin.manipulateOptions; @@ -23,5 +32,7 @@ export default class Plugin { this.generatorOverride = plugin.generatorOverride; this.options = options; + + this.externalDependencies = finalize(externalDependencies); } } diff --git a/packages/babel-core/src/transformation/index.ts b/packages/babel-core/src/transformation/index.ts index bcae18c09937..2375dc128a9c 100644 --- a/packages/babel-core/src/transformation/index.ts +++ b/packages/babel-core/src/transformation/index.ts @@ -13,6 +13,8 @@ import normalizeFile from "./normalize-file"; import generateCode from "./file/generate"; import type File from "./file/file"; +import { flattenToSet } from "../config/helpers/deep-array"; + export type FileResultCallback = { (err: Error, file: null): any; (err: null, file: FileResult | null): any; @@ -25,6 +27,7 @@ export type FileResult = { code: string | null; map: SourceMap | null; sourceType: "string" | "module"; + externalDependencies: Set; }; export function* run( @@ -70,6 +73,7 @@ export function* run( code: outputCode === undefined ? null : outputCode, map: outputMap === undefined ? null : outputMap, sourceType: file.ast.program.sourceType, + externalDependencies: flattenToSet(config.externalDependencies), }; } diff --git a/packages/babel-core/test/external-dependencies.js b/packages/babel-core/test/external-dependencies.js new file mode 100644 index 000000000000..fe041e054484 --- /dev/null +++ b/packages/babel-core/test/external-dependencies.js @@ -0,0 +1,233 @@ +import path from "path"; +import { fileURLToPath } from "url"; +import { transformSync } from "../lib/index.js"; + +const cwd = path.dirname(fileURLToPath(import.meta.url)); + +function transform(code, opts) { + return transformSync(code, { cwd, configFile: false, ...opts }); +} + +describe("externalDependencies", () => { + function makePlugin(external) { + return api => { + api.cache.invalidate(() => ""); + api.addExternalDependency(external); + return { visitor: {} }; + }; + } + + function makePreset(external, plugins = []) { + return api => { + api.cache.invalidate(() => ""); + api.addExternalDependency(external); + return { plugins }; + }; + } + + it("can be set by plugins", () => { + const { externalDependencies } = transform("", { + plugins: [makePlugin("./foo")], + }); + + expect(externalDependencies).toEqual(new Set(["./foo"])); + }); + + it("returns a fresh set", () => { + const options = { + plugins: [makePlugin("./foo")], + }; + const res1 = transform("", options); + const res2 = transform("", options); + + expect(res1.externalDependencies).toEqual(res2.externalDependencies); + expect(res1.externalDependencies).not.toBe(res2.externalDependencies); + }); + + it("can be set multiple times by the same plugin", () => { + const { externalDependencies } = transform("", { + plugins: [ + function (api) { + api.cache.never(); + api.addExternalDependency("./foo"); + api.addExternalDependency("./bar"); + return { visitor: {} }; + }, + ], + }); + + expect(externalDependencies).toEqual(new Set(["./foo", "./bar"])); + }); + + it("can be set by presets", () => { + const { externalDependencies } = transform("", { + presets: [makePreset("./foo")], + }); + + expect(externalDependencies).toEqual(new Set(["./foo"])); + }); + + it("can be set multiple times by the same preset", () => { + const { externalDependencies } = transform("", { + presets: [ + function (api) { + api.cache.never(); + api.addExternalDependency("./foo"); + api.addExternalDependency("./bar"); + return { plugins: [] }; + }, + ], + }); + + expect(externalDependencies).toEqual(new Set(["./foo", "./bar"])); + }); + + it("can be set by multiple plugins and presets", () => { + const { externalDependencies } = transform("", { + plugins: [makePlugin("./plugin1"), makePlugin("./plugin2")], + presets: [ + makePreset("./preset", [ + makePlugin("./preset-plugin1"), + makePlugin("./preset-plugin2"), + ]), + ], + }); + + expect(externalDependencies).toEqual( + new Set([ + "./plugin1", + "./plugin2", + "./preset", + "./preset-plugin1", + "./preset-plugin2", + ]), + ); + }); + + describe("cached plugins", () => { + it("returned when set by cached plugins", () => { + const plugin = jest.fn(makePlugin("./foo")); + + const result1 = transform("", { plugins: [plugin] }); + const result2 = transform("", { plugins: [plugin] }); + + expect(plugin).toHaveBeenCalledTimes(1); + + expect(new Set(result1.externalDependencies)).toEqual(new Set(["./foo"])); + expect(new Set(result2.externalDependencies)).toEqual(new Set(["./foo"])); + }); + + it("cached external depenencies are merged with new ones", () => { + const plugin1 = jest.fn(makePlugin("./foo")); + const plugin2 = jest.fn((api, { file }) => { + api.addExternalDependency(file); + api.cache.never(); + return { visitor: {} }; + }); + + const result1 = transform("", { + plugins: [plugin1, [plugin2, { file: "./file1" }]], + }); + const result2 = transform("", { + plugins: [plugin1, [plugin2, { file: "./file2" }]], + }); + + expect(plugin1).toHaveBeenCalledTimes(1); + expect(plugin2).toHaveBeenCalledTimes(2); + + expect(new Set(result1.externalDependencies)).toEqual( + new Set(["./foo", "./file1"]), + ); + expect(new Set(result2.externalDependencies)).toEqual( + new Set(["./foo", "./file2"]), + ); + }); + }); + + describe("cache validation", () => { + it("cache must be configured", () => { + function plugin(api) { + api.addExternalDependency("./foo"); + return { visitor: {} }; + } + + expect(() => transform("", { plugins: [plugin] })).toThrow( + /A plugin\/preset has external untracked dependencies/, + ); + }); + + it("cache.forever() is disallowed", () => { + function plugin(api) { + api.cache.forever(); + api.addExternalDependency("./foo"); + return { visitor: {} }; + } + + expect(() => transform("", { plugins: [plugin] })).toThrow( + /A plugin\/preset has external untracked dependencies/, + ); + }); + + it("cache.never() is a valid configuration", () => { + function plugin(api) { + api.cache.never(); + api.addExternalDependency("./foo"); + return { visitor: {} }; + } + + expect(() => transform("", { plugins: [plugin] })).not.toThrow(); + }); + + it("cache.using() is a valid configuration", () => { + function plugin(api) { + api.cache.using(() => ""); + api.addExternalDependency("./foo"); + return { visitor: {} }; + } + + expect(() => transform("", { plugins: [plugin] })).not.toThrow(); + }); + + it("cache.invalidate() is a valid configuration", () => { + function plugin(api) { + api.cache.invalidate(() => ""); + api.addExternalDependency("./foo"); + return { visitor: {} }; + } + + expect(() => transform("", { plugins: [plugin] })).not.toThrow(); + }); + + it("cache must be configured in the same plugin that uses addExternalDependency", () => { + function plugin1(api) { + api.cache.never(); + return { visitor: {} }; + } + + function plugin2(api) { + api.addExternalDependency("./foo"); + return { visitor: {} }; + } + + expect(() => transform("", { plugins: [plugin1, plugin2] })).toThrow( + /A plugin\/preset has external untracked dependencies/, + ); + }); + + it("cache does not need to be configured in other plugins", () => { + function plugin1() { + return { visitor: {} }; + } + + function plugin2(api) { + api.cache.never(); + api.addExternalDependency("./foo"); + return { visitor: {} }; + } + + expect(() => + transform("", { plugins: [plugin1, plugin2] }), + ).not.toThrow(); + }); + }); +});