Use stderr/stdout, and add a validateLogs test option

packages/babel-helper-fixtures/src/index.js

@@ -120,6 +120,8 @@ export default function get(entryLoc): Array<Suite> {
       const expectLoc =
         findFile(taskDir + "/output", true /* allowJSON */) ||
         taskDir + "/output.js";
+      const stdoutLoc = taskDir + "/stdout.txt";
+      const stderrLoc = taskDir + "/stderr.txt";
 
       const actualLocAlias =
         suiteName + "/" + taskName + "/" + path.basename(actualLoc);
@@ -141,14 +143,14 @@ export default function get(entryLoc): Array<Suite> {
       const taskOptsLoc = resolve(taskDir + "/options");
       if (taskOptsLoc) extend(taskOpts, require(taskOptsLoc));
 
-      const taskLogLoc = resolve(taskDir + "/expected-log.txt");
-
       const test = {
         optionsDir: taskOptsLoc ? path.dirname(taskOptsLoc) : null,
         title: humanize(taskName, true),
         disabled: taskName[0] === ".",
         options: taskOpts,
-        expectedLog: taskLogLoc ? readFile(taskLogLoc) : null,
+        validateLogs: taskOpts.validateLogs,
+        stdout: { loc: stdoutLoc, code: readFile(stdoutLoc) },
+        stderr: { loc: stderrLoc, code: readFile(stderrLoc) },
         exec: {
           loc: execLoc,
           code: readFile(execLoc),
@@ -226,11 +228,15 @@ export default function get(entryLoc): Array<Suite> {
         }
       }
 
-      if (test.exec.code && test.expectedLog) {
+      if (!test.validateLogs && (test.stdout.code || test.stderr.code)) {
         throw new Error(
-          "Test cannot have logs and also be executed: " + taskLogLoc,
+          "stdout.txt and stderr.txt are only allowed when the 'validateLogs' option is enabled: " +
+            (test.stdout.code ? stdoutLoc : stderrLoc),
         );
       }
+
+      // Delete to avoid option validation error
+      delete test.options.validateLogs;
     }
   }
 

packages/babel-helper-transform-fixture-test-runner/src/index.js

@@ -137,7 +137,9 @@ function run(task) {
     exec,
     options: opts,
     optionsDir,
-    expectedLog,
+    validateLogs,
+    stdout,
+    stderr,
   } = task;
 
   function getOpts(self) {
@@ -197,13 +199,13 @@ function run(task) {
   let actualCode = actual.code;
   const expectCode = expected.code;
   if (!execCode || actualCode) {
-    let actualLogs = "";
-    if (expectedLog !== null) {
+    const actualLogs = { stdout: "", stderr: "" };
+    if (validateLogs) {
       jest.spyOn(console, "log").mockImplementation(msg => {
-        actualLogs += `>>>>>> [console.log] <<<<<<\n${msg}\n\n`;
+        actualLogs.stdout += `${msg}\n`;
       });
       jest.spyOn(console, "warn").mockImplementation(msg => {
-        actualLogs += `>>>>>> [console.warn] <<<<<<\n${msg}\n\n`;
+        actualLogs.stderr += `${msg}\n`;
       });
     }
 
@@ -214,8 +216,6 @@ function run(task) {
       "<CWD>",
     );
 
-    if (expectedLog !== null) expect(actualLogs.trim()).toBe(expectedLog);
-
     checkDuplicatedNodes(babel, result.ast);
     if (
       !expected.code &&
@@ -239,24 +239,19 @@ function run(task) {
       }
     } else {
       actualCode = expectedCode.trim();
-      try {
-        expect(actualCode).toEqualFile({
-          filename: expected.loc,
-          code: expectCode,
-        });
-      } catch (e) {
-        if (!process.env.OVERWRITE) throw e;
-
-        console.log(`Updated test file: ${expected.loc}`);
-        fs.writeFileSync(expected.loc, `${expectedCode}\n`);
-      }
+      validateFile(actualCode, expected.loc, expectCode);
 
       if (actualCode) {
         expect(expected.loc).toMatch(
           result.sourceType === "module" ? /\.mjs$/ : /\.js$/,
         );
       }
     }
+
+    if (validateLogs) {
+      validateFile(actualLogs.stdout.trim(), stdout.loc, stdout.code);
+      validateFile(actualLogs.stderr.trim(), stderr.loc, stderr.code);
+    }
   }
 
   if (task.sourceMap) {
@@ -279,6 +274,20 @@ function run(task) {
   }
 }
 
+function validateFile(actualCode, expectedLoc, expectedCode) {
+  try {
+    expect(actualCode).toEqualFile({
+      filename: expectedLoc,
+      code: expectedCode,
+    });
+  } catch (e) {
+    if (!process.env.OVERWRITE) throw e;
+
+    console.log(`Updated test file: ${expectedLoc}`);
+    fs.writeFileSync(expectedLoc, `${actualCode}\n`);
+  }
+}
+
 const toEqualFile = () => ({
   compare: (actual, { filename, code }) => {
     const pass = actual === code;

packages/babel-plugin-proposal-dynamic-import/test/fixtures/systemjs/missing-plugin/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "plugins": [
     "syntax-dynamic-import",
     "transform-modules-systemjs",

packages/babel-plugin-proposal-dynamic-import/test/fixtures/systemjs/missing-plugin/expected-log.txt → packages/babel-plugin-proposal-dynamic-import/test/fixtures/systemjs/missing-plugin/stderr.txt

@@ -1,4 +1,3 @@
->>>>>> [console.warn] <<<<<<
 WARNING: Dynamic import() transformation must be enabled using the
          @babel/plugin-proposal-dynamic-import plugin. Babel 8 will
          no longer transform import() without using that plugin.

packages/babel-preset-env/test/fixtures/dynamic-import/auto-esm-supported-import-supported/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "caller": {
     "name": "test-fixture",
     "supportsStaticESM": true,

packages/babel-preset-env/test/fixtures/dynamic-import/auto-esm-supported-import-unsupported/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "caller": {
     "name": "test-fixture",
     "supportsStaticESM": true,

packages/babel-preset-env/test/fixtures/dynamic-import/modules-umd/expected-log.txt → packages/babel-preset-env/test/fixtures/dynamic-import/auto-esm-supported-import-unsupported/stderr.txt

@@ -1,2 +1 @@
->>>>>> [console.warn] <<<<<<
 Dynamic import can only be supported when transforming ES modules to AMD, CommonJS or SystemJS. Only the parser plugin will be enabled.

packages/babel-preset-env/test/fixtures/dynamic-import/auto-esm-unsupported-import-unsupported/options.json

@@ -1,4 +1,5 @@
 {
+  "validateLogs": true,
   "caller": {
     "name": "test-fixture",
     "supportsStaticESM": false,

packages/babel-preset-env/test/fixtures/dynamic-import/modules-amd/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": "amd" }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/modules-cjs/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": "cjs" }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/modules-false/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": false }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/modules-systemjs/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": "systemjs" }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/modules-umd/options.json

@@ -1,3 +1,4 @@
 {
+  "validateLogs": true,
   "presets": [["env", { "modules": "umd" }]]
 }

packages/babel-preset-env/test/fixtures/dynamic-import/auto-esm-supported-import-unsupported/expected-log.txt → packages/babel-preset-env/test/fixtures/dynamic-import/modules-umd/stderr.txt

@@ -1,2 +1 @@
->>>>>> [console.warn] <<<<<<
 Dynamic import can only be supported when transforming ES modules to AMD, CommonJS or SystemJS. Only the parser plugin will be enabled.