diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 18a85e1eb57e..625575b18cb3 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -145,9 +145,9 @@ If you need to check for an error that is thrown you can add to the `options.jso
 }
 ```
 
-##### `babylon`
+##### Bootstrapping expected output
 
-For `babylon` specifically, you can easily generate an `expected.json` automatically by just providing the `actual.js` and running `make test-only` as you usually would.
+For both `babel-plugin-x` and `babylon`, you can easily generate an `expected.js`/`expected.json` automatically by just providing `actual.js` and running the tests as you usually would.
 
 ```
 // Example
diff --git a/packages/babel-helper-transform-fixture-test-runner/src/index.js b/packages/babel-helper-transform-fixture-test-runner/src/index.js
index dac7ffdf4299..42ce63a16ebe 100644
--- a/packages/babel-helper-transform-fixture-test-runner/src/index.js
+++ b/packages/babel-helper-transform-fixture-test-runner/src/index.js
@@ -8,6 +8,8 @@ import assert from "assert";
 import chai from "chai";
 import _ from "lodash";
 import "babel-polyfill";
+import fs from "fs";
+import path from "path";
 
 let babelHelpers = eval(buildExternalHelpers(null, "var"));
 
@@ -63,14 +65,12 @@ function run(task) {
   let actualCode = actual.code;
   let expectCode = expect.code;
   if (!execCode || actualCode) {
-    result     = babel.transform(actualCode, getOpts(actual));
-    actualCode = result.code.trim();
-
-    try {
+    result = babel.transform(actualCode, getOpts(actual));
+    if (!expect.code && result.code && !opts.throws && fs.statSync(path.dirname(expect.loc)).isDirectory() && !process.env.CI) {
+      fs.writeFileSync(expect.loc, result.code);
+    } else {
+      actualCode = result.code.trim();
       chai.expect(actualCode).to.be.equal(expectCode, actual.loc + " !== " + expect.loc);
-    } catch (err) {
-      //require("fs").writeFileSync(expect.loc, actualCode);
-      throw err;
     }
   }