Properly compare test files between jobs

lib/compare-results/index.js

@@ -21,6 +21,12 @@ function throwMasterArtifactOutOfSyncError() {
     `);
 }
 
+function getFileNameFromTitle(title) {
+    return title ?
+        title.split('#')[0].trim() :
+        undefined;
+}
+
 function parseTestResults(filePath) {
     return new Promise(resolve => _(fs.createReadStream(filePath).pipe(parser.stream()))
         .map(buffer => JSON.parse(buffer.toString()))
@@ -43,19 +49,23 @@ async function main() {
         throwMasterArtifactOutOfSyncError();
     }
     const masterTestsMap = masterTests.reduce((acc, test) =>  {
-        acc[test.title] = test;
+        if (test.type === 'assertion') {
+            acc[getFileNameFromTitle(test.title)] = test;
+        }
         return acc;
-    }, {});
+    }, Object.create(null));
+
     for (let i = 0; i < prTests.length; i++) {
         if (isCircleCIMaster) {
             return;
         }
         const prTest = prTests[i];
-        const masterTest = masterTestsMap[prTest.title];
+        const fileName = getFileNameFromTitle(prTest.title);
+        const masterTest = masterTestsMap[fileName];
         switch (prTest.type) {
             case 'assertion':
-                if (!Object.prototype.hasOwnProperty.call(masterTestsMap, prTest.title)) {
-                    diag(`Ignoring test '${prTest.title}' as it was not found in master artifact!`);
+                if (!(fileName in masterTestsMap)) {
+                    diag(`Ignoring test '${fileName}' as it was not found in master artifact!`);
                     continue;
                 }
                 if (prTest.ok !== masterTest.ok) {

lib/run-tests/index.js

@@ -92,9 +92,10 @@ function getExpected({ attrs }) {
 
 async function runTest(agent, test) {
   let { attrs, contents, file } = test;
+  const isModule = attrs.flags.module;
 
   try {
-    contents = transpile(contents, attrs.features);
+    contents = transpile(contents, attrs.features, isModule);
   } catch (error) {
     return { result: "parser error", error };
   }
@@ -146,9 +147,11 @@ class RethrownError extends ExtendedError {
     if (!error) throw new Error('RethrownError requires an error');
     this.original = error;
     this.new_stack = this.stack;
+    const errorStackString = error.stack && (typeof error.stack === 'string' ?
+      error.stack : error.stack.map(location => location.source).join('\n'));
     let message_lines = (this.message.match(/\n/g) || []).length + 1;
     this.stack = `${this.name}: ${this.message}\n${
       this.stack.split('\n').slice(0, message_lines + 1).join('\n')
-    }\n${error.stack.map(location => location.source).join('\n')}`;
+      }\n${errorStackString}`;
   }
 }

lib/run-tests/transpile.js

@@ -7,7 +7,7 @@ const getBabelPlugins = require("./get-babel-plugins");
 
 const configCaches = new Map();
 
-function getOptions(features) {
+function getOptions(features, isModule) {
   const featureKey = JSON.stringify(features);
   let config = configCaches.get(featureKey);
   if (config !== undefined) {
@@ -17,13 +17,13 @@ function getOptions(features) {
     configFile: false,
     presets: [[presetEnv, { spec: true }]],
     plugins: getBabelPlugins(features),
-    sourceType: "script",
+    sourceType: isModule ? "module" : "script",
   });
 
   configCaches.set(featureKey, config);
   return config;
 }
 
-module.exports = function transpile(code, features) {
-  return transformSync(code, getOptions(features)).code;
+module.exports = function transpile(code, features, isModule) {
+  return transformSync(code, getOptions(features, isModule)).code;
 };

package.json

@@ -25,6 +25,7 @@
     "progress": "^2.0.3",
     "regenerator-runtime": "^0.13.3",
     "rimraf": "^3.0.0",
+    "tap-merge": "^0.3.1",
     "tap-mocha-reporter": "^5.0.0",
     "targz": "^1.0.1",
     "tempy": "^0.3.0",