Merge pull request #10585 from meteor/invoke-compileOneJsResource-fro…

…m-ImportScanner

Support full ECMAScript compilation of modules unexpectedly imported from node_modules.

History.md

@@ -11,7 +11,9 @@ N/A
 * Node has been updated to version
   [8.16.0](https://nodejs.org/en/blog/release/v8.16.0/).
 
-* The `meteor-babel` npm package has been updated to version 7.4.3.
+* The `meteor-babel` npm package has been updated to version 7.4.13.
+
+* The `reify` npm package has been updated to version 0.20.5.
 
 * The `core-js` npm package used by `ecmascript-runtime-client` and
   `ecmascript-runtime-server` has been updated to version 3.1.4.

meteor

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-BUNDLE_VERSION=8.16.0.4
+BUNDLE_VERSION=8.16.0.7
 
 # OS Check. Put here because here is where we download the precompiled
 # bundles that are arch specific.

packages/babel-compiler/babel-compiler.js

@@ -67,12 +67,6 @@ BCp.processOneFileForTarget = function (inputFile, source) {
     sourceMap: null,
     bare: !! fileOptions.bare
   };
-  var cacheOptions = {
-    cacheDirectory: this.cacheDirectory,
-    cacheDeps: {
-      sourceHash: toBeAdded.hash,
-    },
-  };
 
   // If you need to exclude a specific file within a package from Babel
   // compilation, pass the { transpile: false } options to api.addFiles
@@ -85,8 +79,8 @@ BCp.processOneFileForTarget = function (inputFile, source) {
       // compilation, give it the following file extension: .es5.js
       ! excludedFileExtensionPattern.test(inputFilePath)) {
 
-    var extraFeatures = Object.assign({}, this.extraFeatures);
-    var arch = inputFile.getArch();
+    const extraFeatures = { ...this.extraFeatures };
+    const arch = inputFile.getArch();
 
     if (arch.startsWith("os.")) {
       // Start with a much simpler set of Babel presets and plugins if
@@ -103,9 +97,24 @@ BCp.processOneFileForTarget = function (inputFile, source) {
       extraFeatures.jscript = true;
     }
 
+    if (shouldCompileModulesOnly(inputFilePath)) {
+      // Modules like @babel/runtime/helpers/esm/typeof.js need to be
+      // compiled to support ECMAScript modules syntax, but should *not*
+      // be compiled in any other way (for more explanation, see my longer
+      // comment in shouldCompileModulesOnly).
+      extraFeatures.compileModulesOnly = true;
+    }
+
     var babelOptions = Babel.getDefaultOptions(extraFeatures);
     babelOptions.caller = { name: "meteor", arch };
 
+    const cacheOptions = {
+      cacheDirectory: this.cacheDirectory,
+      cacheDeps: {
+        sourceHash: toBeAdded.hash,
+      },
+    };
+
     this.inferExtraBabelOptions(
       inputFile,
       babelOptions,
@@ -165,6 +174,32 @@ BCp.processOneFileForTarget = function (inputFile, source) {
   return toBeAdded;
 };
 
+function shouldCompileModulesOnly(path) {
+  const parts = path.split("/");
+  const nmi = parts.lastIndexOf("node_modules");
+  if (nmi >= 0) {
+    const part1 = parts[nmi + 1];
+    // We trust that any code related to @babel/runtime has already been
+    // compiled adequately. The @babel/runtime/helpers/typeof module is a
+    // good example of why double-compilation is risky for these packages,
+    // since it uses native typeof syntax to implement its polyfill for
+    // Symbol-aware typeof, so compiling it again would cause the
+    // generated code to try to require itself. In general, compiling code
+    // more than once with Babel should be safe (just unnecessary), except
+    // for code that Babel itself relies upon at runtime. Finally, if this
+    // hard-coded list of package names proves to be incomplete, we can
+    // always add to it (or even replace it completely) by releasing a new
+    // version of the babel-compiler package.
+    if (part1 === "@babel" ||
+        part1 === "core-js" ||
+        part1 === "regenerator-runtime") {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 BCp.setDiskCacheDirectory = function (cacheDir) {
   this.cacheDirectory = cacheDir;
 };

packages/babel-compiler/package.js

@@ -6,11 +6,11 @@ Package.describe({
   // isn't possible because you can't publish a non-recommended
   // release with package versions that don't have a pre-release
   // identifier at the end (eg, -dev)
-  version: '7.3.4'
+  version: '7.4.0'
 });
 
 Npm.depends({
-  'meteor-babel': '7.3.4',
+  'meteor-babel': '7.4.13',
   'json5': '2.1.0'
 });
 

packages/ecmascript/package.js

@@ -1,6 +1,6 @@
 Package.describe({
   name: 'ecmascript',
-  version: '0.12.7',
+  version: '0.13.0',
   summary: 'Compiler plugin that supports ES2015+ in all .js files',
   documentation: 'README.md'
 });

packages/modules/package.js

@@ -1,12 +1,12 @@
 Package.describe({
   name: "modules",
-  version: "0.13.0",
+  version: "0.14.0",
   summary: "CommonJS module system",
   documentation: "README.md"
 });
 
 Npm.depends({
-  reify: "0.17.3"
+  reify: "0.20.5"
 });
 
 Package.onUse(function(api) {

scripts/dev-bundle-tool-package.js

@@ -14,11 +14,11 @@ var packageJson = {
     pacote: "https://github.com/meteor/pacote/tarball/c5043daa1b768594e01d76275e3854fc19f038f9",
     "node-gyp": "3.7.0",
     "node-pre-gyp": "0.10.3",
-    "meteor-babel": "7.4.6",
+    "meteor-babel": "7.4.13",
     // Keep the versions of these packages consistent with the versions
     // found in dev-bundle-server-package.js.
     "meteor-promise": "0.8.7",
-    reify: "0.19.1",
+    reify: "0.20.5",
     fibers: "3.1.1",
     // So that Babel can emit require("@babel/runtime/helpers/...") calls.
     "@babel/runtime": "7.4.4",

tools/isobuild/compiler-plugin.js

@@ -288,19 +288,10 @@ class InputFile extends buildPluginModule.InputFile {
   }
 
   readAndWatchFileWithHash(path) {
-    const osPath = files.convertToOSPath(path);
-    const sourceRoot = this.getSourceRoot();
-    const relPath = files.pathRelative(sourceRoot, osPath);
-    if (relPath.startsWith("..")) {
-      throw new Error(
-        `Attempting to read file outside ${
-          this.getPackageName() || "the app"}: ${osPath}`
-      );
-    }
     const sourceBatch = this._resourceSlot.packageSourceBatch;
     return readAndWatchFileWithHash(
       sourceBatch.unibuild.watchSet,
-      osPath
+      files.convertToOSPath(path),
     );
   }
 
@@ -1056,42 +1047,12 @@ export class PackageSourceBatch {
     self.importExtensions = [".js", ".json"];
     self._nodeModulesPaths = null;
 
-    var sourceProcessorSet = self._getSourceProcessorSet();
-
     self.resourceSlots = [];
-    unibuild.resources.forEach(function (resource) {
-      let sourceProcessor = null;
-      if (resource.type === "source") {
-        var extension = resource.extension;
-        if (extension === null) {
-          const filename = files.pathBasename(resource.path);
-          sourceProcessor = sourceProcessorSet.getByFilename(filename);
-          if (! sourceProcessor) {
-            buildmessage.error(
-              `no plugin found for ${ resource.path } in ` +
-                `${ unibuild.pkg.displayName() }; a plugin for ${ filename } ` +
-                `was active when it was published but none is now`);
-            return;
-            // recover by ignoring
-          }
-        } else {
-          sourceProcessor = sourceProcessorSet.getByExtension(extension);
-          // If resource.extension === 'js', it's ok for there to be no
-          // sourceProcessor, since we #HardcodeJs in ResourceSlot.
-          if (! sourceProcessor && extension !== 'js') {
-            buildmessage.error(
-              `no plugin found for ${ resource.path } in ` +
-                `${ unibuild.pkg.displayName() }; a plugin for *.${ extension } ` +
-                `was active when it was published but none is now`);
-            return;
-            // recover by ignoring
-          }
-
-          self.addImportExtension(extension);
-        }
+    unibuild.resources.forEach(resource => {
+      const slot = self.makeResourceSlot(resource);
+      if (slot) {
+        self.resourceSlots.push(slot);
       }
-
-      self.resourceSlots.push(new ResourceSlot(resource, sourceProcessor, self));
     });
 
     // Compute imports by merging the exports of all of the packages we
@@ -1141,6 +1102,85 @@ export class PackageSourceBatch {
     } : null;
   }
 
+  compileOneJsResource(resource) {
+    const slot = this.makeResourceSlot({
+      type: "source",
+      extension: "js",
+      // Need { data, path, hash } here, at least.
+      ...resource,
+      fileOptions: {
+        lazy: true,
+        ...resource.fileOptions,
+      }
+    });
+
+    if (slot) {
+      // If the resource was not handled by a source processor, it will be
+      // added directly to slot.jsOutputResources by makeResourceSlot,
+      // meaning we do not need to compile it.
+      if (slot.jsOutputResources.length > 0) {
+        return slot.jsOutputResources
+      }
+
+      const inputFile = new InputFile(slot);
+      inputFile.supportsLazyCompilation = false;
+
+      if (slot.sourceProcessor) {
+        const { userPlugin } = slot.sourceProcessor;
+        if (userPlugin) {
+          const markedMethod = buildmessage.markBoundary(
+            userPlugin.processFilesForTarget,
+            userPlugin
+          );
+          try {
+            Promise.await(markedMethod([inputFile]));
+          } catch (e) {
+            buildmessage.exception(e);
+          }
+        }
+      }
+
+      return slot.jsOutputResources;
+    }
+
+    return [];
+  }
+
+  makeResourceSlot(resource) {
+    let sourceProcessor = null;
+    if (resource.type === "source") {
+      var extension = resource.extension;
+      if (extension === null) {
+        const filename = files.pathBasename(resource.path);
+        sourceProcessor = this._getSourceProcessorSet().getByFilename(filename);
+        if (! sourceProcessor) {
+          buildmessage.error(
+            `no plugin found for ${ resource.path } in ` +
+              `${ this.unibuild.pkg.displayName() }; a plugin for ${ filename } ` +
+              `was active when it was published but none is now`);
+          return null;
+          // recover by ignoring
+        }
+      } else {
+        sourceProcessor = this._getSourceProcessorSet().getByExtension(extension);
+        // If resource.extension === 'js', it's ok for there to be no
+        // sourceProcessor, since we #HardcodeJs in ResourceSlot.
+        if (! sourceProcessor && extension !== 'js') {
+          buildmessage.error(
+            `no plugin found for ${ resource.path } in ` +
+              `${ this.unibuild.pkg.displayName() }; a plugin for *.${ extension } ` +
+              `was active when it was published but none is now`);
+          return null;
+          // recover by ignoring
+        }
+
+        this.addImportExtension(extension);
+      }
+    }
+
+    return new ResourceSlot(resource, sourceProcessor, this);
+  }
+
   addImportExtension(extension) {
     extension = extension.toLowerCase();
 
@@ -1181,26 +1221,27 @@ export class PackageSourceBatch {
   }
 
   _getSourceProcessorSet() {
-    const self = this;
-
-    buildmessage.assertInJob();
+    if (! this._sourceProcessorSet) {
+      buildmessage.assertInJob();
 
-    var isopack = self.unibuild.pkg;
-    const activePluginPackages = compiler.getActivePluginPackages(isopack, {
-      uses: self.unibuild.uses,
-      isopackCache: self.processor.isopackCache
-    });
-    const sourceProcessorSet = new buildPluginModule.SourceProcessorSet(
-      isopack.displayName(), { hardcodeJs: true });
+      const isopack = this.unibuild.pkg;
+      const activePluginPackages = compiler.getActivePluginPackages(isopack, {
+        uses: this.unibuild.uses,
+        isopackCache: this.processor.isopackCache
+      });
 
-    _.each(activePluginPackages, function (otherPkg) {
-      otherPkg.ensurePluginsInitialized();
+      this._sourceProcessorSet = new buildPluginModule.SourceProcessorSet(
+        isopack.displayName(), { hardcodeJs: true });
 
-      sourceProcessorSet.merge(
-        otherPkg.sourceProcessors.compiler, {arch: self.processor.arch});
-    });
+      _.each(activePluginPackages, otherPkg => {
+        otherPkg.ensurePluginsInitialized();
+        this._sourceProcessorSet.merge(otherPkg.sourceProcessors.compiler, {
+          arch: this.processor.arch,
+        });
+      });
+    }
 
-    return sourceProcessorSet;
+    return this._sourceProcessorSet;
   }
 
   // Returns a map from package names to arrays of JS output files.
@@ -1302,6 +1343,7 @@ export class PackageSourceBatch {
         sourceRoot: batch.sourceRoot,
         nodeModulesPaths,
         watchSet: batch.unibuild.watchSet,
+        compileOneJsResource: batch.compileOneJsResource.bind(batch),
         cacheDir: batch.scannerCacheDir,
       });