[langhost/node] More ESM Support

This follows up on the changes in #7764 with three additions:

* Support for passing args directly to node so that ESM loaders and other node process level configuration can be accessed via Pulumi
* Support the same default behaviour for loading a ESM module from Pulumi as for node <program>, including support for resolving main entrypoint from package folder.
* Add test cases for .js, pre-complied .ts and ts-node-based `.ts.

This includes test cases that show how to use top-level await in Node.js (which is only possible inside ESM), addressing #5161.

Using ESM via the default ts-node-based TypeScript support is a little tricky, as it is dependent on experimental loader hook support in Node, upon which partially in-progress ts-node support has been added. We cannot make these the default experience yet, but the examples here show how users can configure things themselves to access these features. Once this support solidifies and we can rely on it in all supported Node and TypeScript versions, we may be able to update templates to support more of this by default.

CHANGELOG_PENDING.md

@@ -3,4 +3,7 @@
 - [sdk/nodejs] Support using native ES modules as Pulumi scripts
   [#7764](https://github.com/pulumi/pulumi/pull/7764)
 
+- [sdk/nodejs] Support a `nodeargs` option for passing `node` arguments to the Node language host
+  [#8655](https://github.com/pulumi/pulumi/pull/8655)
+
 ### Bug Fixes

sdk/nodejs/cmd/pulumi-language-nodejs/main.go

@@ -83,13 +83,15 @@ func main() {
 	var typescript bool
 	var root string
 	var tsconfigpath string
+	var nodeargs string
 	flag.StringVar(&tracing, "tracing", "",
 		"Emit tracing to a Zipkin-compatible tracing endpoint")
 	flag.BoolVar(&typescript, "typescript", true,
 		"Use ts-node at runtime to support typescript source natively")
 	flag.StringVar(&root, "root", "", "Project root path to use")
 	flag.StringVar(&tsconfigpath, "tsconfig", "",
 		"Path to tsconfig.json to use")
+	flag.StringVar(&nodeargs, "nodeargs", "", "Arguments for the Node process")
 	flag.Parse()
 
 	args := flag.Args()
@@ -121,7 +123,7 @@ func main() {
 	// Fire up a gRPC server, letting the kernel choose a free port.
 	port, done, err := rpcutil.Serve(0, nil, []func(*grpc.Server) error{
 		func(srv *grpc.Server) error {
-			host := newLanguageHost(nodePath, runPath, engineAddress, tracing, typescript, tsconfigpath)
+			host := newLanguageHost(nodePath, runPath, engineAddress, tracing, typescript, tsconfigpath, nodeargs)
 			pulumirpc.RegisterLanguageRuntimeServer(srv, host)
 			return nil
 		},
@@ -159,17 +161,19 @@ type nodeLanguageHost struct {
 	tracing       string
 	typescript    bool
 	tsconfigpath  string
+	nodeargs      string
 }
 
 func newLanguageHost(nodePath, runPath, engineAddress,
-	tracing string, typescript bool, tsconfigpath string) pulumirpc.LanguageRuntimeServer {
+	tracing string, typescript bool, tsconfigpath string, nodeargs string) pulumirpc.LanguageRuntimeServer {
 	return &nodeLanguageHost{
 		nodeBin:       nodePath,
 		runPath:       runPath,
 		engineAddress: engineAddress,
 		tracing:       tracing,
 		typescript:    typescript,
 		tsconfigpath:  tsconfigpath,
+		nodeargs:      nodeargs,
 	}
 }
 
@@ -513,15 +517,21 @@ func (host *nodeLanguageHost) execNodejs(
 			env = append(env, "PULUMI_NODEJS_TSCONFIG_PATH="+host.tsconfigpath)
 		}
 
+		var nodeargs []string
+		if host.nodeargs != "" {
+			nodeargs = strings.Split(host.nodeargs, " ")
+		}
+		nodeargs = append(nodeargs, args...)
+
 		if logging.V(5) {
-			commandStr := strings.Join(args, " ")
+			commandStr := strings.Join(nodeargs, " ")
 			logging.V(5).Infoln("Language host launching process: ", host.nodeBin, commandStr)
 		}
 
 		// Now simply spawn a process to execute the requested program, wiring up stdout/stderr directly.
 		var errResult string
 		// #nosec G204
-		cmd := exec.Command(host.nodeBin, args...)
+		cmd := exec.Command(host.nodeBin, nodeargs...)
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
 		cmd.Env = env

sdk/nodejs/cmd/run/run.ts

@@ -260,6 +260,10 @@ ${defaultMessage}`);
 
             // We use dynamic import instead of require for projects using native ES modules instead of commonjs
             if (packageObject["type"] === "module") {
+                // Use the same behavior for loading the main entrypoint as `node <program>`.
+                // See https://github.com/nodejs/node/blob/master/lib/internal/modules/run_main.js#L74.
+                const mainPath: string = require("module").Module._findPath(path.resolve(program), null, true) || program;
+                const main = path.isAbsolute(mainPath) ? url.pathToFileURL(mainPath).href : mainPath;
                 // Workaround for typescript transpiling dynamic import into `Promise.resolve().then(() => require`
                 // Follow this issue for progress on when we can remove this:
                 // https://github.com/microsoft/TypeScript/issues/43329
@@ -271,7 +275,7 @@ ${defaultMessage}`);
                 // Import the module and capture any module outputs it exported. Finally, await the value we get
                 // back.  That way, if it is async and throws an exception, we properly capture it here
                 // and handle it.
-                return await dynamicImport(url.pathToFileURL(program).toString());
+                return await dynamicImport(main);
             }
 
             // Execute the module and capture any module outputs it exported. If the exported value

tests/integration/integration_nodejs_test.go

@@ -1142,6 +1142,47 @@ func TestCompilerOptionsNode(t *testing.T) {
 	})
 }
 
+func TestESMJS(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-js"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		Quick:        true,
+	})
+}
+
+func TestESMJSMain(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-js-main"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		Quick:        true,
+	})
+}
+
+func TestESMTS(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-ts"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		Quick:        true,
+	})
+}
+
+func TestESMTSSpecifierResolutionNode(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-ts-specifier-resolution-node"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		Quick:        true,
+	})
+}
+
+func TestESMTSCompiled(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-ts-compiled"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		RunBuild:     true,
+		Quick:        true,
+	})
+}
+
 // Test that the about command works as expected. Because about parses the
 // results of each runtime independently, we have an integration test in each
 // language.

tests/integration/nodejs/.gitignore

@@ -0,0 +1,4 @@
+.pulumi/
+bin/
+node_modules/
+Pulumi.*.yaml

tests/integration/nodejs/esm-js-main/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: esm-js-main
+runtime: nodejs
+description: Use ECMAScript modules for a plain JS program.

tests/integration/nodejs/esm-js-main/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "esm-js-main",
+    "license": "Apache-2.0",
+    "type": "module",
+    "main": "src/main.js",
+    "peerDependencies": {
+        "@pulumi/pulumi": "latest"
+    },
+    "dependencies": {
+        "@pulumi/aws": "^4.33.0"
+    }
+}

tests/integration/nodejs/esm-js-main/src/main.js

@@ -0,0 +1,8 @@
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
+
+import * as fs from "fs";
+import * as aws from "@pulumi/aws";
+
+const b = new aws.s3.Bucket("b");
+
+export const res = fs.readFileSync("Pulumi.yaml").toString();

tests/integration/nodejs/esm-js/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: esm-js
+runtime: nodejs
+description: Use ECMAScript modules for a plain JS program.

tests/integration/nodejs/esm-js/index.js

@@ -0,0 +1,5 @@
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
+
+import * as fs from "fs";
+
+export const res = fs.readFileSync("Pulumi.yaml").toString();

tests/integration/nodejs/esm-js/package.json

@@ -0,0 +1,8 @@
+{
+    "name": "esm-js",
+    "license": "Apache-2.0",
+    "type": "module",
+    "peerDependencies": {
+        "@pulumi/pulumi": "latest"
+    }
+}

tests/integration/nodejs/esm-ts-compiled/Pulumi.yaml

@@ -0,0 +1,6 @@
+name: esm-ts-compiled
+runtime: 
+  name: nodejs
+  options: 
+    typescript: false
+description: Use ECMAScript modules for a TS program.

tests/integration/nodejs/esm-ts-compiled/index.ts

@@ -0,0 +1,10 @@
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
+
+import * as fs from "fs";
+import { x } from "./other.js"; // this is the "by design" way to do this, even in TS.
+
+// Use top-level await
+await new Promise(r => setTimeout(r, 2000));
+
+export const res = fs.readFileSync("Pulumi.yaml").toString();
+export const otherx = x;

tests/integration/nodejs/esm-ts-compiled/other.ts

@@ -0,0 +1 @@
+export const x = 42;

tests/integration/nodejs/esm-ts-compiled/package.json

@@ -0,0 +1,23 @@
+{
+    "name": "esm-ts-compiled",
+    "license": "Apache-2.0",
+    "type": "module",
+    "main": "bin",
+    "scripts": {
+        "build": "tsc"
+    },
+    "devDependencies": {
+        "@types/node": "^17.0.5"
+    },
+    "peerDependencies": {
+        "@pulumi/pulumi": "latest"
+    },
+    "dependencies": {
+        "ts-node": "^10.4.0",
+        "typescript": "^4.5.4"
+    },
+    "resolutions": {
+        "ts-node": "^10.4.0",
+        "typescript": "^4.5.4"
+    }
+}

tests/integration/nodejs/esm-ts-compiled/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "outDir": "bin",
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "declaration": true,
+    "sourceMap": true,
+    "stripInternal": true,
+    "experimentalDecorators": true,
+    "pretty": true,
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitReturns": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "files": [
+    "index.ts"
+  ]
+}
+

tests/integration/nodejs/esm-ts-specifier-resolution-node/Pulumi.yaml

@@ -0,0 +1,7 @@
+name: esm-ts-specifier-resolution-node
+runtime: 
+  name: nodejs
+  options: 
+    # See https://github.com/TypeStrong/ts-node/issues/1007
+    nodeargs: "--experimental-specifier-resolution=node --loader ts-node/esm --no-warnings"
+description: Use ECMAScript modules for a TS program.

tests/integration/nodejs/esm-ts-specifier-resolution-node/index.ts

@@ -0,0 +1,10 @@
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
+
+import * as fs from "fs";
+import { x } from "./other";
+
+// Use top-level await
+await new Promise(r => setTimeout(r, 2000));
+
+export const res = fs.readFileSync("Pulumi.yaml").toString();
+export const otherx = x;

tests/integration/nodejs/esm-ts-specifier-resolution-node/other.ts

@@ -0,0 +1 @@
+export const x = 42;

tests/integration/nodejs/esm-ts-specifier-resolution-node/package.json

@@ -0,0 +1,19 @@
+{
+    "name": "esm-ts-specifier-resolution-node",
+    "license": "Apache-2.0",
+    "type": "module",
+    "devDependencies": {
+        "@types/node": "^17.0.5"
+    },
+    "peerDependencies": {
+        "@pulumi/pulumi": "latest"
+    },
+    "dependencies": {
+        "ts-node": "^10.4.0",
+        "typescript": "^4.5.4"
+    },
+    "resolutions": {
+        "ts-node": "^10.4.0",
+        "typescript": "^4.5.4"
+    }
+}

tests/integration/nodejs/esm-ts-specifier-resolution-node/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "outDir": "bin",
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "declaration": true,
+    "sourceMap": true,
+    "stripInternal": true,
+    "experimentalDecorators": true,
+    "pretty": true,
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitReturns": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "files": [
+    "index.ts"
+  ]
+}
+

tests/integration/nodejs/esm-ts/Pulumi.yaml

@@ -0,0 +1,7 @@
+name: esm-ts
+runtime: 
+  name: nodejs
+  options: 
+    # See https://github.com/TypeStrong/ts-node/issues/1007
+    nodeargs: "--loader ts-node/esm --no-warnings"
+description: Use ECMAScript modules for a TS program.

tests/integration/nodejs/esm-ts/index.ts

@@ -0,0 +1,10 @@
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
+
+import * as fs from "fs";
+import { x } from "./other.js"; // this is the "by design" way to do this, even in TS
+
+// Use top-level await
+await new Promise(r => setTimeout(r, 2000));
+
+export const res = fs.readFileSync("Pulumi.yaml").toString();
+export const otherx = x;

tests/integration/nodejs/esm-ts/other.ts

@@ -0,0 +1 @@
+export const x = 42;

tests/integration/nodejs/esm-ts/package.json

@@ -0,0 +1,19 @@
+{
+    "name": "esm-ts",
+    "license": "Apache-2.0",
+    "type": "module",
+    "devDependencies": {
+        "@types/node": "^17.0.5"
+    },
+    "peerDependencies": {
+        "@pulumi/pulumi": "latest"
+    },
+    "dependencies": {
+        "ts-node": "^10.4.0",
+        "typescript": "^4.5.4"
+    },
+    "resolutions": {
+        "ts-node": "^10.4.0",
+        "typescript": "^4.5.4"
+    }
+}

tests/integration/nodejs/esm-ts/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "outDir": "bin",
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "declaration": true,
+    "sourceMap": true,
+    "stripInternal": true,
+    "experimentalDecorators": true,
+    "pretty": true,
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitReturns": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "files": [
+    "index.ts"
+  ]
+}
+