[langhost/node] More ESM support: nodeargs, main entrypoints, and tests

CHANGELOG_PENDING.md

@@ -1,5 +1,11 @@
 ### Improvements
 
+- [sdk/nodejs] Support using native ES modules as Pulumi scripts
+  [#7764](https://github.com/pulumi/pulumi/pull/7764)
+
+- [sdk/nodejs] Support a `nodeargs` option for passing `node` arguments to the Node language host
+  [#8655](https://github.com/pulumi/pulumi/pull/8655)
+
 ### Bug Fixes
 
 - [cli/engine] - Fix [#3982](https://github.com/pulumi/pulumi/issues/3982), a bug

sdk/go.mod

@@ -8,67 +8,68 @@ require (
 	github.com/Microsoft/go-winio v0.4.14
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/cheggaaa/pb v1.0.18
-	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/djherbis/times v1.2.0
-	github.com/fatih/color v1.9.0 // indirect
 	github.com/gofrs/uuid v3.3.0+incompatible
-	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.4.2
-	github.com/google/go-cmp v0.4.1 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/kr/pretty v0.2.1 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/mattn/go-colorable v0.1.6 // indirect
-	github.com/mattn/go-runewidth v0.0.8 // indirect
 	github.com/mitchellh/go-ps v1.0.0
 	github.com/nxadm/tail v1.4.8
-	github.com/opentracing/basictracer-go v1.0.0 // indirect
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/rivo/uniseg v0.2.0
 	github.com/rogpeppe/go-internal v1.8.1
 	github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94
-	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/spf13/cast v1.3.1
 	github.com/spf13/cobra v1.0.0
 	github.com/stretchr/testify v1.6.1
 	github.com/texttheater/golang-levenshtein v0.0.0-20191208221605-eb6844b05fc6
 	github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7
 	github.com/uber/jaeger-client-go v2.22.1+incompatible
-	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
-	go.uber.org/atomic v1.6.0 // indirect
 	golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/mod v0.3.0
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
 	golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2
-	golang.org/x/tools v0.0.0-20200608174601-1b747fd94509 // indirect
-	google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 // indirect
 	google.golang.org/grpc v1.29.1
-	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v2 v2.2.8
 	pgregory.net/rapid v0.4.7
 	sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0
 )
 
 require (
+	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/fatih/color v1.9.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/gogo/protobuf v1.3.1 // indirect
+	github.com/google/go-cmp v0.4.1 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
+	github.com/kr/pretty v0.2.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mattn/go-colorable v0.1.6 // indirect
+	github.com/mattn/go-runewidth v0.0.8 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/opentracing/basictracer-go v1.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.3 // indirect
 	github.com/src-d/gcfg v1.4.0 // indirect
+	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
 	github.com/xanzy/ssh-agent v0.2.1 // indirect
+	go.uber.org/atomic v1.6.0 // indirect
+	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/text v0.3.2 // indirect
+	golang.org/x/tools v0.0.0-20200608174601-1b747fd94509 // indirect
+	google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 // indirect
 	google.golang.org/protobuf v1.24.0 // indirect
+	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect

sdk/go.sum

@@ -87,6 +87,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=

sdk/nodejs/cmd/pulumi-language-nodejs/main.go

@@ -38,12 +38,14 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/hashicorp/go-multierror"
-
+	"github.com/blang/semver"
 	pbempty "github.com/golang/protobuf/ptypes/empty"
+	"github.com/google/shlex"
+	"github.com/hashicorp/go-multierror"
 	opentracing "github.com/opentracing/opentracing-go"
-
 	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
@@ -52,9 +54,6 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/version"
 	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
-	"google.golang.org/grpc"
-
-	"github.com/blang/semver"
 )
 
 const (
@@ -83,13 +82,15 @@ func main() {
 	var typescript bool
 	var root string
 	var tsconfigpath string
+	var nodeargs string
 	flag.StringVar(&tracing, "tracing", "",
 		"Emit tracing to a Zipkin-compatible tracing endpoint")
 	flag.BoolVar(&typescript, "typescript", true,
 		"Use ts-node at runtime to support typescript source natively")
 	flag.StringVar(&root, "root", "", "Project root path to use")
 	flag.StringVar(&tsconfigpath, "tsconfig", "",
 		"Path to tsconfig.json to use")
+	flag.StringVar(&nodeargs, "nodeargs", "", "Arguments for the Node process")
 	flag.Parse()
 
 	args := flag.Args()
@@ -121,7 +122,7 @@ func main() {
 	// Fire up a gRPC server, letting the kernel choose a free port.
 	port, done, err := rpcutil.Serve(0, nil, []func(*grpc.Server) error{
 		func(srv *grpc.Server) error {
-			host := newLanguageHost(nodePath, runPath, engineAddress, tracing, typescript, tsconfigpath)
+			host := newLanguageHost(nodePath, runPath, engineAddress, tracing, typescript, tsconfigpath, nodeargs)
 			pulumirpc.RegisterLanguageRuntimeServer(srv, host)
 			return nil
 		},
@@ -159,17 +160,19 @@ type nodeLanguageHost struct {
 	tracing       string
 	typescript    bool
 	tsconfigpath  string
+	nodeargs      string
 }
 
 func newLanguageHost(nodePath, runPath, engineAddress,
-	tracing string, typescript bool, tsconfigpath string) pulumirpc.LanguageRuntimeServer {
+	tracing string, typescript bool, tsconfigpath string, nodeargs string) pulumirpc.LanguageRuntimeServer {
 	return &nodeLanguageHost{
 		nodeBin:       nodePath,
 		runPath:       runPath,
 		engineAddress: engineAddress,
 		tracing:       tracing,
 		typescript:    typescript,
 		tsconfigpath:  tsconfigpath,
+		nodeargs:      nodeargs,
 	}
 }
 
@@ -513,15 +516,21 @@ func (host *nodeLanguageHost) execNodejs(
 			env = append(env, "PULUMI_NODEJS_TSCONFIG_PATH="+host.tsconfigpath)
 		}
 
+		nodeargs, err := shlex.Split(host.nodeargs)
+		if err != nil {
+			return &pulumirpc.RunResponse{Error: err.Error()}
+		}
+		nodeargs = append(nodeargs, args...)
+
 		if logging.V(5) {
-			commandStr := strings.Join(args, " ")
+			commandStr := strings.Join(nodeargs, " ")
 			logging.V(5).Infoln("Language host launching process: ", host.nodeBin, commandStr)
 		}
 
 		// Now simply spawn a process to execute the requested program, wiring up stdout/stderr directly.
 		var errResult string
 		// #nosec G204
-		cmd := exec.Command(host.nodeBin, args...)
+		cmd := exec.Command(host.nodeBin, nodeargs...)
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
 		cmd.Env = env

sdk/nodejs/cmd/run/run.ts

@@ -260,6 +260,10 @@ ${defaultMessage}`);
 
             // We use dynamic import instead of require for projects using native ES modules instead of commonjs
             if (packageObject["type"] === "module") {
+                // Use the same behavior for loading the main entrypoint as `node <program>`.
+                // See https://github.com/nodejs/node/blob/master/lib/internal/modules/run_main.js#L74.
+                const mainPath: string = require("module").Module._findPath(path.resolve(program), null, true) || program;
+                const main = path.isAbsolute(mainPath) ? url.pathToFileURL(mainPath).href : mainPath;
                 // Workaround for typescript transpiling dynamic import into `Promise.resolve().then(() => require`
                 // Follow this issue for progress on when we can remove this:
                 // https://github.com/microsoft/TypeScript/issues/43329
@@ -271,7 +275,7 @@ ${defaultMessage}`);
                 // Import the module and capture any module outputs it exported. Finally, await the value we get
                 // back.  That way, if it is async and throws an exception, we properly capture it here
                 // and handle it.
-                return await dynamicImport(url.pathToFileURL(program).toString());
+                return await dynamicImport(main);
             }
 
             // Execute the module and capture any module outputs it exported. If the exported value

tests/integration/integration_nodejs_test.go

@@ -1152,6 +1152,47 @@ func TestCompilerOptionsNode(t *testing.T) {
 	})
 }
 
+func TestESMJS(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-js"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		Quick:        true,
+	})
+}
+
+func TestESMJSMain(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-js-main"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		Quick:        true,
+	})
+}
+
+func TestESMTS(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-ts"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		Quick:        true,
+	})
+}
+
+func TestESMTSSpecifierResolutionNode(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-ts-specifier-resolution-node"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		Quick:        true,
+	})
+}
+
+func TestESMTSCompiled(t *testing.T) {
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir:          filepath.Join("nodejs", "esm-ts-compiled"),
+		Dependencies: []string{"@pulumi/pulumi"},
+		RunBuild:     true,
+		Quick:        true,
+	})
+}
+
 // Test that the about command works as expected. Because about parses the
 // results of each runtime independently, we have an integration test in each
 // language.

tests/integration/nodejs/.gitignore

@@ -0,0 +1,4 @@
+.pulumi/
+bin/
+node_modules/
+Pulumi.*.yaml

tests/integration/nodejs/esm-js-main/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: esm-js-main
+runtime: nodejs
+description: Use ECMAScript modules for a plain JS program.

tests/integration/nodejs/esm-js-main/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "esm-js-main",
+    "license": "Apache-2.0",
+    "type": "module",
+    "main": "src/main.js",
+    "peerDependencies": {
+        "@pulumi/pulumi": "latest"
+    },
+    "dependencies": {
+        "@pulumi/aws": "^4.33.0"
+    }
+}

tests/integration/nodejs/esm-js-main/src/main.js

@@ -0,0 +1,8 @@
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
+
+import * as fs from "fs";
+
+// Use top-level await
+await new Promise(r => setTimeout(r, 2000));
+
+export const res = fs.readFileSync("Pulumi.yaml").toString();

tests/integration/nodejs/esm-js/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: esm-js
+runtime: nodejs
+description: Use ECMAScript modules for a plain JS program.

tests/integration/nodejs/esm-js/index.js

@@ -0,0 +1,5 @@
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
+
+import * as fs from "fs";
+
+export const res = fs.readFileSync("Pulumi.yaml").toString();

tests/integration/nodejs/esm-js/package.json

@@ -0,0 +1,8 @@
+{
+    "name": "esm-js",
+    "license": "Apache-2.0",
+    "type": "module",
+    "peerDependencies": {
+        "@pulumi/pulumi": "latest"
+    }
+}

tests/integration/nodejs/esm-ts-compiled/Pulumi.yaml

@@ -0,0 +1,6 @@
+name: esm-ts-compiled
+runtime: 
+  name: nodejs
+  options: 
+    typescript: false
+description: Use ECMAScript modules for a TS program.

tests/integration/nodejs/esm-ts-compiled/index.ts

@@ -0,0 +1,10 @@
+// Copyright 2016-2021, Pulumi Corporation.  All rights reserved.
+
+import * as fs from "fs";
+import { x } from "./other.js"; // this is the "by design" way to do this, even in TS.
+
+// Use top-level await
+await new Promise(r => setTimeout(r, 2000));
+
+export const res = fs.readFileSync("Pulumi.yaml").toString();
+export const otherx = x;

tests/integration/nodejs/esm-ts-compiled/other.ts

@@ -0,0 +1 @@
+export const x = 42;

tests/integration/nodejs/esm-ts-compiled/package.json

@@ -0,0 +1,23 @@
+{
+    "name": "esm-ts-compiled",
+    "license": "Apache-2.0",
+    "type": "module",
+    "main": "bin",
+    "scripts": {
+        "build": "tsc"
+    },
+    "devDependencies": {
+        "@types/node": "^17.0.5"
+    },
+    "peerDependencies": {
+        "@pulumi/pulumi": "latest"
+    },
+    "dependencies": {
+        "ts-node": "^10.4.0",
+        "typescript": "^4.5.4"
+    },
+    "resolutions": {
+        "ts-node": "^10.4.0",
+        "typescript": "^4.5.4"
+    }
+}

tests/integration/nodejs/esm-ts-compiled/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "outDir": "bin",
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "declaration": true,
+    "sourceMap": true,
+    "stripInternal": true,
+    "experimentalDecorators": true,
+    "pretty": true,
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitReturns": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "files": [
+    "index.ts"
+  ]
+}
+