Move InstallDependencies to the language plugin (#9294)

* Move InstallDependencies to the language plugin

This changes `pulumi new` and `pulumi up <template>` to invoke the language plugin to install dependencies, rather than having the code to install dependencies hardcoded into the cli itself.

This does not change the way policypacks or plugin dependencies are installed. In theory we can make pretty much the same change to just invoke the language plugin, but baby steps we don't need to make that change at the same time as this.

We used to feed the result of these install commands (dotnet build, npm install, etc) directly through to the CLI stdout/stderr. To mostly maintain that behaviour the InstallDependencies gRCP method streams back bytes to be written to stdout/stderr, those bytes are either read from pipes or a pty that we run the install commands with. The use of a pty is controlled by the global colorisation option in the cli.

An alternative designs was to use the Engine interface to Log the results of install commands. This renders very differently to just writing directly to the standard outputs and I don't think would support control codes so well.

The design as is means that `npm install` for example is still able to display a progress bar and colors even though we're running it in a separate process and streaming its output back via gRPC.

The only "oddity" I feel that's fallen out of this work is that InstallDependencies for python used to overwrite the virtualenv runtime option. It looks like this was because our templates don't bother setting that. Because InstallDependencies doesn't have the project file, and at any rate will be used for policy pack projects in the future, I've moved that logic into `pulumi new` when it mutates the other project file settings. I think we should at some point cleanup so the templates correctly indicate to use a venv, or maybe change python to assume a virtual env of "venv" if none is given?

* Just warn if pty fails to open

* Add tests and return real tty files

* Add to CHANGELOG

* lint

* format

* Test strings

* Log pty opening for trace debugging

* s/Hack/Workaround

* Use termios

* Tweak terminal test

* lint

* Fix windows build

CHANGELOG_PENDING.md

@@ -1,5 +1,7 @@
 ### Improvements
 
+- [cli] - Installing of language specific project dependencies is now managed by the language plugins, not the pulumi cli.
+  [#9294](https://github.com/pulumi/pulumi/pull/9294)
 
 ### Bug Fixes
 

pkg/cmd/pulumi/new.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -38,15 +37,12 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/util/executable"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/util/goversion"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
-	"github.com/pulumi/pulumi/sdk/v3/nodejs/npm"
-	"github.com/pulumi/pulumi/sdk/v3/python"
 )
 
 type promptForValueFunc func(yes bool, valueType string, defaultValue string, secret bool,
@@ -254,6 +250,14 @@ func runNew(args newArgs) error {
 	proj.Name = tokens.PackageName(args.name)
 	proj.Description = &args.description
 	proj.Template = nil
+	// Workaround for python, most of our templates don't specify a venv but we want to use one
+	if proj.Runtime.Name() == "python" {
+		// If the template does give virtualenv use it, else default to "venv"
+		if _, has := proj.Runtime.Options()["virtualenv"]; !has {
+			proj.Runtime.SetOption("virtualenv", "venv")
+		}
+	}
+
 	if err = workspace.SaveProject(proj); err != nil {
 		return fmt.Errorf("saving project: %w", err)
 	}
@@ -283,7 +287,13 @@ func runNew(args newArgs) error {
 
 	// Install dependencies.
 	if !args.generateOnly {
-		if err := installDependencies(proj, root); err != nil {
+		projinfo := &engine.Projinfo{Proj: proj, Root: root}
+		pwd, _, ctx, err := engine.ProjectInfoContext(projinfo, nil, nil, cmdutil.Diag(), cmdutil.Diag(), false, nil)
+		if err != nil {
+			return err
+		}
+
+		if err := installDependencies(ctx, &proj.Runtime, pwd); err != nil {
 			return err
 		}
 	}
@@ -593,113 +603,19 @@ func saveConfig(stack backend.Stack, c config.Map) error {
 }
 
 // installDependencies will install dependencies for the project, e.g. by running `npm install` for nodejs projects.
-func installDependencies(proj *workspace.Project, root string) error {
-	// TODO[pulumi/pulumi#1334]: move to the language plugins so we don't have to hard code here.
-	if strings.EqualFold(proj.Runtime.Name(), "nodejs") {
-		if bin, err := nodeInstallDependencies(); err != nil {
-			return fmt.Errorf("%s install failed; rerun manually to try again, "+
-				"then run 'pulumi up' to perform an initial deployment"+": %w", bin, err)
-
-		}
-	} else if strings.EqualFold(proj.Runtime.Name(), "python") {
-		return pythonInstallDependencies(proj, root)
-	} else if strings.EqualFold(proj.Runtime.Name(), "dotnet") {
-		return dotnetInstallDependenciesAndBuild(proj, root)
-	} else if strings.EqualFold(proj.Runtime.Name(), "go") {
-		if err := goInstallDependencies(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// nodeInstallDependencies will install dependencies for the project or Policy Pack by running `npm install` or
-// `yarn install`.
-func nodeInstallDependencies() (string, error) {
-	fmt.Println("Installing dependencies...")
-	fmt.Println()
-
-	bin, err := npm.Install("", false /*production*/, os.Stdout, os.Stderr)
-	if err != nil {
-		return bin, err
-	}
-
-	fmt.Println("Finished installing dependencies")
-	fmt.Println()
-
-	return bin, nil
-}
-
-// pythonInstallDependencies will create a new virtual environment and install dependencies.
-func pythonInstallDependencies(proj *workspace.Project, root string) error {
-	const venvDir = "venv"
-	if err := python.InstallDependencies(root, venvDir, true /*showOutput*/); err != nil {
-		return err
-	}
-
-	// Save project with venv info.
-	proj.Runtime.SetOption("virtualenv", venvDir)
-	if err := workspace.SaveProject(proj); err != nil {
-		return fmt.Errorf("saving project: %w", err)
-	}
-	return nil
-}
-
-// dotnetInstallDependenciesAndBuild will install dependencies and build the project.
-func dotnetInstallDependenciesAndBuild(proj *workspace.Project, root string) error {
-	contract.Assert(proj != nil)
-
-	fmt.Println("Installing dependencies...")
-	fmt.Println()
-
-	projinfo := &engine.Projinfo{Proj: proj, Root: root}
-	pwd, main, plugctx, err := engine.ProjectInfoContext(projinfo, nil, nil, cmdutil.Diag(), cmdutil.Diag(), false, nil)
+func installDependencies(ctx *plugin.Context, runtime *workspace.ProjectRuntimeInfo, directory string) error {
+	// First make sure the language plugin is present.  We need this to load the required resource plugins.
+	// TODO: we need to think about how best to version this.  For now, it always picks the latest.
+	lang, err := ctx.Host.LanguageRuntime(runtime.Name())
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to load language plugin %s: %w", runtime.Name(), err)
 	}
-	defer plugctx.Close()
 
-	// Call RunInstallPlugins, which will run `dotnet build`. This will automatically
-	// restore dependencies, install any plugins, and build the project so it will be
-	// prepped and ready to go for a faster initial `pulumi up`.
-	if err = engine.RunInstallPlugins(proj, pwd, main, nil, plugctx); err != nil {
-		return err
+	if err = lang.InstallDependencies(directory); err != nil {
+		return fmt.Errorf("installing dependencies failed; rerun manually to try again, "+
+			"then run 'pulumi up' to perform an initial deployment: %w", err)
 	}
 
-	fmt.Println("Finished installing dependencies")
-	fmt.Println()
-
-	return nil
-}
-
-// goInstallDependencies will install dependencies for the project by running `go mod tidy`.
-func goInstallDependencies() error {
-	fmt.Println("Installing dependencies...")
-	fmt.Println()
-
-	gobin, err := executable.FindExecutable("go")
-	if err != nil {
-		return err
-	}
-
-	if err = goversion.CheckMinimumGoVersion(gobin); err != nil {
-		return err
-	}
-
-	cmd := exec.Command(gobin, "mod", "tidy")
-	cmd.Env = os.Environ()
-	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
-
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("`go mod tidy` failed to install dependencies; rerun manually to try again, "+
-			"then run 'pulumi up' to perform an initial deployment"+": %w", err)
-
-	}
-
-	fmt.Println("Finished installing dependencies")
-	fmt.Println()
-
 	return nil
 }
 

pkg/cmd/pulumi/policy_new.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
+	"github.com/pulumi/pulumi/sdk/v3/nodejs/npm"
 	"github.com/pulumi/pulumi/sdk/v3/python"
 	"github.com/spf13/cobra"
 	survey "gopkg.in/AlecAivazis/survey.v1"
@@ -189,9 +190,16 @@ func runNewPolicyPack(args newPolicyArgs) error {
 func installPolicyPackDependencies(proj *workspace.PolicyPackProject, projPath, root string) error {
 	// TODO[pulumi/pulumi#1334]: move to the language plugins so we don't have to hard code here.
 	if strings.EqualFold(proj.Runtime.Name(), "nodejs") {
-		if bin, err := nodeInstallDependencies(); err != nil {
+		fmt.Println("Installing dependencies...")
+		fmt.Println()
+
+		bin, err := npm.Install("", false /*production*/, os.Stdout, os.Stderr)
+		if err != nil {
 			return fmt.Errorf("`%s install` failed; rerun manually to try again.: %w", bin, err)
 		}
+
+		fmt.Println("Finished installing dependencies")
+		fmt.Println()
 	} else if strings.EqualFold(proj.Runtime.Name(), "python") {
 		const venvDir = "venv"
 		if err := python.InstallDependencies(root, venvDir, true /*showOutput*/); err != nil {

pkg/cmd/pulumi/up.go

@@ -307,7 +307,14 @@ func newUpCmd() *cobra.Command {
 		}
 
 		// Install dependencies.
-		if err = installDependencies(proj, root); err != nil {
+
+		projinfo := &engine.Projinfo{Proj: proj, Root: root}
+		pwd, _, ctx, err := engine.ProjectInfoContext(projinfo, nil, nil, cmdutil.Diag(), cmdutil.Diag(), false, nil)
+		if err != nil {
+			return result.FromError(fmt.Errorf("building project context: %w", err))
+		}
+
+		if err = installDependencies(ctx, &proj.Runtime, pwd); err != nil {
 			return result.FromError(err)
 		}
 

pkg/engine/lifeycletest/pulumi_test.go

@@ -2293,6 +2293,12 @@ func (ctx *updateContext) GetPluginInfo(_ context.Context, req *pbempty.Empty) (
 	}, nil
 }
 
+func (ctx *updateContext) InstallDependencies(
+	req *pulumirpc.InstallDependenciesRequest,
+	server pulumirpc.LanguageRuntime_InstallDependenciesServer) error {
+	return nil
+}
+
 func TestLanguageClient(t *testing.T) {
 	t.Parallel()
 

pkg/go.mod

@@ -147,7 +147,7 @@ require (
 	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 // indirect
-	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
 	golang.org/x/tools v0.1.0 // indirect
@@ -162,5 +162,6 @@ require (
 
 require (
 	github.com/hashicorp/go-version v1.4.0 // indirect
+	github.com/pkg/term v1.1.0 // indirect
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
 )

pkg/go.sum

@@ -540,6 +540,8 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/term v1.1.0 h1:xIAAdCMh3QIAy+5FrE8Ad8XoDhEU4ufwbaSozViP9kk=
+github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -840,6 +842,7 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -856,14 +859,16 @@ golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/resource/deploy/deploytest/languageruntime.go

@@ -65,3 +65,7 @@ func (p *languageRuntime) Run(info plugin.RunInfo) (string, bool, error) {
 func (p *languageRuntime) GetPluginInfo() (workspace.PluginInfo, error) {
 	return workspace.PluginInfo{Name: "TestLanguage"}, nil
 }
+
+func (p *languageRuntime) InstallDependencies(directory string) error {
+	return nil
+}

sdk/dotnet/cmd/pulumi-language-dotnet/main.go

@@ -33,6 +33,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/executable"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/version"
@@ -660,3 +661,38 @@ func (host *dotnetLanguageHost) GetPluginInfo(ctx context.Context, req *pbempty.
 		Version: version.Version,
 	}, nil
 }
+
+func (host *dotnetLanguageHost) InstallDependencies(
+	req *pulumirpc.InstallDependenciesRequest, server pulumirpc.LanguageRuntime_InstallDependenciesServer) error {
+
+	closer, stdout, stderr, err := rpcutil.MakeStreams(server, req.IsTerminal)
+	if err != nil {
+		return err
+	}
+	// best effort close, but we try an explicit close and error check at the end as well
+	defer closer.Close()
+
+	stdout.Write([]byte("Installing dependencies...\n\n"))
+
+	dotnetbin, err := executable.FindExecutable("dotnet")
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.Command(dotnetbin, "build")
+	cmd.Dir = req.Directory
+	cmd.Env = os.Environ()
+	cmd.Stdout, cmd.Stderr = stdout, stderr
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("`dotnet build` failed to install dependencies: %w", err)
+
+	}
+	stdout.Write([]byte("Finished installing dependencies\n\n"))
+
+	if err := closer.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}

sdk/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/Microsoft/go-winio v0.4.14
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/cheggaaa/pb v1.0.18
+	github.com/creack/pty v1.1.9 // indirect
 	github.com/djherbis/times v1.2.0
 	github.com/gofrs/uuid v3.3.0+incompatible
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
@@ -39,6 +40,12 @@ require (
 	sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0
 )
 
+require (
+	github.com/kr/pty v1.1.8
+	github.com/pkg/term v1.1.0
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+)
+
 require (
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect