Move InstallDependencies to the language plugin

CHANGELOG_PENDING.md

@@ -3,6 +3,8 @@
 - Clear pending operations during `pulumi refresh` or `pulumi up -r`.
   [#8435](https://github.com/pulumi/pulumi/pull/8435)
 
+- [cli] - Installing of language specific project dependencies is now managed by the language plugins, not the pulumi cli.
+  [#9294](https://github.com/pulumi/pulumi/pull/9294)
 ### Bug Fixes
 
 - [codegen/go] - Fix Go SDK function output to check for errors

pkg/cmd/pulumi/new.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -38,15 +37,12 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/config"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/util/executable"
-	"github.com/pulumi/pulumi/sdk/v3/go/common/util/goversion"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
-	"github.com/pulumi/pulumi/sdk/v3/nodejs/npm"
-	"github.com/pulumi/pulumi/sdk/v3/python"
 )
 
 type promptForValueFunc func(yes bool, valueType string, defaultValue string, secret bool,
@@ -254,6 +250,14 @@ func runNew(args newArgs) error {
 	proj.Name = tokens.PackageName(args.name)
 	proj.Description = &args.description
 	proj.Template = nil
+	// Hack for python, most of our templates don't specify a venv but we want to use one
+	if proj.Runtime.Name() == "python" {
+		// If the template does give virtualenv use it, else default to "venv"
+		if _, has := proj.Runtime.Options()["virtualenv"]; !has {
+			proj.Runtime.SetOption("virtualenv", "venv")
+		}
+	}
+
 	if err = workspace.SaveProject(proj); err != nil {
 		return fmt.Errorf("saving project: %w", err)
 	}
@@ -283,7 +287,13 @@ func runNew(args newArgs) error {
 
 	// Install dependencies.
 	if !args.generateOnly {
-		if err := installDependencies(proj, root); err != nil {
+		projinfo := &engine.Projinfo{Proj: proj, Root: root}
+		pwd, _, ctx, err := engine.ProjectInfoContext(projinfo, nil, nil, cmdutil.Diag(), cmdutil.Diag(), false, nil)
+		if err != nil {
+			return err
+		}
+
+		if err := installDependencies(ctx, &proj.Runtime, pwd); err != nil {
 			return err
 		}
 	}
@@ -593,113 +603,19 @@ func saveConfig(stack backend.Stack, c config.Map) error {
 }
 
 // installDependencies will install dependencies for the project, e.g. by running `npm install` for nodejs projects.
-func installDependencies(proj *workspace.Project, root string) error {
-	// TODO[pulumi/pulumi#1334]: move to the language plugins so we don't have to hard code here.
-	if strings.EqualFold(proj.Runtime.Name(), "nodejs") {
-		if bin, err := nodeInstallDependencies(); err != nil {
-			return fmt.Errorf("%s install failed; rerun manually to try again, "+
-				"then run 'pulumi up' to perform an initial deployment"+": %w", bin, err)
-
-		}
-	} else if strings.EqualFold(proj.Runtime.Name(), "python") {
-		return pythonInstallDependencies(proj, root)
-	} else if strings.EqualFold(proj.Runtime.Name(), "dotnet") {
-		return dotnetInstallDependenciesAndBuild(proj, root)
-	} else if strings.EqualFold(proj.Runtime.Name(), "go") {
-		if err := goInstallDependencies(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// nodeInstallDependencies will install dependencies for the project or Policy Pack by running `npm install` or
-// `yarn install`.
-func nodeInstallDependencies() (string, error) {
-	fmt.Println("Installing dependencies...")
-	fmt.Println()
-
-	bin, err := npm.Install("", false /*production*/, os.Stdout, os.Stderr)
-	if err != nil {
-		return bin, err
-	}
-
-	fmt.Println("Finished installing dependencies")
-	fmt.Println()
-
-	return bin, nil
-}
-
-// pythonInstallDependencies will create a new virtual environment and install dependencies.
-func pythonInstallDependencies(proj *workspace.Project, root string) error {
-	const venvDir = "venv"
-	if err := python.InstallDependencies(root, venvDir, true /*showOutput*/); err != nil {
-		return err
-	}
-
-	// Save project with venv info.
-	proj.Runtime.SetOption("virtualenv", venvDir)
-	if err := workspace.SaveProject(proj); err != nil {
-		return fmt.Errorf("saving project: %w", err)
-	}
-	return nil
-}
-
-// dotnetInstallDependenciesAndBuild will install dependencies and build the project.
-func dotnetInstallDependenciesAndBuild(proj *workspace.Project, root string) error {
-	contract.Assert(proj != nil)
-
-	fmt.Println("Installing dependencies...")
-	fmt.Println()
-
-	projinfo := &engine.Projinfo{Proj: proj, Root: root}
-	pwd, main, plugctx, err := engine.ProjectInfoContext(projinfo, nil, nil, cmdutil.Diag(), cmdutil.Diag(), false, nil)
+func installDependencies(ctx *plugin.Context, runtime *workspace.ProjectRuntimeInfo, directory string) error {
+	// First make sure the language plugin is present.  We need this to load the required resource plugins.
+	// TODO: we need to think about how best to version this.  For now, it always picks the latest.
+	lang, err := ctx.Host.LanguageRuntime(runtime.Name())
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to load language plugin %s: %w", runtime.Name(), err)
 	}
-	defer plugctx.Close()
 
-	// Call RunInstallPlugins, which will run `dotnet build`. This will automatically
-	// restore dependencies, install any plugins, and build the project so it will be
-	// prepped and ready to go for a faster initial `pulumi up`.
-	if err = engine.RunInstallPlugins(proj, pwd, main, nil, plugctx); err != nil {
-		return err
+	if err = lang.InstallDependencies(directory); err != nil {
+		return fmt.Errorf("installing dependencies failed; rerun manually to try again, "+
+			"then run 'pulumi up' to perform an initial deployment: %w", err)
 	}
 
-	fmt.Println("Finished installing dependencies")
-	fmt.Println()
-
-	return nil
-}
-
-// goInstallDependencies will install dependencies for the project by running `go mod tidy`.
-func goInstallDependencies() error {
-	fmt.Println("Installing dependencies...")
-	fmt.Println()
-
-	gobin, err := executable.FindExecutable("go")
-	if err != nil {
-		return err
-	}
-
-	if err = goversion.CheckMinimumGoVersion(gobin); err != nil {
-		return err
-	}
-
-	cmd := exec.Command(gobin, "mod", "tidy")
-	cmd.Env = os.Environ()
-	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
-
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("`go mod tidy` failed to install dependencies; rerun manually to try again, "+
-			"then run 'pulumi up' to perform an initial deployment"+": %w", err)
-
-	}
-
-	fmt.Println("Finished installing dependencies")
-	fmt.Println()
-
 	return nil
 }
 

pkg/cmd/pulumi/policy_new.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
+	"github.com/pulumi/pulumi/sdk/v3/nodejs/npm"
 	"github.com/pulumi/pulumi/sdk/v3/python"
 	"github.com/spf13/cobra"
 	survey "gopkg.in/AlecAivazis/survey.v1"
@@ -189,9 +190,16 @@ func runNewPolicyPack(args newPolicyArgs) error {
 func installPolicyPackDependencies(proj *workspace.PolicyPackProject, projPath, root string) error {
 	// TODO[pulumi/pulumi#1334]: move to the language plugins so we don't have to hard code here.
 	if strings.EqualFold(proj.Runtime.Name(), "nodejs") {
-		if bin, err := nodeInstallDependencies(); err != nil {
+		fmt.Println("Installing dependencies...")
+		fmt.Println()
+
+		bin, err := npm.Install("", false /*production*/, os.Stdout, os.Stderr)
+		if err != nil {
 			return fmt.Errorf("`%s install` failed; rerun manually to try again.: %w", bin, err)
 		}
+
+		fmt.Println("Finished installing dependencies")
+		fmt.Println()
 	} else if strings.EqualFold(proj.Runtime.Name(), "python") {
 		const venvDir = "venv"
 		if err := python.InstallDependencies(root, venvDir, true /*showOutput*/); err != nil {

pkg/cmd/pulumi/up.go

@@ -307,7 +307,14 @@ func newUpCmd() *cobra.Command {
 		}
 
 		// Install dependencies.
-		if err = installDependencies(proj, root); err != nil {
+
+		projinfo := &engine.Projinfo{Proj: proj, Root: root}
+		pwd, _, ctx, err := engine.ProjectInfoContext(projinfo, nil, nil, cmdutil.Diag(), cmdutil.Diag(), false, nil)
+		if err != nil {
+			return result.FromError(fmt.Errorf("building project context: %w", err))
+		}
+
+		if err = installDependencies(ctx, &proj.Runtime, pwd); err != nil {
 			return result.FromError(err)
 		}
 

pkg/engine/lifeycletest/pulumi_test.go

@@ -2297,6 +2297,12 @@ func (ctx *updateContext) GetPluginInfo(_ context.Context, req *pbempty.Empty) (
 	}, nil
 }
 
+func (ctx *updateContext) InstallDependencies(
+	req *pulumirpc.InstallDependenciesRequest,
+	server pulumirpc.LanguageRuntime_InstallDependenciesServer) error {
+	return nil
+}
+
 func TestLanguageClient(t *testing.T) {
 	t.Parallel()
 

pkg/go.mod

@@ -148,7 +148,7 @@ require (
 	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 // indirect
-	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
 	golang.org/x/tools v0.1.0 // indirect
@@ -162,6 +162,7 @@ require (
 )
 
 require (
+	github.com/creack/pty v1.1.11 // indirect
 	github.com/hashicorp/go-version v1.4.0 // indirect
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
 )

pkg/go.sum

@@ -898,14 +898,16 @@ golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/resource/deploy/deploytest/languageruntime.go

@@ -65,3 +65,7 @@ func (p *languageRuntime) Run(info plugin.RunInfo) (string, bool, error) {
 func (p *languageRuntime) GetPluginInfo() (workspace.PluginInfo, error) {
 	return workspace.PluginInfo{Name: "TestLanguage"}, nil
 }
+
+func (p *languageRuntime) InstallDependencies(directory string) error {
+	return nil
+}

sdk/dotnet/cmd/pulumi-language-dotnet/main.go

@@ -33,6 +33,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/executable"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/version"
@@ -660,3 +661,38 @@ func (host *dotnetLanguageHost) GetPluginInfo(ctx context.Context, req *pbempty.
 		Version: version.Version,
 	}, nil
 }
+
+func (host *dotnetLanguageHost) InstallDependencies(
+	req *pulumirpc.InstallDependenciesRequest, server pulumirpc.LanguageRuntime_InstallDependenciesServer) error {
+
+	closer, stdout, stderr, err := rpcutil.MakeStreams(server, req.IsTerminal)
+	if err != nil {
+		return err
+	}
+	// best effort close, but we try an explicit close and error check at the end as well
+	defer closer.Close()
+
+	stdout.Write([]byte("Installing dependencies...\n\n"))
+
+	dotnetbin, err := executable.FindExecutable("dotnet")
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.Command(dotnetbin, "build")
+	cmd.Dir = req.Directory
+	cmd.Env = os.Environ()
+	cmd.Stdout, cmd.Stderr = stdout, stderr
+
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("`dotnet build` failed to install dependencies: %w", err)
+
+	}
+	stdout.Write([]byte("Finished installing dependencies\n\n"))
+
+	if err := closer.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}

sdk/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/Microsoft/go-winio v0.4.14
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/cheggaaa/pb v1.0.18
+	github.com/creack/pty v1.1.9
 	github.com/djherbis/times v1.2.0
 	github.com/gofrs/uuid v3.3.0+incompatible
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
@@ -39,6 +40,8 @@ require (
 	sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0
 )
 
+require golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+
 require (
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect

sdk/go.sum

@@ -31,6 +31,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -276,8 +277,11 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

sdk/go/auto/stack.go

@@ -971,6 +971,12 @@ func (s *languageRuntimeServer) GetPluginInfo(ctx context.Context, req *pbempty.
 	}, nil
 }
 
+func (s *languageRuntimeServer) InstallDependencies(
+	req *pulumirpc.InstallDependenciesRequest,
+	server pulumirpc.LanguageRuntime_InstallDependenciesServer) error {
+	return nil
+}
+
 func tailLogs(command string, receivers []chan<- events.EngineEvent) (*tail.Tail, error) {
 	logDir, err := ioutil.TempDir("", fmt.Sprintf("automation-logs-%s-", command))
 	if err != nil {

sdk/go/common/resource/plugin/langruntime.go

@@ -38,6 +38,9 @@ type LanguageRuntime interface {
 	Run(info RunInfo) (string, bool, error)
 	// GetPluginInfo returns this plugin's information.
 	GetPluginInfo() (workspace.PluginInfo, error)
+
+	// InstallDependencies will install dependencies for the project, e.g. by running `npm install` for nodejs projects.
+	InstallDependencies(directory string) error
 }
 
 // ProgInfo contains minimal information about the program to be run.