Merge #11168 #11170 #11174

11168: [auto/go] Support for remote operations r=justinvp a=justinvp

This change adds preview support for remote operations in Go's Automation API.

Here's an example of using it:

```go
package main

import (
	"context"
	"fmt"
	"os"

	"github.com/pulumi/pulumi/sdk/v3/go/auto"
	"github.com/pulumi/pulumi/sdk/v3/go/auto/optremotedestroy"
	"github.com/pulumi/pulumi/sdk/v3/go/auto/optremoteup"
)

func main() {
	// to destroy our program, we can run `go run main.go destroy`
	var preview, destroy, refresh bool
	argsWithoutProg := os.Args[1:]
	destroy := len(argsWithoutProg) > 0 && argsWithoutProg[0] == "destroy"
	ctx := context.Background()

	org := "justinvp"
	projectName := "aws-ts-s3-folder"
	stackName := auto.FullyQualifiedStackName(org, projectName, "devgo")

	repo := auto.GitRepo{
		URL:         "https://github.com/pulumi/examples.git",
		Branch:      "refs/heads/master",
		ProjectPath: projectName,
	}

	env := map[string]auto.EnvVarValue{
		"AWS_REGION":            {Value: "us-west-2"},
		"AWS_ACCESS_KEY_ID":     {Value: os.Getenv("AWS_ACCESS_KEY_ID")},
		"AWS_SECRET_ACCESS_KEY": {Value: os.Getenv("AWS_SECRET_ACCESS_KEY"), Secret: true},
		"AWS_SESSION_TOKEN":     {Value: os.Getenv("AWS_SESSION_TOKEN"), Secret: true},
	}

	s, err := auto.UpsertRemoteStackGitSource(ctx, stackName, repo, auto.RemoteEnvVars(env))
	if err != nil {
		fmt.Printf("Failed to create or select stack: %v\n", err)
		os.Exit(1)
	}

	if destroy {
		stdoutStreamer := optremotedestroy.ProgressStreams(os.Stdout)
		_, err := s.Destroy(ctx, stdoutStreamer)
		if err != nil {
			fmt.Printf("Failed to destroy stack: %v", err)
		}
		fmt.Println("Stack successfully destroyed")
		os.Exit(0)
	}


	stdoutStreamer := optremoteup.ProgressStreams(os.Stdout)
	res, err := s.Up(ctx, stdoutStreamer)
	if err != nil {
		fmt.Printf("Failed to update stack: %v\n\n", err)
		os.Exit(1)
	}

	fmt.Println("Update succeeded!")

	url, ok := res.Outputs["websiteUrl"].Value.(string)
	if !ok {
		fmt.Println("Failed to unmarshall output URL")
		os.Exit(1)
	}
	fmt.Printf("URL: %s\n", url)
}
```

I will add sanity tests subsequently.

11170: [auto/nodejs] Support for remote operations r=justinvp a=justinvp

This change adds preview support for remote operations in Node.js's Automation API.

Here's an example of using it:

```ts
import * as process from "process";
import { RemoteWorkspace, fullyQualifiedStackName } from "`@pulumi/pulumi/automation";`

const args = process.argv.slice(2);
const destroy = args.length > 0 && args[0] === "destroy";

const org = "justinvp";
const projectName = "aws-ts-s3-folder";
const stackName = fullyQualifiedStackName(org, projectName, "devnode");

(async function() {
    const stack = await RemoteWorkspace.createOrSelectStack({
        stackName,
        url: "https://github.com/pulumi/examples.git",
        branch: "refs/heads/master",
        projectPath: projectName,
    }, {
        envVars: {
            AWS_REGION:            "us-west-2",
            AWS_ACCESS_KEY_ID:     process.env.AWS_ACCESS_KEY_ID ?? "",
            AWS_SECRET_ACCESS_KEY: { secret: process.env.AWS_SECRET_ACCESS_KEY ?? "" },
            AWS_SESSION_TOKEN:     { secret: process.env.AWS_SESSION_TOKEN ?? "" },
        },
    });

    if (destroy) {
        await stack.destroy({ onOutput: out => process.stdout.write(out) });
        console.log("Stack successfully destroyed");
        process.exit(0);
    }

    const upRes = await stack.up({ onOutput: out => process.stdout.write(out) });
    console.log("Update succeeded!");
    console.log(`url: ${upRes.outputs.websiteUrl.value}`);
})();
```

I will add sanity tests subsequently.

11174: [auto/python] Support for remote operations r=justinvp a=justinvp

This change adds preview support for remote operations in Python's Automation API.

Here's an example of using it:

```python
import sys
import os

import pulumi.automation as auto


args = sys.argv[1:]
destroy = args and args[0] == "destroy"

org = "justinvp"
project_name = "aws-ts-s3-folder"
stack_name = auto.fully_qualified_stack_name(org, project_name, "devpy")

stack = auto.create_or_select_remote_stack_git_source(
    stack_name=stack_name,
    url="https://github.com/pulumi/examples.git",
    branch="refs/heads/master",
    project_path=project_name,
    opts=auto.RemoteWorkspaceOptions(
        env_vars={
            "AWS_REGION":            "us-west-2",
            "AWS_ACCESS_KEY_ID":     os.environ["AWS_ACCESS_KEY_ID"],
            "AWS_SECRET_ACCESS_KEY": auto.Secret(os.environ["AWS_SECRET_ACCESS_KEY"]),
            "AWS_SESSION_TOKEN":     auto.Secret(os.environ["AWS_SESSION_TOKEN"]),
        },
    ),
)

if destroy:
    stack.destroy(on_output=print)
    print("Stack successfully destroyed")
    sys.exit()

up_res = stack.up(on_output=print)
print(f"Update succeeded!")
print(f"url: {up_res.outputs['websiteUrl'].value}")
```

I will add sanity tests subsequently.

Co-authored-by: Justin Van Patten <jvp@justinvp.com>

changelog/pending/20221027--auto-go--support-for-remote-operations.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: auto/go
+  description: Support for remote operations

changelog/pending/20221027--auto-nodejs--support-for-remote-operations.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: auto/nodejs
+  description: Support for remote operations

changelog/pending/20221027--auto-python--support-for-remote-operations.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: auto/python
+  description: Support for remote operations

sdk/go/auto/local_workspace.go

@@ -1,4 +1,4 @@
-// Copyright 2016-2021, Pulumi Corporation.
+// Copyright 2016-2022, Pulumi Corporation.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -51,6 +51,10 @@ type LocalWorkspace struct {
 	envvars         map[string]string
 	secretsProvider string
 	pulumiVersion   semver.Version
+	repo            *GitRepo
+	remote          bool
+	remoteEnvVars   map[string]EnvVarValue
+	preRunCommands  []string
 }
 
 var settingsExtensions = []string{".yaml", ".yml", ".json"}
@@ -324,6 +328,9 @@ func (l *LocalWorkspace) CreateStack(ctx context.Context, stackName string) erro
 	if l.secretsProvider != "" {
 		args = append(args, "--secrets-provider", l.secretsProvider)
 	}
+	if l.remote {
+		args = append(args, "--no-select")
+	}
 	stdout, stderr, errCode, err := l.runPulumiCmdSync(ctx, args...)
 	if err != nil {
 		return newAutoError(errors.Wrap(err, "failed to create stack"), stdout, stderr, errCode)
@@ -334,7 +341,15 @@ func (l *LocalWorkspace) CreateStack(ctx context.Context, stackName string) erro
 
 // SelectStack selects and sets an existing stack matching the stack name, failing if none exists.
 func (l *LocalWorkspace) SelectStack(ctx context.Context, stackName string) error {
-	stdout, stderr, errCode, err := l.runPulumiCmdSync(ctx, "stack", "select", stackName)
+	// If this is a remote workspace, we don't want to actually select the stack (which would modify global state);
+	// but we will ensure the stack exists by calling `pulumi stack`.
+	args := []string{"stack"}
+	if !l.remote {
+		args = append(args, "select")
+	}
+	args = append(args, "--stack", stackName)
+
+	stdout, stderr, errCode, err := l.runPulumiCmdSync(ctx, args...)
 	if err != nil {
 		return newAutoError(errors.Wrap(err, "failed to select stack"), stdout, stderr, errCode)
 	}
@@ -576,6 +591,28 @@ func (l *LocalWorkspace) runPulumiCmdSync(
 	)
 }
 
+// supportsPulumiCmdFlag runs a command with `--help` to see if the specified flag is found within the resulting
+// output, in which case we assume the flag is supported.
+func (l *LocalWorkspace) supportsPulumiCmdFlag(ctx context.Context, flag string, args ...string) (bool, error) {
+	env := []string{
+		"PULUMI_DEBUG_COMMANDS=true",
+		"PULUMI_EXPERIMENTAL=true",
+	}
+
+	// Run the command with `--help`, and then we'll look for the flag in the output.
+	stdout, _, _, err := runPulumiCommandSync(ctx, l.WorkDir(), nil, nil, env, append(args, "--help")...)
+	if err != nil {
+		return false, err
+	}
+
+	// Does the help test in stdout mention the flag? If so, assume it's supported.
+	if strings.Contains(stdout, flag) {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 // NewLocalWorkspace creates and configures a LocalWorkspace. LocalWorkspaceOptions can be used to
 // configure things like the working directory, the program to execute, and to seed the directory with source code
 // from a git repository.
@@ -598,7 +635,7 @@ func NewLocalWorkspace(ctx context.Context, opts ...LocalWorkspaceOption) (Works
 		workDir = dir
 	}
 
-	if lwOpts.Repo != nil {
+	if lwOpts.Repo != nil && !lwOpts.Remote {
 		// now do the git clone
 		projDir, err := setupGitRepo(ctx, workDir, lwOpts.Repo)
 		if err != nil {
@@ -613,9 +650,13 @@ func NewLocalWorkspace(ctx context.Context, opts ...LocalWorkspaceOption) (Works
 	}
 
 	l := &LocalWorkspace{
-		workDir:    workDir,
-		program:    program,
-		pulumiHome: lwOpts.PulumiHome,
+		workDir:        workDir,
+		preRunCommands: lwOpts.PreRunCommands,
+		program:        program,
+		pulumiHome:     lwOpts.PulumiHome,
+		remote:         lwOpts.Remote,
+		remoteEnvVars:  lwOpts.RemoteEnvVars,
+		repo:           lwOpts.Repo,
 	}
 
 	// optOut indicates we should skip the version check.
@@ -631,6 +672,18 @@ func NewLocalWorkspace(ctx context.Context, opts ...LocalWorkspaceOption) (Works
 		return nil, err
 	}
 
+	// If remote was specified, ensure the CLI supports it.
+	if !optOut && l.remote {
+		// See if `--remote` is present in `pulumi preview --help`'s output.
+		supportsRemote, err := l.supportsPulumiCmdFlag(ctx, "--remote", "preview")
+		if err != nil {
+			return nil, err
+		}
+		if !supportsRemote {
+			return nil, errors.New("Pulumi CLI does not support remote operations; please upgrade")
+		}
+	}
+
 	if lwOpts.Project != nil {
 		err := l.SaveProjectSettings(ctx, lwOpts.Project)
 		if err != nil {
@@ -647,7 +700,7 @@ func NewLocalWorkspace(ctx context.Context, opts ...LocalWorkspaceOption) (Works
 	}
 
 	// setup
-	if lwOpts.Repo != nil && lwOpts.Repo.Setup != nil {
+	if !lwOpts.Remote && lwOpts.Repo != nil && lwOpts.Repo.Setup != nil {
 		err := lwOpts.Repo.Setup(ctx, l)
 		if err != nil {
 			return nil, errors.Wrap(err, "error while running setup function")
@@ -669,6 +722,13 @@ func NewLocalWorkspace(ctx context.Context, opts ...LocalWorkspaceOption) (Works
 	return l, nil
 }
 
+// EnvVarValue represents the value of an envvar. A value can be a secret, which is passed along
+// to remote operations when used with remote workspaces, otherwise, it has no affect.
+type EnvVarValue struct {
+	Value  string
+	Secret bool
+}
+
 type localWorkspaceOptions struct {
 	// WorkDir is the directory to execute commands from and store state.
 	// Defaults to a tmp dir.
@@ -690,6 +750,12 @@ type localWorkspaceOptions struct {
 	// EnvVars is a map of environment values scoped to the workspace.
 	// These values will be passed to all Workspace and Stack level commands.
 	EnvVars map[string]string
+	// Whether the workspace represents a remote workspace.
+	Remote bool
+	// Remote environment variables to be passed to the remote Pulumi operation.
+	RemoteEnvVars map[string]EnvVarValue
+	// PreRunCommands is an optional list of arbitrary commands to run before the remote Pulumi operation is invoked.
+	PreRunCommands []string
 }
 
 // LocalWorkspaceOption is used to customize and configure a LocalWorkspace at initialization time.
@@ -809,6 +875,28 @@ func EnvVars(envvars map[string]string) LocalWorkspaceOption {
 	})
 }
 
+// remoteEnvVars is a map of environment values scoped to the workspace.
+// These values will be passed to the remote Pulumi operation.
+func remoteEnvVars(envvars map[string]EnvVarValue) LocalWorkspaceOption {
+	return localWorkspaceOption(func(lo *localWorkspaceOptions) {
+		lo.RemoteEnvVars = envvars
+	})
+}
+
+// remote is set on the local workspace to indicate it is actually a remote workspace.
+func remote(remote bool) LocalWorkspaceOption {
+	return localWorkspaceOption(func(lo *localWorkspaceOptions) {
+		lo.Remote = remote
+	})
+}
+
+// preRunCommands is an optional list of arbitrary commands to run before the remote Pulumi operation is invoked.
+func preRunCommands(commands ...string) LocalWorkspaceOption {
+	return localWorkspaceOption(func(lo *localWorkspaceOptions) {
+		lo.PreRunCommands = commands
+	})
+}
+
 // NewStackLocalSource creates a Stack backed by a LocalWorkspace created on behalf of the user,
 // from the specified WorkDir. This Workspace will pick up
 // any available Settings files (Pulumi.yaml, Pulumi.<stack>.yaml).

sdk/go/auto/optremotedestroy/optremotedestroy.go

@@ -0,0 +1,68 @@
+// Copyright 2016-2022, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package optremotedestroy contains functional options to be used with remote stack destroy operations
+// github.com/sdk/v3/go/auto RemoteStack.Destroy(...optremotedestroy.Option)
+package optremotedestroy
+
+import (
+	"io"
+
+	"github.com/pulumi/pulumi/sdk/v3/go/auto/events"
+)
+
+// ProgressStreams allows specifying one or more io.Writers to redirect incremental destroy stdout
+func ProgressStreams(writers ...io.Writer) Option {
+	return optionFunc(func(opts *Options) {
+		opts.ProgressStreams = writers
+	})
+}
+
+// ErrorProgressStreams allows specifying one or more io.Writers to redirect incremental destroy stderr
+func ErrorProgressStreams(writers ...io.Writer) Option {
+	return optionFunc(func(opts *Options) {
+		opts.ErrorProgressStreams = writers
+	})
+}
+
+// EventStreams allows specifying one or more channels to receive the Pulumi event stream
+func EventStreams(channels ...chan<- events.EngineEvent) Option {
+	return optionFunc(func(opts *Options) {
+		opts.EventStreams = channels
+	})
+}
+
+// Option is a parameter to be applied to a Stack.Destroy() operation
+type Option interface {
+	ApplyOption(*Options)
+}
+
+// ---------------------------------- implementation details ----------------------------------
+
+// Options is an implementation detail
+type Options struct {
+	// ProgressStreams allows specifying one or more io.Writers to redirect incremental destroy stdout
+	ProgressStreams []io.Writer
+	// ProgressStreams allows specifying one or more io.Writers to redirect incremental destroy stderr
+	ErrorProgressStreams []io.Writer
+	// EventStreams allows specifying one or more channels to receive the Pulumi event stream
+	EventStreams []chan<- events.EngineEvent
+}
+
+type optionFunc func(*Options)
+
+// ApplyOption is an implementation detail
+func (o optionFunc) ApplyOption(opts *Options) {
+	o(opts)
+}

sdk/go/auto/optremotepreview/optremotepreview.go

@@ -0,0 +1,68 @@
+// Copyright 2016-2022, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package optremotepreview contains functional options to be used with remote stack preview operations
+// github.com/sdk/v3/go/auto RemoteStack.Preview(...optremotepreview.Option)
+package optremotepreview
+
+import (
+	"io"
+
+	"github.com/pulumi/pulumi/sdk/v3/go/auto/events"
+)
+
+// ProgressStreams allows specifying one or more io.Writers to redirect incremental preview stdout
+func ProgressStreams(writers ...io.Writer) Option {
+	return optionFunc(func(opts *Options) {
+		opts.ProgressStreams = writers
+	})
+}
+
+// ErrorProgressStreams allows specifying one or more io.Writers to redirect incremental preview stderr
+func ErrorProgressStreams(writers ...io.Writer) Option {
+	return optionFunc(func(opts *Options) {
+		opts.ErrorProgressStreams = writers
+	})
+}
+
+// EventStreams allows specifying one or more channels to receive the Pulumi event stream
+func EventStreams(channels ...chan<- events.EngineEvent) Option {
+	return optionFunc(func(opts *Options) {
+		opts.EventStreams = channels
+	})
+}
+
+// Option is a parameter to be applied to a Stack.Preview() operation
+type Option interface {
+	ApplyOption(*Options)
+}
+
+// ---------------------------------- implementation details ----------------------------------
+
+// Options is an implementation detail
+type Options struct {
+	// ProgressStreams allows specifying one or more io.Writers to redirect incremental preview stdout
+	ProgressStreams []io.Writer
+	// ErrorProgressStreams allows specifying one or more io.Writers to redirect incremental preview stderr
+	ErrorProgressStreams []io.Writer
+	// EventStreams allows specifying one or more channels to receive the Pulumi event stream
+	EventStreams []chan<- events.EngineEvent
+}
+
+type optionFunc func(*Options)
+
+// ApplyOption is an implementation detail
+func (o optionFunc) ApplyOption(opts *Options) {
+	o(opts)
+}