diff --git a/.gitignore b/.gitignore index c2917bfc48ab..f428c3a1c120 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ venv/ **/.idea/ *.iml - +.yarn # VSCode creates this binary when running tests in the debugger **/debug.test diff --git a/changelog/pending/20220922--cli-initial-mvp-config.yaml b/changelog/pending/20220922--cli-initial-mvp-config.yaml new file mode 100644 index 000000000000..233135232623 --- /dev/null +++ b/changelog/pending/20220922--cli-initial-mvp-config.yaml @@ -0,0 +1,4 @@ +changes: +- type: feat + scope: cli + description: Implement initial MVP for hierarchical and structured project configuration. diff --git a/pkg/backend/filestate/crypto.go b/pkg/backend/filestate/crypto.go index aa698797f67e..7379ea47dcbe 100644 --- a/pkg/backend/filestate/crypto.go +++ b/pkg/backend/filestate/crypto.go @@ -26,7 +26,12 @@ func NewPassphraseSecretsManager(stackName tokens.Name, configFile string, rotatePassphraseSecretsProvider bool) (secrets.Manager, error) { contract.Assertf(stackName != "", "stackName %s", "!= \"\"") - info, err := workspace.LoadProjectStack(configFile) + project, _, err := workspace.DetectProjectStackPath(stackName.Q()) + if err != nil { + return nil, err + } + + info, err := workspace.LoadProjectStack(project, configFile) if err != nil { return nil, err } diff --git a/pkg/backend/httpstate/crypto.go b/pkg/backend/httpstate/crypto.go index f4858cd74080..1d3ad0296021 100644 --- a/pkg/backend/httpstate/crypto.go +++ b/pkg/backend/httpstate/crypto.go @@ -25,7 +25,12 @@ import ( func NewServiceSecretsManager(s Stack, stackName tokens.Name, configFile string) (secrets.Manager, error) { contract.Assertf(stackName != "", "stackName %s", "!= \"\"") - info, err := workspace.LoadProjectStack(configFile) + project, _, err := workspace.DetectProjectStackPath(stackName.Q()) + if err != nil { + return nil, err + } + + info, err := workspace.LoadProjectStack(project, configFile) if err != nil { return nil, err } diff --git a/pkg/cmd/pulumi/config.go b/pkg/cmd/pulumi/config.go index 9fbda04d1231..507a1976c6b2 100644 --- a/pkg/cmd/pulumi/config.go +++ b/pkg/cmd/pulumi/config.go @@ -57,12 +57,18 @@ func newConfigCmd() *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + + if err != nil { + return err + } + stack, err := requireStack(ctx, stack, true, opts, true /*setCurrent*/) if err != nil { return err } - return listConfig(ctx, stack, showSecrets, jsonOut) + return listConfig(ctx, project, stack, showSecrets, jsonOut) }), } @@ -106,6 +112,12 @@ func newConfigCopyCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + + if err != nil { + return err + } + // Get current stack and ensure that it is a different stack to the destination stack currentStack, err := requireStack(ctx, *stack, false, opts, true /*setCurrent*/) if err != nil { @@ -114,7 +126,7 @@ func newConfigCopyCmd(stack *string) *cobra.Command { if currentStack.Ref().Name().String() == destinationStackName { return errors.New("current stack and destination stack are the same") } - currentProjectStack, err := loadProjectStack(currentStack) + currentProjectStack, err := loadProjectStack(project, currentStack) if err != nil { return err } @@ -124,7 +136,7 @@ func newConfigCopyCmd(stack *string) *cobra.Command { if err != nil { return err } - destinationProjectStack, err := loadProjectStack(destinationStack) + destinationProjectStack, err := loadProjectStack(project, destinationStack) if err != nil { return err } @@ -302,7 +314,12 @@ func newConfigRmCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } - s, err := requireStack(ctx, *stack, true, opts, true /*setCurrent*/) + project, _, err := readProject() + if err != nil { + return err + } + + stack, err := requireStack(ctx, *stack, true, opts, true /*setCurrent*/) if err != nil { return err } @@ -312,7 +329,7 @@ func newConfigRmCmd(stack *string) *cobra.Command { return fmt.Errorf("invalid configuration key: %w", err) } - ps, err := loadProjectStack(s) + ps, err := loadProjectStack(project, stack) if err != nil { return err } @@ -322,7 +339,7 @@ func newConfigRmCmd(stack *string) *cobra.Command { return err } - return saveProjectStack(s, ps) + return saveProjectStack(stack, ps) }), } rmCmd.PersistentFlags().BoolVar( @@ -351,12 +368,17 @@ func newConfigRmAllCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } - s, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/) + project, _, err := readProject() if err != nil { return err } - ps, err := loadProjectStack(s) + stack, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/) + if err != nil { + return err + } + + ps, err := loadProjectStack(project, stack) if err != nil { return err } @@ -373,7 +395,7 @@ func newConfigRmAllCmd(stack *string) *cobra.Command { } } - return saveProjectStack(s, ps) + return saveProjectStack(stack, ps) }), } rmAllCmd.PersistentFlags().BoolVar( @@ -395,6 +417,11 @@ func newConfigRefreshCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + if err != nil { + return err + } + // Ensure the stack exists. s, err := requireStack(ctx, *stack, false, opts, false /*setCurrent*/) if err != nil { @@ -411,7 +438,7 @@ func newConfigRefreshCmd(stack *string) *cobra.Command { return err } - ps, err := workspace.LoadProjectStack(configPath) + ps, err := workspace.LoadProjectStack(project, configPath) if err != nil { return err } @@ -480,6 +507,12 @@ func newConfigSetCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + + if err != nil { + return err + } + // Ensure the stack exists. s, err := requireStack(ctx, *stack, true, opts, true /*setCurrent*/) if err != nil { @@ -538,7 +571,7 @@ func newConfigSetCmd(stack *string) *cobra.Command { } } - ps, err := loadProjectStack(s) + ps, err := loadProjectStack(project, s) if err != nil { return err } @@ -591,13 +624,18 @@ func newConfigSetAllCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + if err != nil { + return err + } + // Ensure the stack exists. - s, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/) + stack, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/) if err != nil { return err } - ps, err := loadProjectStack(s) + ps, err := loadProjectStack(project, stack) if err != nil { return err } @@ -620,7 +658,7 @@ func newConfigSetAllCmd(stack *string) *cobra.Command { if err != nil { return err } - c, cerr := getStackEncrypter(s) + c, cerr := getStackEncrypter(stack) if cerr != nil { return cerr } @@ -636,7 +674,7 @@ func newConfigSetAllCmd(stack *string) *cobra.Command { } } - return saveProjectStack(s, ps) + return saveProjectStack(stack, ps) }), } @@ -680,16 +718,17 @@ var stackConfigFile string func getProjectStackPath(stack backend.Stack) (string, error) { if stackConfigFile == "" { - return workspace.DetectProjectStackPath(stack.Ref().Name().Q()) + _, path, err := workspace.DetectProjectStackPath(stack.Ref().Name().Q()) + return path, err } return stackConfigFile, nil } -func loadProjectStack(stack backend.Stack) (*workspace.ProjectStack, error) { +func loadProjectStack(project *workspace.Project, stack backend.Stack) (*workspace.ProjectStack, error) { if stackConfigFile == "" { return workspace.DetectProjectStack(stack.Ref().Name().Q()) } - return workspace.LoadProjectStack(stackConfigFile) + return workspace.LoadProjectStack(project, stackConfigFile) } func saveProjectStack(stack backend.Stack, ps *workspace.ProjectStack) error { @@ -741,12 +780,25 @@ type configValueJSON struct { Secret bool `json:"secret"` } -func listConfig(ctx context.Context, stack backend.Stack, showSecrets bool, jsonOut bool) error { - ps, err := loadProjectStack(stack) +func listConfig(ctx context.Context, + project *workspace.Project, + stack backend.Stack, + showSecrets bool, + jsonOut bool) error { + + ps, err := loadProjectStack(project, stack) if err != nil { return err } + stackName := stack.Ref().Name().String() + // when listing configuration values + // also show values coming from the project + configError := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, project, ps.Config) + if configError != nil { + return configError + } + cfg := ps.Config // By default, we will use a blinding decrypter to show "[secret]". If requested, display secrets in plaintext. @@ -827,10 +879,19 @@ func listConfig(ctx context.Context, stack backend.Stack, showSecrets bool, json } func getConfig(ctx context.Context, stack backend.Stack, key config.Key, path, jsonOut bool) error { - ps, err := loadProjectStack(stack) + project, _, err := readProject() if err != nil { return err } + ps, err := loadProjectStack(project, stack) + if err != nil { + return err + } + stackName := stack.Ref().Name().String() + configError := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, project, ps.Config) + if configError != nil { + return configError + } cfg := ps.Config @@ -921,15 +982,20 @@ func looksLikeSecret(k config.Key, v string) bool { // getStackConfiguration loads configuration information for a given stack. If stackConfigFile is non empty, // it is uses instead of the default configuration file for the stack func getStackConfiguration( - ctx context.Context, stack backend.Stack, + ctx context.Context, + stack backend.Stack, + project *workspace.Project, sm secrets.Manager) (backend.StackConfiguration, error) { var cfg config.Map - workspaceStack, err := loadProjectStack(stack) + + defaultStackConfig := backend.StackConfiguration{} + + workspaceStack, err := loadProjectStack(project, stack) if err != nil || workspaceStack == nil { // On first run or the latest configuration is unavailable, fallback to check the project's configuration cfg, err = backend.GetLatestConfiguration(ctx, stack) if err != nil { - return backend.StackConfiguration{}, fmt.Errorf( + return defaultStackConfig, fmt.Errorf( "stack configuration could not be loaded from either Pulumi.yaml or the backend: %w", err) } } else { @@ -948,8 +1014,9 @@ func getStackConfiguration( crypter, err := sm.Decrypter() if err != nil { - return backend.StackConfiguration{}, fmt.Errorf("getting configuration decrypter: %w", err) + return defaultStackConfig, fmt.Errorf("getting configuration decrypter: %w", err) } + return backend.StackConfiguration{ Config: cfg, Decrypter: crypter, diff --git a/pkg/cmd/pulumi/crypto.go b/pkg/cmd/pulumi/crypto.go index 88578f95796c..a348a85dd96c 100644 --- a/pkg/cmd/pulumi/crypto.go +++ b/pkg/cmd/pulumi/crypto.go @@ -45,7 +45,13 @@ func getStackDecrypter(s backend.Stack) (config.Decrypter, error) { } func getStackSecretsManager(s backend.Stack) (secrets.Manager, error) { - ps, err := loadProjectStack(s) + project, _, err := readProject() + + if err != nil { + return nil, err + } + + ps, err := loadProjectStack(project, s) if err != nil { return nil, err } diff --git a/pkg/cmd/pulumi/crypto_cloud.go b/pkg/cmd/pulumi/crypto_cloud.go index 160c889248ba..8b49cf3b09c2 100644 --- a/pkg/cmd/pulumi/crypto_cloud.go +++ b/pkg/cmd/pulumi/crypto_cloud.go @@ -27,8 +27,12 @@ import ( func newCloudSecretsManager(stackName tokens.Name, configFile, secretsProvider string) (secrets.Manager, error) { contract.Assertf(stackName != "", "stackName %s", "!= \"\"") + proj, _, err := workspace.DetectProjectStackPath(stackName.Q()) + if err != nil { + return nil, err + } - info, err := workspace.LoadProjectStack(configFile) + info, err := workspace.LoadProjectStack(proj, configFile) if err != nil { return nil, err } diff --git a/pkg/cmd/pulumi/crypto_cloud_test.go b/pkg/cmd/pulumi/crypto_cloud_test.go index 587bb9c64fa3..d609ff58f78f 100644 --- a/pkg/cmd/pulumi/crypto_cloud_test.go +++ b/pkg/cmd/pulumi/crypto_cloud_test.go @@ -22,49 +22,71 @@ import ( "testing" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/stretchr/testify/assert" "gocloud.dev/secrets" "gocloud.dev/secrets/driver" ) +func deleteFiles(t *testing.T, files map[string]string) { + for file := range files { + err := os.Remove(file) + assert.Nil(t, err, "Should be able to remove the file directory") + } +} + +func createTempFiles(t *testing.T, files map[string]string, f func()) { + for file, content := range files { + fileError := os.WriteFile(file, []byte(content), 0600) + assert.Nil(t, fileError, "should be able to write the file contents") + } + + defer deleteFiles(t, files) + f() +} + //nolint:paralleltest -func TestSecretsproviderOverride(t *testing.T) { +func TestSecretsProviderOverride(t *testing.T) { // Don't call t.Parallel because we temporarily modify // PULUMI_CLOUD_SECRET_OVERRIDE env var and it may interfere with other // tests. - const stackConfig = "Pulumi.TestSecretsproviderOverride.yaml" - var stackName = tokens.Name("TestSecretsproviderOverride") - // Cleanup the generated stack config after the test. - t.Cleanup(func() { os.Remove(stackConfig) }) + stackConfigFileName := "Pulumi.TestSecretsProviderOverride.yaml" + files := make(map[string]string) + files["Pulumi.yaml"] = "{\"name\":\"test\", \"runtime\":\"dotnet\"}" + files[stackConfigFileName] = "" + + var stackName = tokens.Name("TestSecretsProviderOverride") opener := &mockSecretsKeeperOpener{} secrets.DefaultURLMux().RegisterKeeper("test", opener) //nolint:paralleltest t.Run("without override", func(t *testing.T) { - opener.wantURL = "test://foo" - - if _, err := newCloudSecretsManager(stackName, stackConfig, "test://foo"); err != nil { - t.Fatalf("newCloudSecretsManager failed: %v", err) - } - if _, err := newCloudSecretsManager(stackName, stackConfig, "test://bar"); err == nil { - t.Fatal("newCloudSecretsManager with unexpected secretsProvider URL succeeded, expected an error") - } + createTempFiles(t, files, func() { + opener.wantURL = "test://foo" + _, createSecretsManagerError := newCloudSecretsManager(stackName, stackConfigFileName, "test://foo") + assert.Nil(t, createSecretsManagerError, "Creating the cloud secret manager should succeed") + + _, createSecretsManagerError = newCloudSecretsManager(stackName, stackConfigFileName, "test://bar") + msg := "newCloudSecretsManager with unexpected secretsProvider URL succeeded, expected an error" + assert.NotNil(t, createSecretsManagerError, msg) + }) }) //nolint:paralleltest t.Run("with override", func(t *testing.T) { - opener.wantURL = "test://bar" - t.Setenv("PULUMI_CLOUD_SECRET_OVERRIDE", "test://bar") - - // Last argument here shouldn't matter anymore, since it gets overridden - // by the env var. Both calls should succeed. - if _, err := newCloudSecretsManager(stackName, stackConfig, "test://foo"); err != nil { - t.Fatalf("newCloudSecretsManager failed: %v", err) - } - if _, err := newCloudSecretsManager(stackName, stackConfig, "test://bar"); err != nil { - t.Fatalf("newCloudSecretsManager failed: %v", err) - } + createTempFiles(t, files, func() { + opener.wantURL = "test://bar" + t.Setenv("PULUMI_CLOUD_SECRET_OVERRIDE", "test://bar") + + // Last argument here shouldn't matter anymore, since it gets overridden + // by the env var. Both calls should succeed. + msg := "creating the secrets manager should succeed regardless of secrets provider" + _, createSecretsManagerError := newCloudSecretsManager(stackName, stackConfigFileName, "test://foo") + assert.Nil(t, createSecretsManagerError, msg) + _, createSecretsManagerError = newCloudSecretsManager(stackName, stackConfigFileName, "test://bar") + assert.Nil(t, createSecretsManagerError, msg) + }) }) } diff --git a/pkg/cmd/pulumi/destroy.go b/pkg/cmd/pulumi/destroy.go index b283f860eae7..8ecc50ca5516 100644 --- a/pkg/cmd/pulumi/destroy.go +++ b/pkg/cmd/pulumi/destroy.go @@ -159,11 +159,18 @@ func newDestroyCmd() *cobra.Command { sm = snap.SecretsManager } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) + if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) + } + stackName := s.Ref().Name().String() + configError := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config) + if configError != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configError)) } + targetUrns := []resource.URN{} for _, t := range *targets { targetUrns = append(targetUrns, snap.GlobUrn(resource.URN(t))...) diff --git a/pkg/cmd/pulumi/import.go b/pkg/cmd/pulumi/import.go index 42a5403db06a..81f6386ed2a0 100644 --- a/pkg/cmd/pulumi/import.go +++ b/pkg/cmd/pulumi/import.go @@ -43,6 +43,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" javagen "github.com/pulumi/pulumi-java/pkg/codegen/java" yamlgen "github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/codegen" @@ -492,11 +493,16 @@ func newImportCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + opts.Engine = engine.UpdateOptions{ Parallel: parallel, Debug: debug, diff --git a/pkg/cmd/pulumi/logs.go b/pkg/cmd/pulumi/logs.go index a29113167b9c..973fd85e2806 100644 --- a/pkg/cmd/pulumi/logs.go +++ b/pkg/cmd/pulumi/logs.go @@ -27,6 +27,7 @@ import ( "github.com/pulumi/pulumi/pkg/v3/operations" "github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) // We use RFC 5424 timestamps with millisecond precision for displaying time stamps on log entries. Go does not @@ -57,6 +58,12 @@ func newLogsCmd() *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + // Fetch the project. + proj, _, err := readProject() + if err != nil { + return err + } + s, err := requireStack(ctx, stack, false, opts, false /*setCurrent*/) if err != nil { return err @@ -67,11 +74,16 @@ func newLogsCmd() *cobra.Command { return fmt.Errorf("getting secrets manager: %w", err) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return fmt.Errorf("getting stack configuration: %w", err) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return fmt.Errorf("validating stack config: %w", configErr) + } + startTime, err := parseSince(since, time.Now()) if err != nil { return fmt.Errorf("failed to parse argument to '--since' as duration or timestamp: %w", err) diff --git a/pkg/cmd/pulumi/new.go b/pkg/cmd/pulumi/new.go index f9a3c7d08553..00eaf98c75e7 100644 --- a/pkg/cmd/pulumi/new.go +++ b/pkg/cmd/pulumi/new.go @@ -674,7 +674,12 @@ func stackInit( // saveConfig saves the config for the stack. func saveConfig(stack backend.Stack, c config.Map) error { - ps, err := loadProjectStack(stack) + project, _, err := readProject() + if err != nil { + return err + } + + ps, err := loadProjectStack(project, stack) if err != nil { return err } diff --git a/pkg/cmd/pulumi/preview.go b/pkg/cmd/pulumi/preview.go index 04b3e34477c5..f98dffd4700d 100644 --- a/pkg/cmd/pulumi/preview.go +++ b/pkg/cmd/pulumi/preview.go @@ -29,6 +29,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) func newPreviewCmd() *cobra.Command { @@ -148,7 +149,16 @@ func newPreviewCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) + if err != nil { + return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) + } + + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } diff --git a/pkg/cmd/pulumi/refresh.go b/pkg/cmd/pulumi/refresh.go index 05e5a2c5f6fc..d06287f84cc7 100644 --- a/pkg/cmd/pulumi/refresh.go +++ b/pkg/cmd/pulumi/refresh.go @@ -33,6 +33,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/resource" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) func newRefreshCmd() *cobra.Command { @@ -146,11 +147,16 @@ func newRefreshCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + if skipPendingCreates && clearPendingCreates { return result.FromError(fmt.Errorf( "cannot set both --skip-pending-creates and --clear-pending-creates")) diff --git a/pkg/cmd/pulumi/stack_change_secrets_provider.go b/pkg/cmd/pulumi/stack_change_secrets_provider.go index 007728061511..13570c13feb9 100644 --- a/pkg/cmd/pulumi/stack_change_secrets_provider.go +++ b/pkg/cmd/pulumi/stack_change_secrets_provider.go @@ -25,6 +25,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" "github.com/spf13/cobra" ) @@ -58,6 +59,12 @@ func newStackChangeSecretsProviderCmd() *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + + if err != nil { + return err + } + // Validate secrets provider type if err := validateSecretsProvider(args[0]); err != nil { return err @@ -68,7 +75,7 @@ func newStackChangeSecretsProviderCmd() *cobra.Command { if err != nil { return err } - currentProjectStack, err := loadProjectStack(currentStack) + currentProjectStack, err := loadProjectStack(project, currentStack) if err != nil { return err } @@ -96,7 +103,7 @@ func newStackChangeSecretsProviderCmd() *cobra.Command { // Fixup the checkpoint fmt.Printf("Migrating old configuration and state to new secrets provider\n") - return migrateOldConfigAndCheckpointToNewSecretsProvider(ctx, currentStack, currentConfig, decrypter) + return migrateOldConfigAndCheckpointToNewSecretsProvider(ctx, project, currentStack, currentConfig, decrypter) }), } @@ -107,7 +114,9 @@ func newStackChangeSecretsProviderCmd() *cobra.Command { return cmd } -func migrateOldConfigAndCheckpointToNewSecretsProvider(ctx context.Context, currentStack backend.Stack, +func migrateOldConfigAndCheckpointToNewSecretsProvider(ctx context.Context, + project *workspace.Project, + currentStack backend.Stack, currentConfig config.Map, decrypter config.Decrypter) error { // The order of operations here should be to load the secrets manager current stack // Get the newly created secrets manager for the stack @@ -129,7 +138,7 @@ func migrateOldConfigAndCheckpointToNewSecretsProvider(ctx context.Context, curr } // Reload the project stack after the new secretsProvider is in place - reloadedProjectStack, err := loadProjectStack(currentStack) + reloadedProjectStack, err := loadProjectStack(project, currentStack) if err != nil { return err } diff --git a/pkg/cmd/pulumi/stack_init.go b/pkg/cmd/pulumi/stack_init.go index a2a6b351866d..585629e712ee 100644 --- a/pkg/cmd/pulumi/stack_init.go +++ b/pkg/cmd/pulumi/stack_init.go @@ -72,6 +72,10 @@ func newStackInitCmd() *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + proj, _, err := readProject() + if err != nil { + return err + } b, err := currentBackend(ctx, opts) if err != nil { return err @@ -129,13 +133,13 @@ func newStackInitCmd() *cobra.Command { if err != nil { return err } - copyProjectStack, err := loadProjectStack(copyStack) + copyProjectStack, err := loadProjectStack(proj, copyStack) if err != nil { return err } // get the project for the newly created stack - newProjectStack, err := loadProjectStack(newStack) + newProjectStack, err := loadProjectStack(proj, newStack) if err != nil { return err } diff --git a/pkg/cmd/pulumi/stack_rename.go b/pkg/cmd/pulumi/stack_rename.go index 56f9715fb622..3c4283895adb 100644 --- a/pkg/cmd/pulumi/stack_rename.go +++ b/pkg/cmd/pulumi/stack_rename.go @@ -55,7 +55,7 @@ func newStackRenameCmd() *cobra.Command { if err != nil { return err } - oldConfigPath, err := workspace.DetectProjectStackPath(s.Ref().Name().Q()) + _, oldConfigPath, err := workspace.DetectProjectStackPath(s.Ref().Name().Q()) if err != nil { return err } @@ -66,7 +66,7 @@ func newStackRenameCmd() *cobra.Command { if err != nil { return err } - newConfigPath, err := workspace.DetectProjectStackPath(newStackRef.Name().Q()) + _, newConfigPath, err := workspace.DetectProjectStackPath(newStackRef.Name().Q()) if err != nil { return err } diff --git a/pkg/cmd/pulumi/stack_rm.go b/pkg/cmd/pulumi/stack_rm.go index ddf602931e6b..307af9609846 100644 --- a/pkg/cmd/pulumi/stack_rm.go +++ b/pkg/cmd/pulumi/stack_rm.go @@ -86,7 +86,7 @@ func newStackRmCmd() *cobra.Command { if !preserveConfig { // Blow away stack specific settings if they exist. If we get an ENOENT error, ignore it. - if path, err := workspace.DetectProjectStackPath(s.Ref().Name().Q()); err == nil { + if _, path, err := workspace.DetectProjectStackPath(s.Ref().Name().Q()); err == nil { if err = os.Remove(path); err != nil && !os.IsNotExist(err) { return result.FromError(err) } diff --git a/pkg/cmd/pulumi/up.go b/pkg/cmd/pulumi/up.go index dad153fa4c41..5643a04add38 100644 --- a/pkg/cmd/pulumi/up.go +++ b/pkg/cmd/pulumi/up.go @@ -106,11 +106,16 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + targetURNs, replaceURNs := []resource.URN{}, []resource.URN{} if len(targets)+len(replaces)+len(targetReplaces) > 0 { @@ -340,11 +345,16 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + refreshOption, err := getRefreshOption(proj, refresh) if err != nil { return result.FromError(err) diff --git a/pkg/cmd/pulumi/watch.go b/pkg/cmd/pulumi/watch.go index 65afc3e7371a..f1de99ee8819 100644 --- a/pkg/cmd/pulumi/watch.go +++ b/pkg/cmd/pulumi/watch.go @@ -26,6 +26,7 @@ import ( "github.com/pulumi/pulumi/pkg/v3/engine" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) // intentionally disabling here for cleaner err declaration/assignment. @@ -111,11 +112,16 @@ func newWatchCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + opts.Engine = engine.UpdateOptions{ LocalPolicyPacks: engine.MakeLocalPolicyPacks(policyPackPaths, policyPackConfigPaths), Parallel: parallel, diff --git a/pkg/testing/integration/pulumi.go b/pkg/testing/integration/pulumi.go index 25ada9777352..1c74600da636 100644 --- a/pkg/testing/integration/pulumi.go +++ b/pkg/testing/integration/pulumi.go @@ -38,6 +38,16 @@ func CreateBasicPulumiRepo(e *testing.Environment) { assert.NoError(e, err, "writing %s file", filePath) } +// CreatePulumiRepo will initialize the environment with a basic Pulumi repository and +// project file definition based on the project file content. +// Returns the repo owner and name used. +func CreatePulumiRepo(e *testing.Environment, projectFileContent string) { + e.RunCommand("git", "init") + filePath := path.Join(e.CWD, fmt.Sprintf("%s.yaml", workspace.ProjectFile)) + err := ioutil.WriteFile(filePath, []byte(projectFileContent), os.ModePerm) + assert.NoError(e, err, "writing %s file", filePath) +} + // GetStacks returns the list of stacks and current stack by scraping `pulumi stack ls`. // Assumes .pulumi is in the current working directory. Fails the test on IO errors. func GetStacks(e *testing.Environment) ([]string, *string) { diff --git a/sdk/go/auto/local_workspace.go b/sdk/go/auto/local_workspace.go index 15aae0f6c44c..834299c2b3ab 100644 --- a/sdk/go/auto/local_workspace.go +++ b/sdk/go/auto/local_workspace.go @@ -75,11 +75,16 @@ func (l *LocalWorkspace) SaveProjectSettings(ctx context.Context, settings *work // StackSettings returns the settings object for the stack matching the specified stack name if any. // LocalWorkspace reads this from a Pulumi..yaml file in Workspace.WorkDir(). func (l *LocalWorkspace) StackSettings(ctx context.Context, stackName string) (*workspace.ProjectStack, error) { + project, err := l.ProjectSettings(ctx) + if err != nil { + return nil, err + } + name := getStackSettingsName(stackName) for _, ext := range settingsExtensions { stackPath := filepath.Join(l.WorkDir(), fmt.Sprintf("Pulumi.%s%s", name, ext)) if _, err := os.Stat(stackPath); err == nil { - proj, err := workspace.LoadProjectStack(stackPath) + proj, err := workspace.LoadProjectStack(project, stackPath) if err != nil { return nil, errors.Wrap(err, "found stack settings, but failed to load") } diff --git a/sdk/go/common/resource/config/value.go b/sdk/go/common/resource/config/value.go index e6d19d6fee8e..613990013fec 100644 --- a/sdk/go/common/resource/config/value.go +++ b/sdk/go/common/resource/config/value.go @@ -140,7 +140,7 @@ func (c Value) ToObject() (interface{}, error) { } func (c Value) MarshalJSON() ([]byte, error) { - v, err := c.marshalValue() + v, err := c.MarshalValue() if err != nil { return nil, err } @@ -158,7 +158,7 @@ func (c *Value) UnmarshalJSON(b []byte) error { } func (c Value) MarshalYAML() (interface{}, error) { - return c.marshalValue() + return c.MarshalValue() } func (c *Value) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -202,7 +202,7 @@ func (c *Value) unmarshalValue(unmarshal func(interface{}) error, fix func(inter return nil } -func (c Value) marshalValue() (interface{}, error) { +func (c Value) MarshalValue() (interface{}, error) { if c.object { return c.unmarshalObjectJSON() } diff --git a/sdk/go/common/workspace/config.go b/sdk/go/common/workspace/config.go new file mode 100644 index 000000000000..06a157bdce05 --- /dev/null +++ b/sdk/go/common/workspace/config.go @@ -0,0 +1,174 @@ +package workspace + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" +) + +func formatMissingKeys(missingKeys []string) string { + if len(missingKeys) == 1 { + return fmt.Sprintf("'%v'", missingKeys[0]) + } + + sort.Strings(missingKeys) + + formattedMissingKeys := "" + for index, key := range missingKeys { + // if last index, then use and before the key + if index == len(missingKeys)-1 { + formattedMissingKeys += fmt.Sprintf("and '%s'", key) + } else if index == len(missingKeys)-2 { + // no comma before the last key + formattedMissingKeys += fmt.Sprintf("'%s' ", key) + } else { + formattedMissingKeys += fmt.Sprintf("'%s', ", key) + } + } + + return formattedMissingKeys +} + +func missingStackConfigurationKeysError(missingKeys []string, stackName string) error { + valueOrValues := "value" + if len(missingKeys) > 1 { + valueOrValues = "values" + } + + return fmt.Errorf( + "Stack '%v' is missing configuration %v %v", + stackName, + valueOrValues, + formatMissingKeys(missingKeys)) +} + +func missingProjectConfigurationKeysError(missingProjectKeys []string, stackName string) error { + valueOrValues := "value" + if len(missingProjectKeys) > 1 { + valueOrValues = "values" + } + + isOrAre := "is" + if len(missingProjectKeys) > 1 { + isOrAre = "are" + } + + return fmt.Errorf( + "Stack '%v' uses configuration %v %v which %v not defined by the project configuration", + stackName, + valueOrValues, + formatMissingKeys(missingProjectKeys), + isOrAre) +} + +func ValidateStackConfigAndApplyProjectConfig( + stackName string, + project *Project, + stackConfig config.Map) error { + + if len(project.Config) > 0 { + // only when the project defines config values, do we need to validate the stack config + // for each stack config key, check if it is in the project config + stackConfigKeysNotDefinedByProject := []string{} + for key := range stackConfig { + namespacedKey := fmt.Sprintf("%s:%s", key.Namespace(), key.Name()) + if key.Namespace() == string(project.Name) { + // then the namespace is implied and can be omitted + namespacedKey = key.Name() + } + + if _, ok := project.Config[namespacedKey]; !ok { + stackConfigKeysNotDefinedByProject = append(stackConfigKeysNotDefinedByProject, namespacedKey) + } + } + + if len(stackConfigKeysNotDefinedByProject) > 0 { + return missingProjectConfigurationKeysError(stackConfigKeysNotDefinedByProject, stackName) + } + } + + missingConfigurationKeys := make([]string, 0) + for projectConfigKey, projectConfigType := range project.Config { + var key config.Key + if strings.Contains(projectConfigKey, ":") { + // key is already namespaced + parsedKey, parseError := config.ParseKey(projectConfigKey) + if parseError != nil { + return parseError + } + + key = parsedKey + } else { + // key is not namespaced + // use the project as namespace + key = config.MustMakeKey(string(project.Name), projectConfigKey) + } + + stackValue, found, err := stackConfig.Get(key, true) + if err != nil { + return fmt.Errorf("Error while getting stack config value for key '%v': %v", key.String(), err) + } + + hasDefault := projectConfigType.Default != nil + if !found && !hasDefault { + // add it to the list to collect all missing configuration keys, + // then return them as a single error + missingConfigurationKeys = append(missingConfigurationKeys, projectConfigKey) + } else if !found && hasDefault { + // not found at the stack level + // but has a default value at the project level + // assign the value to the stack + var configValue config.Value + + if projectConfigType.Type == "array" { + // for array types, JSON-ify the default value + configValueJSON, jsonError := json.Marshal(projectConfigType.Default) + if jsonError != nil { + return jsonError + } + configValue = config.NewObjectValue(string(configValueJSON)) + + } else { + // for primitive types + // pass the values as is + configValueContent := fmt.Sprintf("%v", projectConfigType.Default) + configValue = config.NewValue(configValueContent) + } + + setError := stackConfig.Set(key, configValue, true) + if setError != nil { + return setError + } + } else { + // found value on the stack level + // retrieve it and validate it against + // the config defined at the project level + content, contentError := stackValue.MarshalValue() + if contentError != nil { + return contentError + } + + if !ValidateConfigValue(projectConfigType.Type, projectConfigType.Items, content) { + typeName := InferFullTypeName(projectConfigType.Type, projectConfigType.Items) + validationError := fmt.Errorf( + "Stack '%v' with configuration key '%v' must be of type '%v'", + stackName, + projectConfigKey, + typeName) + + return validationError + } + } + } + + if len(missingConfigurationKeys) > 0 { + // there are missing configuration keys in the stack + // return them as a single error. + return missingStackConfigurationKeysError(missingConfigurationKeys, stackName) + } + + return nil +} diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index d5864baf1bcf..c5e6f9ba5b2f 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -18,6 +18,7 @@ import ( "fmt" "io/ioutil" "os" + "strings" "sync" "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" @@ -105,8 +106,27 @@ func (singleton *projectLoader) load(path string) (*Project, error) { return nil, fmt.Errorf("could not validate '%s': %w", path, err) } + // just before marshalling, we will rewrite the config values + projectDef, err := SimplifyMarshalledProject(raw) + if err != nil { + return nil, err + } + + projectDef, rewriteError := RewriteConfigPathIntoStackConfigDir(projectDef) + if rewriteError != nil { + return nil, rewriteError + } + + projectDef = RewriteShorthandConfigValues(projectDef) + modifiedProject, _ := marshaller.Marshal(projectDef) + var project Project - err = marshaller.Unmarshal(b, &project) + err = marshaller.Unmarshal(modifiedProject, &project) + if err != nil { + return nil, err + } + + err = project.Validate() if err != nil { return nil, fmt.Errorf("could not unmarshal '%s': %w", path, err) } @@ -121,8 +141,48 @@ type projectStackLoader struct { internal map[string]*ProjectStack } +// Rewrite config values to make them namespaced. Using the project name as the default namespace +// for example: +// +// config: +// instanceSize: t3.micro +// +// is valid configuration and will be rewritten in the form +// +// config: +// {projectName}:instanceSize:t3.micro +func stackConfigNamespacedWithProject(project *Project, projectStack map[string]interface{}) map[string]interface{} { + if project == nil { + // return the original config if we don't have a project + return projectStack + } + + config, ok := projectStack["config"] + if ok { + configAsMap, isMap := config.(map[string]interface{}) + if isMap { + modifiedConfig := make(map[string]interface{}) + for key, value := range configAsMap { + if strings.Contains(key, ":") { + // key is already namespaced + // use it as is + modifiedConfig[key] = value + } else { + namespacedKey := fmt.Sprintf("%s:%s", project.Name, key) + modifiedConfig[namespacedKey] = value + } + } + + projectStack["config"] = modifiedConfig + return projectStack + } + } + + return projectStack +} + // Load a ProjectStack config file from the specified path. The configuration will be cached for subsequent loads. -func (singleton *projectStackLoader) load(path string) (*ProjectStack, error) { +func (singleton *projectStackLoader) load(project *Project, path string) (*ProjectStack, error) { singleton.Lock() defer singleton.Unlock() @@ -130,24 +190,56 @@ func (singleton *projectStackLoader) load(path string) (*ProjectStack, error) { return v, nil } - marshaler, err := marshallerForPath(path) + marshaller, err := marshallerForPath(path) if err != nil { return nil, err } - var projectStack ProjectStack b, err := readFileStripUTF8BOM(path) if os.IsNotExist(err) { - projectStack = ProjectStack{ + defaultProjectStack := ProjectStack{ Config: make(config.Map), } - singleton.internal[path] = &projectStack - return &projectStack, nil + singleton.internal[path] = &defaultProjectStack + return &defaultProjectStack, nil } else if err != nil { return nil, err } - err = marshaler.Unmarshal(b, &projectStack) + var projectStackRaw interface{} + err = marshaller.Unmarshal(b, &projectStackRaw) + if err != nil { + return nil, err + } + + if projectStackRaw == nil { + // for example when reading an empty stack file + defaultProjectStack := ProjectStack{ + Config: make(config.Map), + } + singleton.internal[path] = &defaultProjectStack + return &defaultProjectStack, nil + } + + simplifiedStackForm, err := SimplifyMarshalledProject(projectStackRaw) + if err != nil { + return nil, err + } + + // rewrite config values to make them namespaced + // for example: + // config: + // instanceSize: t3.micro + // + // is valid configuration and will be rewritten in the form + // + // config: + // {projectName}:instanceSize: t3.micro + projectStackWithNamespacedConfig := stackConfigNamespacedWithProject(project, simplifiedStackForm) + modifiedProjectStack, _ := marshaller.Marshal(projectStackWithNamespacedConfig) + + var projectStack ProjectStack + err = marshaller.Unmarshal(modifiedProjectStack, &projectStack) if err != nil { return nil, err } @@ -248,10 +340,10 @@ func LoadProject(path string) (*Project, error) { } // LoadProjectStack reads a stack definition from a file. -func LoadProjectStack(path string) (*ProjectStack, error) { +func LoadProjectStack(project *Project, path string) (*ProjectStack, error) { contract.Require(path != "", "path") - return projectStackSingleton.load(path) + return projectStackSingleton.load(project, path) } // LoadPluginProject reads a plugin project definition from a file. diff --git a/sdk/go/common/workspace/paths.go b/sdk/go/common/workspace/paths.go index dc60082f819f..85409aaa7f53 100644 --- a/sdk/go/common/workspace/paths.go +++ b/sdk/go/common/workspace/paths.go @@ -93,34 +93,19 @@ func DetectProjectPath() (string, error) { // DetectProjectStackPath returns the name of the file to store stack specific project settings in. We place stack // specific settings next to the Pulumi.yaml file, named like: Pulumi..yaml -func DetectProjectStackPath(stackName tokens.QName) (string, error) { +func DetectProjectStackPath(stackName tokens.QName) (*Project, string, error) { proj, projPath, err := DetectProjectAndPath() if err != nil { - return "", err + return nil, "", err } fileName := fmt.Sprintf("%s.%s%s", ProjectFile, qnameFileName(stackName), filepath.Ext(projPath)) - // Back compat: StackConfigDir used to be called Config. - configValue, hasConfigValue := proj.Config.(string) - hasConfigValue = hasConfigValue && configValue != "" - if proj.StackConfigDir != "" { - // If config and stackConfigDir are both set return an error - if hasConfigValue { - return "", fmt.Errorf("can not set `config` and `stackConfigDir`, remove the `config` entry") - } - - return filepath.Join(filepath.Dir(projPath), proj.StackConfigDir, fileName), nil - } - - // Back compat: If StackConfigDir is not present and Config is given and it's a non-empty string use it - // for the stacks directory. - if hasConfigValue { - return filepath.Join(filepath.Dir(projPath), configValue, fileName), nil + return proj, filepath.Join(filepath.Dir(projPath), proj.StackConfigDir, fileName), nil } - return filepath.Join(filepath.Dir(projPath), fileName), nil + return proj, filepath.Join(filepath.Dir(projPath), fileName), nil } var ErrProjectNotFound = errors.New("no project file found") @@ -159,12 +144,12 @@ func DetectProject() (*Project, error) { } func DetectProjectStack(stackName tokens.QName) (*ProjectStack, error) { - path, err := DetectProjectStackPath(stackName) + project, path, err := DetectProjectStackPath(stackName) if err != nil { return nil, err } - return LoadProjectStack(path) + return LoadProjectStack(project, path) } // DetectProjectAndPath loads the closest package from the current working directory, or an error if not found. It @@ -193,7 +178,7 @@ func SaveProject(proj *Project) error { } func SaveProjectStack(stackName tokens.QName, stack *ProjectStack) error { - path, err := DetectProjectStackPath(stackName) + _, path, err := DetectProjectStackPath(stackName) if err != nil { return err } diff --git a/sdk/go/common/workspace/paths_test.go b/sdk/go/common/workspace/paths_test.go index 3c8f5a025c84..be36e18a64c0 100644 --- a/sdk/go/common/workspace/paths_test.go +++ b/sdk/go/common/workspace/paths_test.go @@ -90,7 +90,9 @@ func TestProjectStackPath(t *testing.T) { "name: some_project\ndescription: Some project\nruntime: nodejs\nconfig: stacksA\nstackConfigDir: stacksB\n", func(t *testing.T, projectDir, path string, err error) { assert.Error(t, err) - assert.Equal(t, "can not set `config` and `stackConfigDir`, remove the `config` entry", err.Error()) + errorMsg := "Should not use both config and stackConfigDir to define the stack directory. " + + "Use only stackConfigDir instead." + assert.Contains(t, err.Error(), errorMsg) }, }} @@ -110,7 +112,7 @@ func TestProjectStackPath(t *testing.T) { 0600) assert.NoError(t, err) - path, err := DetectProjectStackPath("my_stack") + _, path, err := DetectProjectStackPath("my_stack") tt.validate(t, tmpDir, path, err) }) } diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 1739ad3005f6..bcaffb8eb841 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "strings" "github.com/hashicorp/go-multierror" @@ -97,6 +98,18 @@ type Plugins struct { Analyzers []PluginOptions `json:"analyzers,omitempty" yaml:"analyzers,omitempty"` } +type ProjectConfigItemsType struct { + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Items *ProjectConfigItemsType `json:"items,omitempty" yaml:"items,omitempty"` +} + +type ProjectConfigType struct { + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Items *ProjectConfigItemsType `json:"items,omitempty" yaml:"items,omitempty"` + Default interface{} `json:"default,omitempty" yaml:"default,omitempty"` +} + // Project is a Pulumi project manifest. // // We explicitly add yaml tags (instead of using the default behavior from https://github.com/ghodss/yaml which works @@ -122,7 +135,7 @@ type Project struct { License *string `json:"license,omitempty" yaml:"license,omitempty"` // Config has been renamed to StackConfigDir. - Config interface{} `json:"config,omitempty" yaml:"config,omitempty"` + Config map[string]ProjectConfigType `json:"config,omitempty" yaml:"config,omitempty"` // StackConfigDir indicates where to store the Pulumi..yaml files, combined with the folder // Pulumi.yaml is in. @@ -143,8 +156,85 @@ type Project struct { AdditionalKeys map[string]interface{} `yaml:",inline"` } -func ValidateProject(raw interface{}) error { - // Cast any map[interface{}] from the yaml decoder to map[string] +func isPrimitiveValue(value interface{}) (string, bool) { + switch value.(type) { + case string: + return "string", true + case int: + return "integer", true + case bool: + return "boolean", true + default: + return "", false + } +} + +// RewriteConfigPathIntoStackConfigDir checks if the project is using the old "config" property +// to declare a path to the stack configuration directory. If that is the case, we rewrite it +// such that the value in config: {value} is moved to stackConfigDir: {value}. +// if the user defines both values as strings, we error out. +func RewriteConfigPathIntoStackConfigDir(project map[string]interface{}) (map[string]interface{}, error) { + config, hasConfig := project["config"] + _, hasStackConfigDir := project["stackConfigDir"] + + if hasConfig { + configText, configIsText := config.(string) + if configIsText && hasStackConfigDir { + return nil, errors.New("Should not use both config and stackConfigDir to define the stack directory. " + + "Use only stackConfigDir instead.") + } else if configIsText && !hasStackConfigDir { + // then we have config: {value}. Move this to stackConfigDir: {value} + project["stackConfigDir"] = configText + // reset the config property + project["config"] = nil + return project, nil + } + } + + return project, nil +} + +// RewriteShorthandConfigValues rewrites short-hand version of configuration into a configuration type +// for example the following config block definition: +// +// config: +// instanceSize: t3.mirco +// +// will be rewritten into a typed value: +// +// config: +// instanceSize: +// type: string +// default: t3.mirco +func RewriteShorthandConfigValues(project map[string]interface{}) map[string]interface{} { + configMap, foundConfig := project["config"] + + if !foundConfig { + // no config defined, return as is + return project + } + + config, ok := configMap.(map[string]interface{}) + + if !ok { + return project + } + + for key, value := range config { + typeName, isLiteral := isPrimitiveValue(value) + if isLiteral { + configTypeDefinition := make(map[string]interface{}) + configTypeDefinition["type"] = typeName + configTypeDefinition["default"] = value + config[key] = configTypeDefinition + } + } + + return project +} + +// Cast any map[interface{}] from the yaml decoder to map[string] +func SimplifyMarshalledProject(raw interface{}) (map[string]interface{}, error) { var cast func(value interface{}) (interface{}, error) cast = func(value interface{}) (interface{}, error) { if objMap, ok := value.(map[interface{}]interface{}); ok { @@ -176,29 +266,39 @@ func ValidateProject(raw interface{}) error { } result, err := cast(raw) if err != nil { - return err + return nil, err } var ok bool var obj map[string]interface{} if obj, ok = result.(map[string]interface{}); !ok { - return fmt.Errorf("expected project to be an object, was '%T'", result) + return nil, fmt.Errorf("expected project to be an object, was '%T'", result) + } + + return obj, nil +} + +func ValidateProject(raw interface{}) error { + + project, err := SimplifyMarshalledProject(raw) + if err != nil { + return err } // Couple of manual errors to match Validate - name, ok := obj["name"] + name, ok := project["name"] if !ok { return errors.New("project is missing a 'name' attribute") } if strName, ok := name.(string); !ok || strName == "" { return errors.New("project is missing a non-empty string 'name' attribute") } - if _, ok := obj["runtime"]; !ok { + if _, ok := project["runtime"]; !ok { return errors.New("project is missing a 'runtime' attribute") } // Let everything else be caught by jsonschema - if err = ProjectSchema.Validate(obj); err == nil { + if err = ProjectSchema.Validate(project); err == nil { return nil } validationError, ok := err.(*jsonschema.ValidationError) @@ -226,6 +326,65 @@ func ValidateProject(raw interface{}) error { return errs } +func InferFullTypeName(typeName string, itemsType *ProjectConfigItemsType) string { + if itemsType != nil { + return fmt.Sprintf("array<%v>", InferFullTypeName(itemsType.Type, itemsType.Items)) + } + + return typeName +} + +// ValidateConfig validates the config value against its config type definition. +// We use this to validate the default config values alongside their type definition but +// also to validate config values coming from individual stacks. +func ValidateConfigValue(typeName string, itemsType *ProjectConfigItemsType, value interface{}) bool { + + if typeName == "string" { + _, ok := value.(string) + return ok + } + + if typeName == "integer" { + _, ok := value.(int) + if !ok { + valueAsText, isText := value.(string) + if isText { + _, integerParseError := strconv.Atoi(valueAsText) + return integerParseError == nil + } + } + return ok + } + + if typeName == "boolean" { + // check to see if the value is a literal string "true" | "false" + literalValue, ok := value.(string) + if ok && (literalValue == "true" || literalValue == "false") { + return true + } + + _, ok = value.(bool) + return ok + } + + items, isArray := value.([]interface{}) + + if !isArray || itemsType == nil { + return false + } + + // validate each item + for _, item := range items { + itemType := itemsType.Type + underlyingItems := itemsType.Items + if !ValidateConfigValue(itemType, underlyingItems, item) { + return false + } + } + + return true +} + func (proj *Project) Validate() error { if proj.Name == "" { return errors.New("project is missing a 'name' attribute") @@ -234,6 +393,18 @@ func (proj *Project) Validate() error { return errors.New("project is missing a 'runtime' attribute") } + for configKey, configType := range proj.Config { + if configType.Default != nil { + // when the default value is specified, validate it against the type + if !ValidateConfigValue(configType.Type, configType.Items, configType.Default) { + inferredTypeName := InferFullTypeName(configType.Type, configType.Items) + return errors.Errorf("The default value specified for configuration key '%v' is not of the expected type '%v'", + configKey, + inferredTypeName) + } + } + } + return nil } diff --git a/sdk/go/common/workspace/project.json b/sdk/go/common/workspace/project.json index 930f154fd2da..c1aa08d8e427 100644 --- a/sdk/go/common/workspace/project.json +++ b/sdk/go/common/workspace/project.json @@ -1,178 +1,324 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/pulumi/pulumi/blob/master/sdk/go/common/workspace/project.json", - "title": "Pulumi Project", - "description": "A schema for Pulumi project files.", - "type": "object", - "properties": { - "name": { - "description": "Name of the project containing alphanumeric characters, hyphens, underscores, and periods.", - "type": "string", - "minLength": 1 + "$schema":"https://json-schema.org/draft/2020-12/schema", + "$id":"https://github.com/pulumi/pulumi/blob/master/sdk/go/common/workspace/project.json", + "title":"Pulumi Project", + "description":"A schema for Pulumi project files.", + "type":"object", + "properties":{ + "name":{ + "description":"Name of the project containing alphanumeric characters, hyphens, underscores, and periods.", + "type":"string", + "minLength":1 }, - "description": { - "description": "Description of the project.", - "type": ["string", "null"] + "description":{ + "description":"Description of the project.", + "type":[ + "string", + "null" + ] }, - "author": { - "description": "Author is an optional author that created this project.", - "type": ["string", "null"] + "author":{ + "description":"Author is an optional author that created this project.", + "type":[ + "string", + "null" + ] }, - "website": { - "description": "Website is an optional website for additional info about this project.", - "type": ["string", "null"] + "website":{ + "description":"Website is an optional website for additional info about this project.", + "type":[ + "string", + "null" + ] }, - "license": { - "description": "License is the optional license governing this project's usage.", - "type": ["string", "null"] + "license":{ + "description":"License is the optional license governing this project's usage.", + "type":[ + "string", + "null" + ] }, - "runtime": { - "title": "ProjectRuntimeInfo", - "oneOf": [ + "runtime":{ + "title":"ProjectRuntimeInfo", + "oneOf":[ { - "title": "Name", - "type": "string", - "minLength": 1 + "title":"Name", + "type":"string", + "minLength":1 }, { - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string", - "minLength": 1 + "type":"object", + "properties":{ + "name":{ + "title":"Name", + "type":"string", + "minLength":1 }, - "options": { - "title": "Options", - "type": "object", - "additionalProperties": true + "options":{ + "title":"Options", + "type":"object", + "additionalProperties":true } }, - "additionalProperties": false + "additionalProperties":false } ] }, - "main": { - "description": "Path to the Pulumi program. The default is the working directory.", - "type": ["string", "null"] + "main":{ + "description":"Path to the Pulumi program. The default is the working directory.", + "type":[ + "string", + "null" + ] }, - "config": { - "description": "Config directory relative to the location of Pulumi.yaml.", - "type": ["string", "null"], - "deprecated": true + "config":{ + "description":"A map of configuration keys to their types. Using config directory location relative to the location of Pulumi.yaml is a deprecated use of this key. Use stackConfigDir instead.", + "type":[ + "object", + "string", + "null" + ], + "additionalProperties":{ + "oneOf":[ + { + "type":"string" + }, + { + "type":"integer" + }, + { + "type":"boolean" + }, + { + "$ref":"#/$defs/configTypeDeclaration" + } + ] + } }, - "stackConfigDir": { - "description": "Config directory location relative to the location of Pulumi.yaml.", - "type": ["string", "null"] + "stackConfigDir":{ + "description":"Config directory location relative to the location of Pulumi.yaml.", + "type":[ + "string", + "null" + ] }, - "backend": { - "description": "Backend of the project.", - "type": ["object", "null"], - "properties": { - "url": { - "description": "URL is optional field to explicitly set backend url", - "type": "string" + "backend":{ + "description":"Backend of the project.", + "type":[ + "object", + "null" + ], + "properties":{ + "url":{ + "description":"URL is optional field to explicitly set backend url", + "type":"string" } }, - "additionalProperties": false + "additionalProperties":false }, - "options": { - "description": "Additional project options.", - "type": ["object", "null"], - "properties": { - "refresh": { - "description": "Set to \"always\" to refresh the state before performing a Pulumi operation.", - "type": "string", - "const": "always" + "options":{ + "description":"Additional project options.", + "type":[ + "object", + "null" + ], + "properties":{ + "refresh":{ + "description":"Set to \"always\" to refresh the state before performing a Pulumi operation.", + "type":"string", + "const":"always" } }, - "additionalProperties": false + "additionalProperties":false }, - "template": { - "title": "ProjectTemplate", - "description": "ProjectTemplate is a Pulumi project template manifest.", - "type": ["object", "null"], - "properties": { - "description": { - "description": "Description of the template.", - "type": ["string", "null"] + "template":{ + "title":"ProjectTemplate", + "description":"ProjectTemplate is a Pulumi project template manifest.", + "type":[ + "object", + "null" + ], + "properties":{ + "description":{ + "description":"Description of the template.", + "type":[ + "string", + "null" + ] }, - "quickstart": { - "description": "Quickstart contains optional text to be displayed after template creation.", - "type":["string", "null"] + "quickstart":{ + "description":"Quickstart contains optional text to be displayed after template creation.", + "type":[ + "string", + "null" + ] }, - "important": { - "description": "Important indicates the template is important and should be listed by default.", - "type": ["boolean", "null"] + "important":{ + "description":"Important indicates the template is important and should be listed by default.", + "type":[ + "boolean", + "null" + ] }, - "config": { - "description": "Config to apply to each stack in the project.", - "type": ["object", "null"], - "additionalProperties": { - "properties": { - "description": { - "description": "Description of the config.", - "type": ["string", "null"] + "config":{ + "description":"Config to apply to each stack in the project.", + "type":[ + "object", + "null" + ], + "additionalProperties":{ + "properties":{ + "description":{ + "description":"Description of the config.", + "type":[ + "string", + "null" + ] }, - "default": { - "description": "Default value of the config." + "default":{ + "description":"Default value of the config." }, - "secret": { - "description": "Boolean indicating if the configuration is labeled as a secret.", - "type": ["boolean", "null"] + "secret":{ + "description":"Boolean indicating if the configuration is labeled as a secret.", + "type":[ + "boolean", + "null" + ] } } } } }, - "additionalProperties": false + "additionalProperties":false }, - "plugins": { - "description": "Override for the plugin selection. Intended for use in developing pulumi plugins.", - "type": "object", - "properties": { - "providers": { - "description": "Plugins for resource providers.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" + "plugins":{ + "description":"Override for the plugin selection. Intended for use in developing pulumi plugins.", + "type":"object", + "properties":{ + "providers":{ + "description":"Plugins for resource providers.", + "type":"array", + "items":{ + "$ref":"#/$defs/pluginOptions" } }, - "analyzers": { - "description": "Plugins for policy analyzers.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" + "analyzers":{ + "description":"Plugins for policy analyzers.", + "type":"array", + "items":{ + "$ref":"#/$defs/pluginOptions" } }, - "languages": { - "description": "Plugins for languages.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" + "languages":{ + "description":"Plugins for languages.", + "type":"array", + "items":{ + "$ref":"#/$defs/pluginOptions" } } } } }, - "required": ["name", "runtime"], - "additionalProperties": true, - "$defs": { - "pluginOptions": { - "title": "PluginOptions", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "Name of the plugin" + "required":[ + "name", + "runtime" + ], + "additionalProperties":true, + "$defs":{ + "pluginOptions":{ + "title":"PluginOptions", + "type":"object", + "properties":{ + "name":{ + "type":"string", + "description":"Name of the plugin" + }, + "path":{ + "type":"string", + "description":"Path to the plugin folder" + }, + "version":{ + "type":"string", + "description":"Version of the plugin, if not set, will match any version the engine requests." + } + } + }, + "simpleConfigType":{ + "title":"SimpleConfigType", + "enum":[ + "string", + "integer", + "boolean", + "array" + ] + }, + "configItemsType":{ + "type":"object", + "required":[ + "type" + ], + "properties":{ + "type":{ + "oneOf":[ + { + "$ref":"#/$defs/simpleConfigType" + }, + { + "$ref":"#/$defs/configItemsType" + } + ] + }, + "items":{ + "$ref":"#/$defs/configItemsType" + } + }, + "if":{ + "properties":{ + "type":{ + "const":"array" + } + } + }, + "then":{ + "required":[ + "items" + ] + } + }, + "configTypeDeclaration":{ + "title":"ConfigTypeDeclaration", + "type":"object", + "additionalProperties":false, + "required":[ + "type" + ], + "if":{ + "properties":{ + "type":{ + "const":"array" + } + } + }, + "then":{ + "required":[ + "items" + ] + }, + "properties":{ + "type":{ + "$ref":"#/$defs/simpleConfigType" + }, + "items":{ + "$ref":"#/$defs/configItemsType" + }, + "description":{ + "type":"string" }, - "path": { - "type": "string", - "description": "Path to the plugin folder" + "secret":{ + "type":"boolean" }, - "version": { - "type": "string", - "description": "Version of the plugin, if not set, will match any version the engine requests." + "default":{ + } } } diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index 527add6033b9..3e20b9c35cbe 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -3,8 +3,10 @@ package workspace import ( "encoding/json" "io/ioutil" + "os" "testing" + "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" @@ -41,7 +43,7 @@ func TestProjectRuntimeInfoRoundtripYAML(t *testing.T) { doTest(json.Marshal, json.Unmarshal) } -func TestProjectValidation(t *testing.T) { +func TestProjectValidationForNameAndRuntime(t *testing.T) { t.Parallel() var err error @@ -62,6 +64,86 @@ func TestProjectValidation(t *testing.T) { assert.NoError(t, err) } +func TestProjectValidationFailsForIncorrectDefaultValueType(t *testing.T) { + t.Parallel() + project := Project{Name: "test", Runtime: NewProjectRuntimeInfo("dotnet", nil)} + invalidConfig := make(map[string]ProjectConfigType) + invalidConfig["instanceSize"] = ProjectConfigType{ + Type: "integer", + Items: nil, + Default: "hello", + } + + project.Config = invalidConfig + err := project.Validate() + assert.Contains(t, + err.Error(), + "The default value specified for configuration key 'instanceSize' is not of the expected type 'integer'") + + invalidValues := make([]interface{}, 0) + invalidValues = append(invalidValues, "hello") + // default value here has type array + // config type specified is array> + // should fail! + invalidConfigWithArray := make(map[string]ProjectConfigType) + invalidConfigWithArray["values"] = ProjectConfigType{ + Type: "array", + Items: &ProjectConfigItemsType{ + Type: "array", + Items: &ProjectConfigItemsType{ + Type: "string", + }, + }, + Default: invalidValues, + } + project.Config = invalidConfigWithArray + err = project.Validate() + assert.Error(t, err, "There is a validation error") + assert.Contains(t, + err.Error(), + "The default value specified for configuration key 'values' is not of the expected type 'array>'") +} + +func TestProjectValidationSucceedsForCorrectDefaultValueType(t *testing.T) { + t.Parallel() + project := Project{Name: "test", Runtime: NewProjectRuntimeInfo("dotnet", nil)} + validConfig := make(map[string]ProjectConfigType) + validConfig["instanceSize"] = ProjectConfigType{ + Type: "integer", + Items: nil, + Default: 1, + } + + project.Config = validConfig + err := project.Validate() + assert.NoError(t, err, "There should be no validation error") + + // validValues = ["hello"] + validValues := make([]interface{}, 0) + validValues = append(validValues, "hello") + // validValuesArray = [["hello"]] + validValuesArray := make([]interface{}, 0) + validValuesArray = append(validValuesArray, validValues) + + // default value here has type array> + // config type specified is also array> + // should succeed + validConfigWithArray := make(map[string]ProjectConfigType) + validConfigWithArray["values"] = ProjectConfigType{ + Type: "array", + Items: &ProjectConfigItemsType{ + Type: "array", + Items: &ProjectConfigItemsType{ + Type: "string", + }, + }, + Default: validValuesArray, + } + project.Config = validConfigWithArray + err = project.Validate() + assert.NoError(t, err, "There should be no validation error") +} + func TestProjectLoadJSON(t *testing.T) { t.Parallel() @@ -125,44 +207,489 @@ func TestProjectLoadJSON(t *testing.T) { assert.Equal(t, "", proj.Main) } -func TestProjectLoadYAML(t *testing.T) { +func deleteFile(t *testing.T, file *os.File) { + if file != nil { + err := os.Remove(file.Name()) + assert.NoError(t, err, "Error while deleting file") + } +} + +func loadProjectFromText(t *testing.T, content string) (*Project, error) { + tmp, err := ioutil.TempFile("", "*.yaml") + assert.NoError(t, err) + path := tmp.Name() + err = ioutil.WriteFile(path, []byte(content), 0600) + assert.NoError(t, err) + defer deleteFile(t, tmp) + return LoadProject(path) +} + +func loadProjectStackFromText(t *testing.T, project *Project, content string) (*ProjectStack, error) { + tmp, err := ioutil.TempFile("", "*.yaml") + assert.NoError(t, err) + path := tmp.Name() + err = ioutil.WriteFile(path, []byte(content), 0600) + assert.NoError(t, err) + defer deleteFile(t, tmp) + return LoadProjectStack(project, path) +} + +func TestProjectLoadsConfigSchemas(t *testing.T) { t.Parallel() + projectContent := ` +name: test +runtime: dotnet +config: + integerSchemaFull: + type: integer + description: a very important value + default: 1 + integerSchemaSimple: 20 + textSchemaFull: + type: string + default: t3.micro + textSchemaSimple: t4.large + booleanSchemaFull: + type: boolean + default: true + booleanSchemaSimple: false + simpleArrayOfStrings: + type: array + items: + type: string + default: [hello] + arrayOfArrays: + type: array + items: + type: array + items: + type: string + ` + + project, err := loadProjectFromText(t, projectContent) + assert.NoError(t, err, "Should be able to load the project") + assert.Equal(t, 8, len(project.Config), "There are 8 config type definition") + // full integer config schema + integerSchemFull, ok := project.Config["integerSchemaFull"] + assert.True(t, ok, "should be able to read integerSchemaFull") + assert.Equal(t, "integer", integerSchemFull.Type) + assert.Equal(t, "a very important value", integerSchemFull.Description) + assert.Equal(t, 1, integerSchemFull.Default) + assert.Nil(t, integerSchemFull.Items, "Primtive config type doesn't have an items type") + + integerSchemaSimple, ok := project.Config["integerSchemaSimple"] + assert.True(t, ok, "should be able to read integerSchemaSimple") + assert.Equal(t, "integer", integerSchemaSimple.Type, "integer type is inferred correctly") + assert.Equal(t, 20, integerSchemaSimple.Default, "Default integer value is parsed correctly") + + textSchemaFull, ok := project.Config["textSchemaFull"] + assert.True(t, ok, "should be able to read textSchemaFull") + assert.Equal(t, "string", textSchemaFull.Type) + assert.Equal(t, "t3.micro", textSchemaFull.Default) + assert.Equal(t, "", textSchemaFull.Description) + + textSchemaSimple, ok := project.Config["textSchemaSimple"] + assert.True(t, ok, "should be able to read textSchemaSimple") + assert.Equal(t, "string", textSchemaSimple.Type) + assert.Equal(t, "t4.large", textSchemaSimple.Default) + + booleanSchemaFull, ok := project.Config["booleanSchemaFull"] + assert.True(t, ok, "should be able to read booleanSchemaFull") + assert.Equal(t, "boolean", booleanSchemaFull.Type) + assert.Equal(t, true, booleanSchemaFull.Default) + + booleanSchemaSimple, ok := project.Config["booleanSchemaSimple"] + assert.True(t, ok, "should be able to read booleanSchemaSimple") + assert.Equal(t, "boolean", booleanSchemaSimple.Type) + assert.Equal(t, false, booleanSchemaSimple.Default) + + simpleArrayOfStrings, ok := project.Config["simpleArrayOfStrings"] + assert.True(t, ok, "should be able to read simpleArrayOfStrings") + assert.Equal(t, "array", simpleArrayOfStrings.Type) + assert.NotNil(t, simpleArrayOfStrings.Items) + assert.Equal(t, "string", simpleArrayOfStrings.Items.Type) + arrayValues := simpleArrayOfStrings.Default.([]interface{}) + assert.Equal(t, "hello", arrayValues[0]) + + arrayOfArrays, ok := project.Config["arrayOfArrays"] + assert.True(t, ok, "should be able to read arrayOfArrays") + assert.Equal(t, "array", arrayOfArrays.Type) + assert.NotNil(t, arrayOfArrays.Items) + assert.Equal(t, "array", arrayOfArrays.Items.Type) + assert.NotNil(t, arrayOfArrays.Items.Items) + assert.Equal(t, "string", arrayOfArrays.Items.Items.Type) +} - writeAndLoad := func(str string) (*Project, error) { - tmp, err := ioutil.TempFile("", "*.yaml") - assert.NoError(t, err) - path := tmp.Name() - err = ioutil.WriteFile(path, []byte(str), 0600) - assert.NoError(t, err) - return LoadProject(path) - } +func getConfigValue(t *testing.T, stackConfig config.Map, key string) string { + parsedKey, err := config.ParseKey(key) + assert.NoErrorf(t, err, "There should be no error parsing the config key '%v'", key) + configValue, foundValue := stackConfig[parsedKey] + assert.Truef(t, foundValue, "Couldn't find a value for config key %v", key) + value, valueError := configValue.Value(config.NopDecrypter) + assert.NoErrorf(t, valueError, "Error while getting the value for key %v", key) + return value +} + +func TestStackConfigIsInheritedFromProjectConfig(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + instanceSize: t3.micro + instanceCount: 20 + protect: true` + + projectStackYaml := ` +config: + test:instanceSize: t4.large` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NoError(t, configError, "Config override should be valid") + + assert.Equal(t, 3, len(stack.Config), "Stack config now has three values") + // value of instanceSize is overwritten from the stack + assert.Equal(t, "t4.large", getConfigValue(t, stack.Config, "test:instanceSize")) + // instanceCount and protect are inherited from the project + assert.Equal(t, "20", getConfigValue(t, stack.Config, "test:instanceCount")) + assert.Equal(t, "true", getConfigValue(t, stack.Config, "test:protect")) +} + +func TestNamespacedConfigValuesAreInheritedCorrectly(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + aws:region: us-west-1 + instanceSize: t3.micro` + + projectStackYaml := ` +config: + test:instanceSize: t4.large` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NoError(t, configError, "Config override should be valid") + + assert.Equal(t, 2, len(stack.Config), "Stack config now has three values") + // value of instanceSize is overwritten from the stack + assert.Equal(t, "t4.large", getConfigValue(t, stack.Config, "test:instanceSize")) + // aws:region is namespaced and is inherited from the project + assert.Equal(t, "us-west-1", getConfigValue(t, stack.Config, "aws:region")) +} + +func TestLoadingStackConfigWithoutNamespacingTheProject(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + aws:region: us-west-1 + instanceSize: t3.micro` + + projectStackYaml := ` +config: + instanceSize: t4.large` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NoError(t, configError, "Config override should be valid") + + assert.Equal(t, 2, len(stack.Config), "Stack config now has three values") + // value of instanceSize is overwritten from the stack + assert.Equal(t, "t4.large", getConfigValue(t, stack.Config, "test:instanceSize")) + // aws:region is namespaced and is inherited from the project + assert.Equal(t, "us-west-1", getConfigValue(t, stack.Config, "aws:region")) +} + +func TestStackConfigErrorsWhenStackValueIsNotCorrectlyTyped(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + values: + type: array + items: + type: string + default: [value]` + + projectStackYaml := ` +config: + test:values: someValue +` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, configError.Error(), "Stack 'dev' with configuration key 'values' must be of type 'array'") +} + +func TestLoadingConfigIsRewrittenToStackConfigDir(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: ./some/path` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + assert.Equal(t, "./some/path", project.StackConfigDir, "Stack config dir is read from the config property") + assert.Equal(t, 0, len(project.Config), "Config should be empty") +} + +func TestDefningBothConfigAndStackConfigDirErrorsOut(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: ./some/path +stackConfigDir: ./some/other/path` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.Nil(t, project, "Should NOT be able to load the project") + assert.NotNil(t, projectError, "There is a project error") + assert.Contains(t, projectError.Error(), "Should not use both config and stackConfigDir") +} + +func TestConfigObjectAndStackConfigDirSuccessfullyLoadProject(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +stackConfigDir: ./some/other/path +config: + value: hello +` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.Nil(t, projectError, "There is no error") + assert.NotNil(t, project, "The project can be loaded correctly") + assert.Equal(t, "./some/other/path", project.StackConfigDir) + assert.Equal(t, 1, len(project.Config), "there is one config value") +} + +func TestStackConfigIntegerTypeIsCorrectlyValidated(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + importantNumber: + type: integer +` + + projectStackYamlValid := ` +config: + test:importantNumber: 20 +` + + projectStackYamlInvalid := ` +config: + test:importantNumber: hello +` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYamlValid) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NoError(t, configError, "there should no config type error") + + invalidStackConfig, stackError := loadProjectStackFromText(t, project, projectStackYamlInvalid) + assert.NoError(t, stackError, "Should be able to read the stack") + configError = ValidateStackConfigAndApplyProjectConfig("dev", project, invalidStackConfig.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, + configError.Error(), + "Stack 'dev' with configuration key 'importantNumber' must be of type 'integer'") +} + +func TestStackConfigErrorsWhenMissingStackValueForConfigTypeWithNoDefault(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + values: + type: array + items: + type: string` + + projectStackYaml := `` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, configError.Error(), "Stack 'dev' is missing configuration value 'values'") +} + +func TestStackConfigErrorsWhenMissingTwoStackValueForConfigTypeWithNoDefault(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + another: + type: string + values: + type: array + items: + type: string` + + projectStackYaml := `` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, configError.Error(), "Stack 'dev' is missing configuration values 'another' and 'values'") +} + +func TestStackConfigErrorsWhenMissingMultipleStackValueForConfigTypeWithNoDefault(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + hello: + type: integer + values: + type: array + items: + type: string + world: + type: string` + + projectStackYaml := `` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, configError.Error(), "Stack 'dev' is missing configuration values 'hello', 'values' and 'world'") +} + +func TestStackConfigErrorsWhenUsingConfigValuesNotDefinedByProject(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + hello: + type: integer` + + projectStackYaml := ` +config: + hello: 21 + world: 42` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + expectedErrorMsg := "Stack 'dev' uses configuration value 'world' which is not defined by the project configuration" + assert.Contains(t, configError.Error(), expectedErrorMsg) +} + +func TestStackConfigErrorsWhenUsingMultipleConfigValuesNotDefinedByProject(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + hello: + type: integer` + + projectStackYaml := ` +config: + hello: 21 + world: 42 + another: 42` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + expectedErrorMsg := "Stack 'dev' uses configuration values 'another' and 'world'" + + " which are not defined by the project configuration" + assert.Contains(t, configError.Error(), expectedErrorMsg) +} + +func TestStackConfigDoesNotErrorWhenProjectHasNotDefinedConfig(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet` + + projectStackYaml := ` +config: + hello: 21 + world: 42 + another: 42` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.Nil(t, configError, "there should not be a config type error") +} + +func TestProjectLoadYAML(t *testing.T) { + t.Parallel() // Test wrong type - _, err := writeAndLoad("\"hello\"") - assert.Contains(t, err.Error(), "expected project to be an object, was 'string'") + _, err := loadProjectFromText(t, "\"hello\"") + assert.Contains(t, err.Error(), "expected project to be an object") // Test bad key - _, err = writeAndLoad("4: hello") + _, err = loadProjectFromText(t, "4: hello") assert.Contains(t, err.Error(), "expected only string keys, got '%!s(int=4)'") // Test nested bad key - _, err = writeAndLoad("hello:\n 6: bad") + _, err = loadProjectFromText(t, "hello:\n 6: bad") assert.Contains(t, err.Error(), "expected only string keys, got '%!s(int=6)'") // Test lack of name - _, err = writeAndLoad("{}") + _, err = loadProjectFromText(t, "{}") assert.Contains(t, err.Error(), "project is missing a 'name' attribute") // Test bad name - _, err = writeAndLoad("name:") + _, err = loadProjectFromText(t, "name:") assert.Contains(t, err.Error(), "project is missing a non-empty string 'name' attribute") // Test missing runtime - _, err = writeAndLoad("name: project") + _, err = loadProjectFromText(t, "name: project") assert.Contains(t, err.Error(), "project is missing a 'runtime' attribute") // Test other schema errors - _, err = writeAndLoad("name: project\nruntime: 4") + _, err = loadProjectFromText(t, "name: project\nruntime: 4") // These can vary in order, so contains not equals check expected := []string{ "3 errors occurred:", @@ -173,7 +700,7 @@ func TestProjectLoadYAML(t *testing.T) { assert.Contains(t, err.Error(), e) } - _, err = writeAndLoad("name: project\nruntime: test\nbackend: 4\nmain: {}") + _, err = loadProjectFromText(t, "name: project\nruntime: test\nbackend: 4\nmain: {}") expected = []string{ "2 errors occurred:", "* #/main: expected string or null, but got object", @@ -183,13 +710,13 @@ func TestProjectLoadYAML(t *testing.T) { } // Test success - proj, err := writeAndLoad("name: project\nruntime: test") + proj, err := loadProjectFromText(t, "name: project\nruntime: test") assert.NoError(t, err) assert.Equal(t, tokens.PackageName("project"), proj.Name) assert.Equal(t, "test", proj.Runtime.Name()) // Test null optionals should work - proj, err = writeAndLoad("name: project\nruntime: test\ndescription:\nmain: null\nbackend:\n") + proj, err = loadProjectFromText(t, "name: project\nruntime: test\ndescription:\nmain: null\nbackend:\n") assert.NoError(t, err) assert.Nil(t, proj.Description) assert.Equal(t, "", proj.Main) diff --git a/tests/config_test.go b/tests/config_test.go index bc76b09a0d2a..7232e28fe271 100644 --- a/tests/config_test.go +++ b/tests/config_test.go @@ -15,6 +15,7 @@ package tests import ( + "encoding/json" "os" "path/filepath" "regexp" @@ -274,3 +275,70 @@ $` e.RunCommand("pulumi", "stack", "rm", "--yes") }) } + +func TestBasicConfigGetRetrievedValueFromProject(t *testing.T) { + t.Parallel() + + e := ptesting.NewEnvironment(t) + defer func() { + if !t.Failed() { + e.DeleteEnvironment() + } + }() + + pulumiProject := ` +name: pulumi-test +runtime: go +config: + first-value: + type: string + default: first` + + integration.CreatePulumiRepo(e, pulumiProject) + e.SetBackend(e.LocalURL()) + e.RunCommand("pulumi", "stack", "init", "test") + stdout, _ := e.RunCommand("pulumi", "config", "get", "first-value") + assert.Equal(t, "first", strings.Trim(stdout, "\r\n")) +} + +func TestConfigGetRetrievedValueFromBothStackAndProjectUsingJson(t *testing.T) { + t.Parallel() + + e := ptesting.NewEnvironment(t) + defer func() { + if !t.Failed() { + e.DeleteEnvironment() + } + }() + + pulumiProject := ` +name: pulumi-test +runtime: go +config: + first-value: + type: string + default: first + second-value: + type: string + third-value: + type: array + items: + type: string + default: [third]` + + integration.CreatePulumiRepo(e, pulumiProject) + e.SetBackend(e.LocalURL()) + e.RunCommand("pulumi", "stack", "init", "test") + e.RunCommand("pulumi", "config", "set", "second-value", "second") + stdout, _ := e.RunCommand("pulumi", "config", "--json") + // check that stdout is an array containing 2 objects + var config map[string]interface{} + jsonError := json.Unmarshal([]byte(stdout), &config) + assert.Nil(t, jsonError) + assert.Equal(t, 3, len(config)) + assert.Equal(t, "first", config["pulumi-test:first-value"].(map[string]interface{})["value"]) + assert.Equal(t, "second", config["pulumi-test:second-value"].(map[string]interface{})["value"]) + thirdValue := config["pulumi-test:third-value"].(map[string]interface{}) + assert.Equal(t, "[\"third\"]", thirdValue["value"]) + assert.Equal(t, []interface{}{"third"}, thirdValue["objectValue"]) +} diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 28886b972bb2..996d287a6cbc 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -146,10 +146,7 @@ func TestStackInitValidation(t *testing.T) { stdout, stderr := e.RunCommandExpectError("pulumi", "stack", "init", "valid-name") assert.Equal(t, "", stdout) - assert.Contains(t, stderr, "error: could not get cloud url: could not load current project: could not unmarshal '") - // Project path is printed between these two 's, but due to macos having multiple paths for temporary - // files we don't try to check the path in the test. - assert.Contains(t, stderr, "': invalid YAML file: yaml: line 1: did not find expected key") + assert.Contains(t, stderr, "invalid YAML file") }) } @@ -165,10 +162,12 @@ func TestConfigSave(t *testing.T) { // Initialize an empty stack. path := filepath.Join(e.RootPath, "Pulumi.yaml") - err := (&workspace.Project{ + project := workspace.Project{ Name: "testing-config", Runtime: workspace.NewProjectRuntimeInfo("nodejs", nil), - }).Save(path) + } + + err := project.Save(path) assert.NoError(t, err) e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) e.RunCommand("pulumi", "stack", "init", "testing-2") @@ -211,9 +210,9 @@ func TestConfigSave(t *testing.T) { assert.Equal(t, v, dv) } - testStack1, err := workspace.LoadProjectStack(filepath.Join(e.CWD, "Pulumi.testing-1.yaml")) + testStack1, err := workspace.LoadProjectStack(&project, filepath.Join(e.CWD, "Pulumi.testing-1.yaml")) assert.NoError(t, err) - testStack2, err := workspace.LoadProjectStack(filepath.Join(e.CWD, "Pulumi.testing-2.yaml")) + testStack2, err := workspace.LoadProjectStack(&project, filepath.Join(e.CWD, "Pulumi.testing-2.yaml")) assert.NoError(t, err) assert.Equal(t, 2, len(testStack1.Config)) @@ -986,7 +985,6 @@ func TestDestroyStackRef(t *testing.T) { e.RunCommand("yarn", "install") e.RunCommand("pulumi", "up", "--skip-preview", "--yes") - e.CWD = os.TempDir() e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "-s", "dev") }