From 6c6b5cec7eaf62e1093894cfc907bf4957a2fc50 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Tue, 20 Sep 2022 20:13:45 +0200 Subject: [PATCH 01/32] Initial work on project-level config (MVP) --- .gitignore | 2 +- pkg/cmd/pulumi/preview.go | 32 ++ pkg/cmd/pulumi/up.go | 34 ++ sdk/go/common/workspace/loaders.go | 5 + sdk/go/common/workspace/paths.go | 15 - sdk/go/common/workspace/project.go | 68 +++- sdk/go/common/workspace/project.json | 413 +++++++++++++++--------- sdk/go/common/workspace/project_test.go | 82 ++++- 8 files changed, 482 insertions(+), 169 deletions(-) diff --git a/.gitignore b/.gitignore index 74db3dbf4bc5..7d423498ba59 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ venv/ **/.idea/ *.iml - +.yarn # VSCode creates this binary when running tests in the debugger **/debug.test diff --git a/pkg/cmd/pulumi/preview.go b/pkg/cmd/pulumi/preview.go index 9f6b88e975a6..b0c924c83bb4 100644 --- a/pkg/cmd/pulumi/preview.go +++ b/pkg/cmd/pulumi/preview.go @@ -16,6 +16,7 @@ package main import ( "bytes" + "encoding/json" "errors" "fmt" @@ -26,6 +27,7 @@ import ( "github.com/pulumi/pulumi/pkg/v3/engine" "github.com/pulumi/pulumi/sdk/v3/go/common/diag" "github.com/pulumi/pulumi/sdk/v3/go/common/resource" + "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" @@ -153,6 +155,36 @@ func newPreviewCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + for projectConfigKey, projectConfigType := range proj.Config { + key := config.MustMakeKey(string(proj.Name), projectConfigKey) + + _, found, _ := cfg.Config.Get(key, true) + hasDefault := projectConfigType.Default != nil + if !found && !hasDefault { + stackName := s.Ref().Name().String() + missingConfigError := fmt.Errorf( + "Stack '%v' missing configuration value '%v'", + stackName, + projectConfigKey) + return result.FromError(missingConfigError) + } + + if !found && hasDefault { + // not found at the stack level + // but has a default value at the project level + // assign the value to the stack + configValueJson, jsonError := json.Marshal(projectConfigType.Default) + if jsonError != nil { + return result.FromError(jsonError) + } + configValue := config.NewValue(string(configValueJson)) + setError := cfg.Config.Set(key, configValue, true) + if setError != nil { + return result.FromError(setError) + } + } + } + targetURNs := []resource.URN{} for _, t := range targets { targetURNs = append(targetURNs, resource.URN(t)) diff --git a/pkg/cmd/pulumi/up.go b/pkg/cmd/pulumi/up.go index 0612cf11a879..b416bef70cfc 100644 --- a/pkg/cmd/pulumi/up.go +++ b/pkg/cmd/pulumi/up.go @@ -16,6 +16,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -111,6 +112,39 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + for projectConfigKey, projectConfigType := range proj.Config { + key, err := config.ParseKey(projectConfigKey) + if err != nil { + return result.FromError(fmt.Errorf("parsing project config key: %w", err)) + } + + associatedStackValue, found, _ := cfg.Config.Get(key, true) + hasDefault := projectConfigType.Default != nil + if !found && !hasDefault { + stackName := s.Ref().Name().String() + missingConfigError := fmt.Errorf( + "Stack '%v' missing configuration value '%v'", + stackName, + associatedStackValue) + return result.FromError(missingConfigError) + } + + if !found && hasDefault { + // not found at the stack level + // but has a default value at the project level + // assign the value to the stack + configValueJson, jsonError := json.Marshal(projectConfigType.Default) + if jsonError != nil { + return result.FromError(jsonError) + } + configValue := config.NewValue(string(configValueJson)) + setError := cfg.Config.Set(key, configValue, true) + if setError != nil { + return result.FromError(setError) + } + } + } + targetURNs := []resource.URN{} snap, err := s.Snapshot(ctx) if err != nil { diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index e035adda5ab3..c97583f3ea24 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -110,6 +110,11 @@ func (singleton *projectLoader) load(path string) (*Project, error) { return nil, err } + err = project.Validate() + if err != nil { + return nil, err + } + singleton.internal[path] = &project return &project, nil } diff --git a/sdk/go/common/workspace/paths.go b/sdk/go/common/workspace/paths.go index dc60082f819f..6de3556fb4f2 100644 --- a/sdk/go/common/workspace/paths.go +++ b/sdk/go/common/workspace/paths.go @@ -101,25 +101,10 @@ func DetectProjectStackPath(stackName tokens.QName) (string, error) { fileName := fmt.Sprintf("%s.%s%s", ProjectFile, qnameFileName(stackName), filepath.Ext(projPath)) - // Back compat: StackConfigDir used to be called Config. - configValue, hasConfigValue := proj.Config.(string) - hasConfigValue = hasConfigValue && configValue != "" - if proj.StackConfigDir != "" { - // If config and stackConfigDir are both set return an error - if hasConfigValue { - return "", fmt.Errorf("can not set `config` and `stackConfigDir`, remove the `config` entry") - } - return filepath.Join(filepath.Dir(projPath), proj.StackConfigDir, fileName), nil } - // Back compat: If StackConfigDir is not present and Config is given and it's a non-empty string use it - // for the stacks directory. - if hasConfigValue { - return filepath.Join(filepath.Dir(projPath), configValue, fileName), nil - } - return filepath.Join(filepath.Dir(projPath), fileName), nil } diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 307391f70f05..4412fbf4f346 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -97,6 +97,17 @@ type Plugins struct { Analyzers []PluginOptions `json:"analyzers,omitempty" yaml:"analyzers,omitempty"` } +type ProjectConfigItemsType struct { + Type string `json:"type" yaml:"type"` + Items *ProjectConfigItemsType `json:"items" yaml:"items"` +} + +type ProjectConfigType struct { + Type string `json:"type" yaml:"type"` + Items *ProjectConfigItemsType `json:"items" yaml:"items"` + Default interface{} `json:"default" yaml:"default"` +} + // Project is a Pulumi project manifest. // // We explicitly add yaml tags (instead of using the default behavior from https://github.com/ghodss/yaml which works @@ -122,7 +133,7 @@ type Project struct { License *string `json:"license,omitempty" yaml:"license,omitempty"` // Config has been renamed to StackConfigDir. - Config interface{} `json:"config,omitempty" yaml:"config,omitempty"` + Config map[string]ProjectConfigType `json:"config,omitempty" yaml:"config,omitempty"` // StackConfigDir indicates where to store the Pulumi..yaml files, combined with the folder // Pulumi.yaml is in. @@ -226,6 +237,49 @@ func ValidateProject(raw interface{}) error { return errs } +func InterFullTypeName(typeName string, itemsType *ProjectConfigItemsType) string { + if itemsType != nil { + return fmt.Sprintf("array<%v>", InterFullTypeName(itemsType.Type, itemsType.Items)) + } + + return typeName +} + +func ValidateConfigValue(typeName string, itemsType *ProjectConfigItemsType, value interface{}) bool { + + if typeName == "string" { + _, ok := value.(string) + return ok + } + + if typeName == "integer" { + _, ok := value.(int) + return ok + } + + if typeName == "boolean" { + _, ok := value.(bool) + return ok + } + + items, isArray := value.([]interface{}) + + if !isArray || itemsType == nil { + return false + } + + // validate each item + for _, item := range items { + itemType := itemsType.Type + underlyingItems := itemsType.Items + if !ValidateConfigValue(itemType, underlyingItems, item) { + return false + } + } + + return true +} + func (proj *Project) Validate() error { if proj.Name == "" { return errors.New("project is missing a 'name' attribute") @@ -234,6 +288,18 @@ func (proj *Project) Validate() error { return errors.New("project is missing a 'runtime' attribute") } + for configKey, configType := range proj.Config { + if configType.Default != nil { + // when the default value is specified, validate it against the type + if !ValidateConfigValue(configType.Type, configType.Items, configType.Default) { + inferredTypeName := InterFullTypeName(configType.Type, configType.Items) + return errors.Errorf("The default value specified for configuration key '%v' is not of the expected type '%v'", + configKey, + inferredTypeName) + } + } + } + return nil } diff --git a/sdk/go/common/workspace/project.json b/sdk/go/common/workspace/project.json index 58ef899280bf..de1ad20cf4d2 100644 --- a/sdk/go/common/workspace/project.json +++ b/sdk/go/common/workspace/project.json @@ -5,176 +5,287 @@ "description": "A schema for Pulumi project files.", "type": "object", "properties": { - "name": { - "description": "Name of the project containing alphanumeric characters, hyphens, underscores, and periods.", + "name": { + "description": "Name of the project containing alphanumeric characters, hyphens, underscores, and periods.", + "type": "string", + "minLength": 1 + }, + "description": { + "description": "Description of the project.", + "type": [ + "string", + "null" + ] + }, + "author": { + "description": "Author is an optional author that created this project.", + "type": [ + "string", + "null" + ] + }, + "website": { + "description": "Website is an optional website for additional info about this project.", + "type": [ + "string", + "null" + ] + }, + "license": { + "description": "License is the optional license governing this project's usage.", + "type": [ + "string", + "null" + ] + }, + "runtime": { + "title": "ProjectRuntimeInfo", + "oneOf": [ + { + "title": "Name", "type": "string", "minLength": 1 - }, - "description": { - "description": "Description of the project.", - "type": ["string", "null"] - }, - "author": { - "description": "Author is an optional author that created this project.", - "type": ["string", "null"] - }, - "website": { - "description": "Website is an optional website for additional info about this project.", - "type": ["string", "null"] - }, - "license": { - "description": "License is the optional license governing this project's usage.", - "type": ["string", "null"] - }, - "runtime": { - "title": "ProjectRuntimeInfo", - "oneOf": [ - { - "title": "Name", - "type": "string", - "minLength": 1 - }, - { - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string", - "minLength": 1 - }, - "options": { - "title": "Options", - "type": "object", - "additionalProperties": true - } - }, - "additionalProperties": false - } - ] - }, - "main": { - "description": "Path to the Pulumi program. The default is the working directory.", - "type": ["string", "null"] - }, - "config": { - "description": "Config directory relative to the location of Pulumi.yaml.", - "type": ["string", "null"], - "deprecated": true - }, - "stackConfigDir": { - "description": "Config directory location relative to the location of Pulumi.yaml.", - "type": ["string", "null"] - }, - "backend": { - "description": "Backend of the project.", - "type": ["object", "null"], + }, + { + "type": "object", "properties": { - "url": { - "description": "URL is optional field to explicitly set backend url", - "type": "string" - } + "name": { + "title": "Name", + "type": "string", + "minLength": 1 + }, + "options": { + "title": "Options", + "type": "object", + "additionalProperties": true + } }, "additionalProperties": false + } + ] + }, + "main": { + "description": "Path to the Pulumi program. The default is the working directory.", + "type": [ + "string", + "null" + ] + }, + "config": { + "description": "A map of configuration keys to their types", + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/configTypeDeclaration" + } + }, + "stackConfigDir": { + "description": "Config directory location relative to the location of Pulumi.yaml.", + "type": [ + "string", + "null" + ] + }, + "backend": { + "description": "Backend of the project.", + "type": [ + "object", + "null" + ], + "properties": { + "url": { + "description": "URL is optional field to explicitly set backend url", + "type": "string" + } }, - "options": { - "description": "Additional project options.", - "type": ["object", "null"], - "properties": { - "refresh": { - "description": "Boolean indicating whether to refresh the state before performing a Pulumi operation.", - "type": "boolean", - "default": false - } - }, - "additionalProperties": false + "additionalProperties": false + }, + "options": { + "description": "Additional project options.", + "type": [ + "object", + "null" + ], + "properties": { + "refresh": { + "description": "Boolean indicating whether to refresh the state before performing a Pulumi operation.", + "type": "boolean", + "default": false + } }, - "template": { - "title": "ProjectTemplate", - "description": "ProjectTemplate is a Pulumi project template manifest.", - "type": ["object", "null"], - "properties": { + "additionalProperties": false + }, + "template": { + "title": "ProjectTemplate", + "description": "ProjectTemplate is a Pulumi project template manifest.", + "type": [ + "object", + "null" + ], + "properties": { + "description": { + "description": "Description of the template.", + "type": [ + "string", + "null" + ] + }, + "quickstart": { + "description": "Quickstart contains optional text to be displayed after template creation.", + "type": [ + "string", + "null" + ] + }, + "important": { + "description": "Important indicates the template is important and should be listed by default.", + "type": [ + "boolean", + "null" + ] + }, + "config": { + "description": "Config to apply to each stack in the project.", + "type": [ + "object", + "null" + ], + "additionalProperties": { + "properties": { "description": { - "description": "Description of the template.", - "type": ["string", "null"] - }, - "quickstart": { - "description": "Quickstart contains optional text to be displayed after template creation.", - "type":["string", "null"] + "description": "Description of the config.", + "type": [ + "string", + "null" + ] }, - "important": { - "description": "Important indicates the template is important and should be listed by default.", - "type": ["boolean", "null"] + "default": { + "description": "Default value of the config." }, - "config": { - "description": "Config to apply to each stack in the project.", - "type": ["object", "null"], - "additionalProperties": { - "properties": { - "description": { - "description": "Description of the config.", - "type": ["string", "null"] - }, - "default": { - "description": "Default value of the config." - }, - "secret": { - "description": "Boolean indicating if the configuration is labeled as a secret.", - "type": ["boolean", "null"] - } - } - } + "secret": { + "description": "Boolean indicating if the configuration is labeled as a secret.", + "type": [ + "boolean", + "null" + ] } - }, - "additionalProperties": false + } + } + } }, - "plugins": { - "description": "Override for the plugin selection. Intended for use in developing pulumi plugins.", - "type": "object", - "properties": { - "providers": { - "description": "Plugins for resource providers.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" - } - }, - "analyzers": { - "description": "Plugins for policy analyzers.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" - } - }, - "languages": { - "description": "Plugins for languages.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" - } - } + "additionalProperties": false + }, + "plugins": { + "description": "Override for the plugin selection. Intended for use in developing pulumi plugins.", + "type": "object", + "properties": { + "providers": { + "description": "Plugins for resource providers.", + "type": "array", + "items": { + "$ref": "#/$defs/pluginOptions" + } + }, + "analyzers": { + "description": "Plugins for policy analyzers.", + "type": "array", + "items": { + "$ref": "#/$defs/pluginOptions" } + }, + "languages": { + "description": "Plugins for languages.", + "type": "array", + "items": { + "$ref": "#/$defs/pluginOptions" + } + } } + } }, - "required": ["name", "runtime"], + "required": [ + "name", + "runtime" + ], "additionalProperties": true, "$defs": { - "pluginOptions": { - "title": "PluginOptions", - "type": "object", + "pluginOptions": { + "title": "PluginOptions", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the plugin" + }, + "path": { + "type": "string", + "description": "Path to the plugin folder" + }, + "version": { + "type": "string", + "description": "Version of the plugin, if not set, will match any version the engine requests." + } + } + }, + "simpleConfigType": { + "title": "SimpleConfigType", + "enum": ["string","integer", "boolean", "array"] + }, + "configItemsType": { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "oneOf": [ + { "$ref": "#/$defs/simpleConfigType" }, + { "$ref": "#/$defs/configItemsType" } + ] + }, + "items": { + "$ref": "#/$defs/configItemsType" + } + }, + "if": { "properties": { - "name": { - "type": "string", - "description": "Name of the plugin" - }, - "path": { - "type": "string", - "description": "Path to the plugin folder" - }, - "version": { - "type": "string", - "description": "Version of the plugin, if not set, will match any version the engine requests." - } + "type": { + "const": "array" + } + } + }, + "then": { + "required": ["items"] + } + }, + "configTypeDeclaration": { + "title": "ConfigTypeDeclaration", + "type": "object", + "additionalProperties": false, + "required": [ + "type" + ], + "if": { + "properties": { + "type": { + "const": "array" } + } + }, + "then": { + "required": ["items"] + }, + "properties": { + "type": { + "$ref": "#/$defs/simpleConfigType" + }, + "items": { + "$ref": "#/$defs/configItemsType" + }, + "secret": { + "type": "boolean" + }, + "default": { + + } } + } } } \ No newline at end of file diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index dd6aeb52413a..3b2ba26c6045 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -41,7 +41,7 @@ func TestProjectRuntimeInfoRoundtripYAML(t *testing.T) { doTest(json.Marshal, json.Unmarshal) } -func TestProjectValidation(t *testing.T) { +func TestProjectValidationForNameAndRuntime(t *testing.T) { t.Parallel() var err error @@ -62,6 +62,86 @@ func TestProjectValidation(t *testing.T) { assert.NoError(t, err) } +func TestProjectValidationFailsForIncorrectDefaultValueType(t *testing.T) { + t.Parallel() + project := Project{Name: "test", Runtime: NewProjectRuntimeInfo("dotnet", nil)} + invalidConfig := make(map[string]ProjectConfigType) + invalidConfig["instanceSize"] = ProjectConfigType{ + Type: "integer", + Items: nil, + Default: "hello", + } + + project.Config = invalidConfig + err := project.Validate() + assert.Contains(t, + err.Error(), + "The default value specified for configuration key 'instanceSize' is not of the expected type 'integer'") + + invalidValues := make([]interface{}, 0) + invalidValues = append(invalidValues, "hello") + // default value here has type array + // config type specified is array> + // should fail! + invalidConfigWithArray := make(map[string]ProjectConfigType) + invalidConfigWithArray["values"] = ProjectConfigType{ + Type: "array", + Items: &ProjectConfigItemsType{ + Type: "array", + Items: &ProjectConfigItemsType{ + Type: "string", + }, + }, + Default: invalidValues, + } + project.Config = invalidConfigWithArray + err = project.Validate() + assert.Error(t, err, "There is a validation error") + assert.Contains(t, + err.Error(), + "The default value specified for configuration key 'values' is not of the expected type 'array>'") +} + +func TestProjectValidationSucceedsForCorrectDefaultValueType(t *testing.T) { + t.Parallel() + project := Project{Name: "test", Runtime: NewProjectRuntimeInfo("dotnet", nil)} + validConfig := make(map[string]ProjectConfigType) + validConfig["instanceSize"] = ProjectConfigType{ + Type: "integer", + Items: nil, + Default: 1, + } + + project.Config = validConfig + err := project.Validate() + assert.NoError(t, err, "There should be no validation error") + + // validValues = ["hello"] + validValues := make([]interface{}, 0) + validValues = append(validValues, "hello") + // validValuesArray = [["hello"]] + validValuesArray := make([]interface{}, 0) + validValuesArray = append(validValuesArray, validValues) + + // default value here has type array> + // config type specified is also array> + // should succeed + validConfigWithArray := make(map[string]ProjectConfigType) + validConfigWithArray["values"] = ProjectConfigType{ + Type: "array", + Items: &ProjectConfigItemsType{ + Type: "array", + Items: &ProjectConfigItemsType{ + Type: "string", + }, + }, + Default: validValuesArray, + } + project.Config = validConfigWithArray + err = project.Validate() + assert.NoError(t, err, "There should be no validation error") +} + func TestProjectLoadJSON(t *testing.T) { t.Parallel() From 683f7e1941be2331416e04a5f9f0ebb74e404601 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Tue, 20 Sep 2022 23:29:16 +0200 Subject: [PATCH 02/32] Implement stack config validation and inheritance --- pkg/cmd/pulumi/preview.go | 34 ++--------------- pkg/cmd/pulumi/up.go | 36 ++---------------- pkg/cmd/pulumi/util.go | 52 ++++++++++++++++++++++++++ sdk/go/common/resource/config/value.go | 5 +++ sdk/go/common/workspace/project.go | 6 +-- 5 files changed, 68 insertions(+), 65 deletions(-) diff --git a/pkg/cmd/pulumi/preview.go b/pkg/cmd/pulumi/preview.go index b0c924c83bb4..3e0a4ff5453e 100644 --- a/pkg/cmd/pulumi/preview.go +++ b/pkg/cmd/pulumi/preview.go @@ -16,7 +16,6 @@ package main import ( "bytes" - "encoding/json" "errors" "fmt" @@ -27,7 +26,6 @@ import ( "github.com/pulumi/pulumi/pkg/v3/engine" "github.com/pulumi/pulumi/sdk/v3/go/common/diag" "github.com/pulumi/pulumi/sdk/v3/go/common/resource" - "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" @@ -155,34 +153,10 @@ func newPreviewCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } - for projectConfigKey, projectConfigType := range proj.Config { - key := config.MustMakeKey(string(proj.Name), projectConfigKey) - - _, found, _ := cfg.Config.Get(key, true) - hasDefault := projectConfigType.Default != nil - if !found && !hasDefault { - stackName := s.Ref().Name().String() - missingConfigError := fmt.Errorf( - "Stack '%v' missing configuration value '%v'", - stackName, - projectConfigKey) - return result.FromError(missingConfigError) - } - - if !found && hasDefault { - // not found at the stack level - // but has a default value at the project level - // assign the value to the stack - configValueJson, jsonError := json.Marshal(projectConfigType.Default) - if jsonError != nil { - return result.FromError(jsonError) - } - configValue := config.NewValue(string(configValueJson)) - setError := cfg.Config.Set(key, configValue, true) - if setError != nil { - return result.FromError(setError) - } - } + stackName := s.Ref().Name().String() + configValidationError := validateStackConfigWithProject(stackName, proj, cfg) + if configValidationError != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configValidationError)) } targetURNs := []resource.URN{} diff --git a/pkg/cmd/pulumi/up.go b/pkg/cmd/pulumi/up.go index b416bef70cfc..483f5495751e 100644 --- a/pkg/cmd/pulumi/up.go +++ b/pkg/cmd/pulumi/up.go @@ -16,7 +16,6 @@ package main import ( "context" - "encoding/json" "errors" "fmt" "io/ioutil" @@ -112,37 +111,10 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } - for projectConfigKey, projectConfigType := range proj.Config { - key, err := config.ParseKey(projectConfigKey) - if err != nil { - return result.FromError(fmt.Errorf("parsing project config key: %w", err)) - } - - associatedStackValue, found, _ := cfg.Config.Get(key, true) - hasDefault := projectConfigType.Default != nil - if !found && !hasDefault { - stackName := s.Ref().Name().String() - missingConfigError := fmt.Errorf( - "Stack '%v' missing configuration value '%v'", - stackName, - associatedStackValue) - return result.FromError(missingConfigError) - } - - if !found && hasDefault { - // not found at the stack level - // but has a default value at the project level - // assign the value to the stack - configValueJson, jsonError := json.Marshal(projectConfigType.Default) - if jsonError != nil { - return result.FromError(jsonError) - } - configValue := config.NewValue(string(configValueJson)) - setError := cfg.Config.Set(key, configValue, true) - if setError != nil { - return result.FromError(setError) - } - } + stackName := s.Ref().Name().String() + configValidationError := validateStackConfigWithProject(stackName, proj, cfg) + if configValidationError != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configValidationError)) } targetURNs := []resource.URN{} diff --git a/pkg/cmd/pulumi/util.go b/pkg/cmd/pulumi/util.go index 89b2a8d2b874..ffd3ecbd26d2 100644 --- a/pkg/cmd/pulumi/util.go +++ b/pkg/cmd/pulumi/util.go @@ -941,3 +941,55 @@ func log3rdPartySecretsProviderDecryptionEvent(ctx context.Context, backend back } } } + +func validateStackConfigWithProject( + stackName string, + project *workspace.Project, + stackConfig backend.StackConfiguration) error { + for projectConfigKey, projectConfigType := range project.Config { + key := config.MustMakeKey(string(project.Name), projectConfigKey) + stackValue, found, _ := stackConfig.Config.Get(key, true) + hasDefault := projectConfigType.Default != nil + if !found && !hasDefault { + missingConfigError := fmt.Errorf( + "Stack '%v' missing configuration value '%v'", + stackName, + projectConfigKey) + return missingConfigError + } else if !found && hasDefault { + // not found at the stack level + // but has a default value at the project level + // assign the value to the stack + configValueJson, jsonError := json.Marshal(projectConfigType.Default) + if jsonError != nil { + return jsonError + } + configValue := config.NewValue(string(configValueJson)) + setError := stackConfig.Config.Set(key, configValue, true) + if setError != nil { + return setError + } + } else { + // found value on the stack level + // retrieve it and validate it against + // the config defined at the project level + content, contentError := stackValue.MarshalValue() + if contentError != nil { + return contentError + } + + if !workspace.ValidateConfigValue(projectConfigType.Type, projectConfigType.Items, content) { + typeName := workspace.InferFullTypeName(projectConfigType.Type, projectConfigType.Items) + validationError := fmt.Errorf( + "Stack '%v' with configuration key '%v' must of of type '%v'", + stackName, + projectConfigKey, + typeName) + + return validationError + } + } + } + + return nil +} diff --git a/sdk/go/common/resource/config/value.go b/sdk/go/common/resource/config/value.go index e6d19d6fee8e..0be5451b61cd 100644 --- a/sdk/go/common/resource/config/value.go +++ b/sdk/go/common/resource/config/value.go @@ -139,6 +139,11 @@ func (c Value) ToObject() (interface{}, error) { return c.unmarshalObjectJSON() } +// MarshalValue returns the underlying content of the config value +func (c Value) MarshalValue() (interface{}, error) { + return c.marshalValue() +} + func (c Value) MarshalJSON() ([]byte, error) { v, err := c.marshalValue() if err != nil { diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 4412fbf4f346..585c87cec248 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -237,9 +237,9 @@ func ValidateProject(raw interface{}) error { return errs } -func InterFullTypeName(typeName string, itemsType *ProjectConfigItemsType) string { +func InferFullTypeName(typeName string, itemsType *ProjectConfigItemsType) string { if itemsType != nil { - return fmt.Sprintf("array<%v>", InterFullTypeName(itemsType.Type, itemsType.Items)) + return fmt.Sprintf("array<%v>", InferFullTypeName(itemsType.Type, itemsType.Items)) } return typeName @@ -292,7 +292,7 @@ func (proj *Project) Validate() error { if configType.Default != nil { // when the default value is specified, validate it against the type if !ValidateConfigValue(configType.Type, configType.Items, configType.Default) { - inferredTypeName := InterFullTypeName(configType.Type, configType.Items) + inferredTypeName := InferFullTypeName(configType.Type, configType.Items) return errors.Errorf("The default value specified for configuration key '%v' is not of the expected type '%v'", configKey, inferredTypeName) From f6593767631b06a5338b5dc75c9b87fbd04c2731 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Tue, 20 Sep 2022 23:33:03 +0200 Subject: [PATCH 03/32] Add description to the config type declaration --- sdk/go/common/workspace/project.go | 7 ++++--- sdk/go/common/workspace/project.json | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 585c87cec248..6bb01d485738 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -103,9 +103,10 @@ type ProjectConfigItemsType struct { } type ProjectConfigType struct { - Type string `json:"type" yaml:"type"` - Items *ProjectConfigItemsType `json:"items" yaml:"items"` - Default interface{} `json:"default" yaml:"default"` + Type string `json:"type" yaml:"type"` + Description string `json:"description" yaml:"description"` + Items *ProjectConfigItemsType `json:"items" yaml:"items"` + Default interface{} `json:"default" yaml:"default"` } // Project is a Pulumi project manifest. diff --git a/sdk/go/common/workspace/project.json b/sdk/go/common/workspace/project.json index de1ad20cf4d2..4f77189833ea 100644 --- a/sdk/go/common/workspace/project.json +++ b/sdk/go/common/workspace/project.json @@ -279,6 +279,9 @@ "items": { "$ref": "#/$defs/configItemsType" }, + "description": { + "type": "string" + }, "secret": { "type": "boolean" }, From 966e96f86e18393e7572eaf8baaf482b63993517 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Wed, 21 Sep 2022 15:52:14 +0200 Subject: [PATCH 04/32] Apply project config to stack config when appropriate --- pkg/backend/filestate/crypto.go | 13 +- pkg/backend/httpstate/crypto.go | 13 +- pkg/cmd/pulumi/config.go | 195 +++++++++++++++--- pkg/cmd/pulumi/crypto.go | 8 +- pkg/cmd/pulumi/crypto_cloud.go | 13 +- pkg/cmd/pulumi/destroy.go | 11 +- pkg/cmd/pulumi/import.go | 6 +- pkg/cmd/pulumi/logs.go | 6 +- pkg/cmd/pulumi/new.go | 7 +- pkg/cmd/pulumi/preview.go | 11 +- pkg/cmd/pulumi/refresh.go | 5 +- .../pulumi/stack_change_secrets_provider.go | 17 +- pkg/cmd/pulumi/stack_init.go | 8 +- pkg/cmd/pulumi/stack_rename.go | 4 +- pkg/cmd/pulumi/stack_rm.go | 2 +- pkg/cmd/pulumi/up.go | 16 +- pkg/cmd/pulumi/util.go | 52 ----- pkg/cmd/pulumi/watch.go | 5 +- sdk/go/auto/local_workspace.go | 7 +- sdk/go/common/workspace/loaders.go | 6 +- sdk/go/common/workspace/paths.go | 14 +- sdk/go/common/workspace/paths_test.go | 2 +- sdk/go/common/workspace/project.go | 8 +- 23 files changed, 289 insertions(+), 140 deletions(-) diff --git a/pkg/backend/filestate/crypto.go b/pkg/backend/filestate/crypto.go index ef17d4af3927..b4b3c677f2eb 100644 --- a/pkg/backend/filestate/crypto.go +++ b/pkg/backend/filestate/crypto.go @@ -26,15 +26,16 @@ func NewPassphraseSecretsManager(stackName tokens.Name, configFile string, rotatePassphraseSecretsProvider bool) (secrets.Manager, error) { contract.Assertf(stackName != "", "stackName %s", "!= \"\"") + project, path, err := workspace.DetectProjectStackPath(stackName.Q()) + if err != nil { + return nil, err + } + if configFile == "" { - f, err := workspace.DetectProjectStackPath(stackName.Q()) - if err != nil { - return nil, err - } - configFile = f + configFile = path } - info, err := workspace.LoadProjectStack(configFile) + info, err := workspace.LoadProjectStack(project, configFile) if err != nil { return nil, err } diff --git a/pkg/backend/httpstate/crypto.go b/pkg/backend/httpstate/crypto.go index d826b1f8abac..4e8023ff2c05 100644 --- a/pkg/backend/httpstate/crypto.go +++ b/pkg/backend/httpstate/crypto.go @@ -25,15 +25,16 @@ import ( func NewServiceSecretsManager(s Stack, stackName tokens.Name, configFile string) (secrets.Manager, error) { contract.Assertf(stackName != "", "stackName %s", "!= \"\"") + project, path, err := workspace.DetectProjectStackPath(stackName.Q()) + if err != nil { + return nil, err + } + if configFile == "" { - f, err := workspace.DetectProjectStackPath(stackName.Q()) - if err != nil { - return nil, err - } - configFile = f + configFile = path } - info, err := workspace.LoadProjectStack(configFile) + info, err := workspace.LoadProjectStack(project, configFile) if err != nil { return nil, err } diff --git a/pkg/cmd/pulumi/config.go b/pkg/cmd/pulumi/config.go index 9fbda04d1231..3bc5ee518fa8 100644 --- a/pkg/cmd/pulumi/config.go +++ b/pkg/cmd/pulumi/config.go @@ -39,6 +39,10 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) +type StackConfigOptions struct { + applyProjectConfig bool +} + func newConfigCmd() *cobra.Command { var stack string var showSecrets bool @@ -57,12 +61,18 @@ func newConfigCmd() *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + + if err != nil { + return err + } + stack, err := requireStack(ctx, stack, true, opts, true /*setCurrent*/) if err != nil { return err } - return listConfig(ctx, stack, showSecrets, jsonOut) + return listConfig(ctx, project, stack, showSecrets, jsonOut) }), } @@ -106,6 +116,12 @@ func newConfigCopyCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + + if err != nil { + return err + } + // Get current stack and ensure that it is a different stack to the destination stack currentStack, err := requireStack(ctx, *stack, false, opts, true /*setCurrent*/) if err != nil { @@ -114,7 +130,7 @@ func newConfigCopyCmd(stack *string) *cobra.Command { if currentStack.Ref().Name().String() == destinationStackName { return errors.New("current stack and destination stack are the same") } - currentProjectStack, err := loadProjectStack(currentStack) + currentProjectStack, err := loadProjectStack(project, currentStack) if err != nil { return err } @@ -124,7 +140,7 @@ func newConfigCopyCmd(stack *string) *cobra.Command { if err != nil { return err } - destinationProjectStack, err := loadProjectStack(destinationStack) + destinationProjectStack, err := loadProjectStack(project, destinationStack) if err != nil { return err } @@ -302,7 +318,12 @@ func newConfigRmCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } - s, err := requireStack(ctx, *stack, true, opts, true /*setCurrent*/) + project, _, err := readProject() + if err != nil { + return err + } + + stack, err := requireStack(ctx, *stack, true, opts, true /*setCurrent*/) if err != nil { return err } @@ -312,7 +333,7 @@ func newConfigRmCmd(stack *string) *cobra.Command { return fmt.Errorf("invalid configuration key: %w", err) } - ps, err := loadProjectStack(s) + ps, err := loadProjectStack(project, stack) if err != nil { return err } @@ -322,7 +343,7 @@ func newConfigRmCmd(stack *string) *cobra.Command { return err } - return saveProjectStack(s, ps) + return saveProjectStack(stack, ps) }), } rmCmd.PersistentFlags().BoolVar( @@ -351,12 +372,17 @@ func newConfigRmAllCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } - s, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/) + project, _, err := readProject() if err != nil { return err } - ps, err := loadProjectStack(s) + stack, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/) + if err != nil { + return err + } + + ps, err := loadProjectStack(project, stack) if err != nil { return err } @@ -373,7 +399,7 @@ func newConfigRmAllCmd(stack *string) *cobra.Command { } } - return saveProjectStack(s, ps) + return saveProjectStack(stack, ps) }), } rmAllCmd.PersistentFlags().BoolVar( @@ -395,6 +421,11 @@ func newConfigRefreshCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + if err != nil { + return err + } + // Ensure the stack exists. s, err := requireStack(ctx, *stack, false, opts, false /*setCurrent*/) if err != nil { @@ -411,7 +442,7 @@ func newConfigRefreshCmd(stack *string) *cobra.Command { return err } - ps, err := workspace.LoadProjectStack(configPath) + ps, err := workspace.LoadProjectStack(project, configPath) if err != nil { return err } @@ -480,6 +511,12 @@ func newConfigSetCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + + if err != nil { + return err + } + // Ensure the stack exists. s, err := requireStack(ctx, *stack, true, opts, true /*setCurrent*/) if err != nil { @@ -538,7 +575,7 @@ func newConfigSetCmd(stack *string) *cobra.Command { } } - ps, err := loadProjectStack(s) + ps, err := loadProjectStack(project, s) if err != nil { return err } @@ -591,13 +628,18 @@ func newConfigSetAllCmd(stack *string) *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + if err != nil { + return err + } + // Ensure the stack exists. - s, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/) + stack, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/) if err != nil { return err } - ps, err := loadProjectStack(s) + ps, err := loadProjectStack(project, stack) if err != nil { return err } @@ -620,7 +662,7 @@ func newConfigSetAllCmd(stack *string) *cobra.Command { if err != nil { return err } - c, cerr := getStackEncrypter(s) + c, cerr := getStackEncrypter(stack) if cerr != nil { return cerr } @@ -636,7 +678,7 @@ func newConfigSetAllCmd(stack *string) *cobra.Command { } } - return saveProjectStack(s, ps) + return saveProjectStack(stack, ps) }), } @@ -680,16 +722,17 @@ var stackConfigFile string func getProjectStackPath(stack backend.Stack) (string, error) { if stackConfigFile == "" { - return workspace.DetectProjectStackPath(stack.Ref().Name().Q()) + _, path, err := workspace.DetectProjectStackPath(stack.Ref().Name().Q()) + return path, err } return stackConfigFile, nil } -func loadProjectStack(stack backend.Stack) (*workspace.ProjectStack, error) { +func loadProjectStack(project *workspace.Project, stack backend.Stack) (*workspace.ProjectStack, error) { if stackConfigFile == "" { return workspace.DetectProjectStack(stack.Ref().Name().Q()) } - return workspace.LoadProjectStack(stackConfigFile) + return workspace.LoadProjectStack(project, stackConfigFile) } func saveProjectStack(stack backend.Stack, ps *workspace.ProjectStack) error { @@ -741,12 +784,89 @@ type configValueJSON struct { Secret bool `json:"secret"` } -func listConfig(ctx context.Context, stack backend.Stack, showSecrets bool, jsonOut bool) error { - ps, err := loadProjectStack(stack) +func validateStackConfigAndApplyProjectConfig( + stackName string, + project *workspace.Project, + stackConfig config.Map) error { + for projectConfigKey, projectConfigType := range project.Config { + key := config.MustMakeKey(string(project.Name), projectConfigKey) + stackValue, found, _ := stackConfig.Get(key, true) + hasDefault := projectConfigType.Default != nil + if !found && !hasDefault { + missingConfigError := fmt.Errorf( + "Stack '%v' missing configuration value '%v'", + stackName, + projectConfigKey) + return missingConfigError + } else if !found && hasDefault { + // not found at the stack level + // but has a default value at the project level + // assign the value to the stack + var configValue config.Value + + if projectConfigType.Type == "array" { + // for array types, JSON-ify the default value + configValueJson, jsonError := json.Marshal(projectConfigType.Default) + if jsonError != nil { + return jsonError + } + configValue = config.NewObjectValue(string(configValueJson)) + + } else { + // for primitive types + // pass the values as is + configValueContent := fmt.Sprintf("%v", projectConfigType.Default) + configValue = config.NewValue(configValueContent) + } + + setError := stackConfig.Set(key, configValue, true) + if setError != nil { + return setError + } + } else { + // found value on the stack level + // retrieve it and validate it against + // the config defined at the project level + content, contentError := stackValue.MarshalValue() + if contentError != nil { + return contentError + } + + if !workspace.ValidateConfigValue(projectConfigType.Type, projectConfigType.Items, content) { + typeName := workspace.InferFullTypeName(projectConfigType.Type, projectConfigType.Items) + validationError := fmt.Errorf( + "Stack '%v' with configuration key '%v' must of of type '%v'", + stackName, + projectConfigKey, + typeName) + + return validationError + } + } + } + + return nil +} + +func listConfig(ctx context.Context, + project *workspace.Project, + stack backend.Stack, + showSecrets bool, + jsonOut bool) error { + + ps, err := loadProjectStack(project, stack) if err != nil { return err } + stackName := stack.Ref().Name().String() + // when listing configuration values + // also show values coming from the project + configError := validateStackConfigAndApplyProjectConfig(stackName, project, ps.Config) + if configError != nil { + return configError + } + cfg := ps.Config // By default, we will use a blinding decrypter to show "[secret]". If requested, display secrets in plaintext. @@ -827,10 +947,19 @@ func listConfig(ctx context.Context, stack backend.Stack, showSecrets bool, json } func getConfig(ctx context.Context, stack backend.Stack, key config.Key, path, jsonOut bool) error { - ps, err := loadProjectStack(stack) + project, _, err := readProject() + if err != nil { + return err + } + ps, err := loadProjectStack(project, stack) if err != nil { return err } + stackName := stack.Ref().Name().String() + configError := validateStackConfigAndApplyProjectConfig(stackName, project, ps.Config) + if configError != nil { + return configError + } cfg := ps.Config @@ -922,17 +1051,32 @@ func looksLikeSecret(k config.Key, v string) bool { // it is uses instead of the default configuration file for the stack func getStackConfiguration( ctx context.Context, stack backend.Stack, - sm secrets.Manager) (backend.StackConfiguration, error) { + sm secrets.Manager, + options StackConfigOptions) (backend.StackConfiguration, error) { var cfg config.Map - workspaceStack, err := loadProjectStack(stack) + project, _, err := readProject() + defaultStackConfig := backend.StackConfiguration{} + if err != nil { + return defaultStackConfig, err + } + + workspaceStack, err := loadProjectStack(project, stack) if err != nil || workspaceStack == nil { // On first run or the latest configuration is unavailable, fallback to check the project's configuration cfg, err = backend.GetLatestConfiguration(ctx, stack) if err != nil { - return backend.StackConfiguration{}, fmt.Errorf( + return defaultStackConfig, fmt.Errorf( "stack configuration could not be loaded from either Pulumi.yaml or the backend: %w", err) } } else { + if options.applyProjectConfig { + stackName := stack.Ref().Name().String() + configErr := validateStackConfigAndApplyProjectConfig(stackName, project, workspaceStack.Config) + if configErr != nil { + return defaultStackConfig, configErr + } + } + cfg = workspaceStack.Config } @@ -948,8 +1092,9 @@ func getStackConfiguration( crypter, err := sm.Decrypter() if err != nil { - return backend.StackConfiguration{}, fmt.Errorf("getting configuration decrypter: %w", err) + return defaultStackConfig, fmt.Errorf("getting configuration decrypter: %w", err) } + return backend.StackConfiguration{ Config: cfg, Decrypter: crypter, diff --git a/pkg/cmd/pulumi/crypto.go b/pkg/cmd/pulumi/crypto.go index db9145cf38fe..d59c3c8fc854 100644 --- a/pkg/cmd/pulumi/crypto.go +++ b/pkg/cmd/pulumi/crypto.go @@ -45,7 +45,13 @@ func getStackDecrypter(s backend.Stack) (config.Decrypter, error) { } func getStackSecretsManager(s backend.Stack) (secrets.Manager, error) { - ps, err := loadProjectStack(s) + project, _, err := readProject() + + if err != nil { + return nil, err + } + + ps, err := loadProjectStack(project, s) if err != nil { return nil, err } diff --git a/pkg/cmd/pulumi/crypto_cloud.go b/pkg/cmd/pulumi/crypto_cloud.go index a2e5b7feaaf4..392a462daf01 100644 --- a/pkg/cmd/pulumi/crypto_cloud.go +++ b/pkg/cmd/pulumi/crypto_cloud.go @@ -26,16 +26,15 @@ import ( func newCloudSecretsManager(stackName tokens.Name, configFile, secretsProvider string) (secrets.Manager, error) { contract.Assertf(stackName != "", "stackName %s", "!= \"\"") - + proj, path, err := workspace.DetectProjectStackPath(stackName.Q()) + if err != nil { + return nil, err + } if configFile == "" { - f, err := workspace.DetectProjectStackPath(stackName.Q()) - if err != nil { - return nil, err - } - configFile = f + configFile = path } - info, err := workspace.LoadProjectStack(configFile) + info, err := workspace.LoadProjectStack(proj, configFile) if err != nil { return nil, err } diff --git a/pkg/cmd/pulumi/destroy.go b/pkg/cmd/pulumi/destroy.go index ec284cd2f2f8..c7194c66d756 100644 --- a/pkg/cmd/pulumi/destroy.go +++ b/pkg/cmd/pulumi/destroy.go @@ -157,11 +157,20 @@ func newDestroyCmd() *cobra.Command { sm = snap.SecretsManager } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ + applyProjectConfig: true, + }) + if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) + } + stackName := s.Ref().Name().String() + configError := validateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config) + if configError != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configError)) } + targetUrns := []resource.URN{} for _, t := range *targets { targetUrns = append(targetUrns, snap.GlobUrn(resource.URN(t))...) diff --git a/pkg/cmd/pulumi/import.go b/pkg/cmd/pulumi/import.go index 42a5403db06a..32c253cdfe85 100644 --- a/pkg/cmd/pulumi/import.go +++ b/pkg/cmd/pulumi/import.go @@ -492,7 +492,11 @@ func newImportCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ + // we don't need project config here + applyProjectConfig: false, + }) + if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } diff --git a/pkg/cmd/pulumi/logs.go b/pkg/cmd/pulumi/logs.go index a29113167b9c..cb7b435de4d3 100644 --- a/pkg/cmd/pulumi/logs.go +++ b/pkg/cmd/pulumi/logs.go @@ -67,7 +67,11 @@ func newLogsCmd() *cobra.Command { return fmt.Errorf("getting secrets manager: %w", err) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ + // we don't need project config here + applyProjectConfig: false, + }) + if err != nil { return fmt.Errorf("getting stack configuration: %w", err) } diff --git a/pkg/cmd/pulumi/new.go b/pkg/cmd/pulumi/new.go index 64d7ab5b37fd..ddbda315e0c7 100644 --- a/pkg/cmd/pulumi/new.go +++ b/pkg/cmd/pulumi/new.go @@ -675,7 +675,12 @@ func stackInit( // saveConfig saves the config for the stack. func saveConfig(stack backend.Stack, c config.Map) error { - ps, err := loadProjectStack(stack) + project, _, err := readProject() + if err != nil { + return err + } + + ps, err := loadProjectStack(project, stack) if err != nil { return err } diff --git a/pkg/cmd/pulumi/preview.go b/pkg/cmd/pulumi/preview.go index 3e0a4ff5453e..b082604fa20e 100644 --- a/pkg/cmd/pulumi/preview.go +++ b/pkg/cmd/pulumi/preview.go @@ -148,17 +148,14 @@ func newPreviewCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ + applyProjectConfig: true, + }) + if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } - stackName := s.Ref().Name().String() - configValidationError := validateStackConfigWithProject(stackName, proj, cfg) - if configValidationError != nil { - return result.FromError(fmt.Errorf("validating stack config: %w", configValidationError)) - } - targetURNs := []resource.URN{} for _, t := range targets { targetURNs = append(targetURNs, resource.URN(t)) diff --git a/pkg/cmd/pulumi/refresh.go b/pkg/cmd/pulumi/refresh.go index 05e5a2c5f6fc..d3119ed48e01 100644 --- a/pkg/cmd/pulumi/refresh.go +++ b/pkg/cmd/pulumi/refresh.go @@ -146,7 +146,10 @@ func newRefreshCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ + applyProjectConfig: true, + }) + if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } diff --git a/pkg/cmd/pulumi/stack_change_secrets_provider.go b/pkg/cmd/pulumi/stack_change_secrets_provider.go index 007728061511..13570c13feb9 100644 --- a/pkg/cmd/pulumi/stack_change_secrets_provider.go +++ b/pkg/cmd/pulumi/stack_change_secrets_provider.go @@ -25,6 +25,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" "github.com/spf13/cobra" ) @@ -58,6 +59,12 @@ func newStackChangeSecretsProviderCmd() *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + project, _, err := readProject() + + if err != nil { + return err + } + // Validate secrets provider type if err := validateSecretsProvider(args[0]); err != nil { return err @@ -68,7 +75,7 @@ func newStackChangeSecretsProviderCmd() *cobra.Command { if err != nil { return err } - currentProjectStack, err := loadProjectStack(currentStack) + currentProjectStack, err := loadProjectStack(project, currentStack) if err != nil { return err } @@ -96,7 +103,7 @@ func newStackChangeSecretsProviderCmd() *cobra.Command { // Fixup the checkpoint fmt.Printf("Migrating old configuration and state to new secrets provider\n") - return migrateOldConfigAndCheckpointToNewSecretsProvider(ctx, currentStack, currentConfig, decrypter) + return migrateOldConfigAndCheckpointToNewSecretsProvider(ctx, project, currentStack, currentConfig, decrypter) }), } @@ -107,7 +114,9 @@ func newStackChangeSecretsProviderCmd() *cobra.Command { return cmd } -func migrateOldConfigAndCheckpointToNewSecretsProvider(ctx context.Context, currentStack backend.Stack, +func migrateOldConfigAndCheckpointToNewSecretsProvider(ctx context.Context, + project *workspace.Project, + currentStack backend.Stack, currentConfig config.Map, decrypter config.Decrypter) error { // The order of operations here should be to load the secrets manager current stack // Get the newly created secrets manager for the stack @@ -129,7 +138,7 @@ func migrateOldConfigAndCheckpointToNewSecretsProvider(ctx context.Context, curr } // Reload the project stack after the new secretsProvider is in place - reloadedProjectStack, err := loadProjectStack(currentStack) + reloadedProjectStack, err := loadProjectStack(project, currentStack) if err != nil { return err } diff --git a/pkg/cmd/pulumi/stack_init.go b/pkg/cmd/pulumi/stack_init.go index a2a6b351866d..585629e712ee 100644 --- a/pkg/cmd/pulumi/stack_init.go +++ b/pkg/cmd/pulumi/stack_init.go @@ -72,6 +72,10 @@ func newStackInitCmd() *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + proj, _, err := readProject() + if err != nil { + return err + } b, err := currentBackend(ctx, opts) if err != nil { return err @@ -129,13 +133,13 @@ func newStackInitCmd() *cobra.Command { if err != nil { return err } - copyProjectStack, err := loadProjectStack(copyStack) + copyProjectStack, err := loadProjectStack(proj, copyStack) if err != nil { return err } // get the project for the newly created stack - newProjectStack, err := loadProjectStack(newStack) + newProjectStack, err := loadProjectStack(proj, newStack) if err != nil { return err } diff --git a/pkg/cmd/pulumi/stack_rename.go b/pkg/cmd/pulumi/stack_rename.go index 56f9715fb622..3c4283895adb 100644 --- a/pkg/cmd/pulumi/stack_rename.go +++ b/pkg/cmd/pulumi/stack_rename.go @@ -55,7 +55,7 @@ func newStackRenameCmd() *cobra.Command { if err != nil { return err } - oldConfigPath, err := workspace.DetectProjectStackPath(s.Ref().Name().Q()) + _, oldConfigPath, err := workspace.DetectProjectStackPath(s.Ref().Name().Q()) if err != nil { return err } @@ -66,7 +66,7 @@ func newStackRenameCmd() *cobra.Command { if err != nil { return err } - newConfigPath, err := workspace.DetectProjectStackPath(newStackRef.Name().Q()) + _, newConfigPath, err := workspace.DetectProjectStackPath(newStackRef.Name().Q()) if err != nil { return err } diff --git a/pkg/cmd/pulumi/stack_rm.go b/pkg/cmd/pulumi/stack_rm.go index ddf602931e6b..307af9609846 100644 --- a/pkg/cmd/pulumi/stack_rm.go +++ b/pkg/cmd/pulumi/stack_rm.go @@ -86,7 +86,7 @@ func newStackRmCmd() *cobra.Command { if !preserveConfig { // Blow away stack specific settings if they exist. If we get an ENOENT error, ignore it. - if path, err := workspace.DetectProjectStackPath(s.Ref().Name().Q()); err == nil { + if _, path, err := workspace.DetectProjectStackPath(s.Ref().Name().Q()); err == nil { if err = os.Remove(path); err != nil && !os.IsNotExist(err) { return result.FromError(err) } diff --git a/pkg/cmd/pulumi/up.go b/pkg/cmd/pulumi/up.go index 483f5495751e..e1216b7bf2f0 100644 --- a/pkg/cmd/pulumi/up.go +++ b/pkg/cmd/pulumi/up.go @@ -106,17 +106,14 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ + applyProjectConfig: true, + }) + if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } - stackName := s.Ref().Name().String() - configValidationError := validateStackConfigWithProject(stackName, proj, cfg) - if configValidationError != nil { - return result.FromError(fmt.Errorf("validating stack config: %w", configValidationError)) - } - targetURNs := []resource.URN{} snap, err := s.Snapshot(ctx) if err != nil { @@ -338,7 +335,10 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ + applyProjectConfig: true, + }) + if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } diff --git a/pkg/cmd/pulumi/util.go b/pkg/cmd/pulumi/util.go index ffd3ecbd26d2..89b2a8d2b874 100644 --- a/pkg/cmd/pulumi/util.go +++ b/pkg/cmd/pulumi/util.go @@ -941,55 +941,3 @@ func log3rdPartySecretsProviderDecryptionEvent(ctx context.Context, backend back } } } - -func validateStackConfigWithProject( - stackName string, - project *workspace.Project, - stackConfig backend.StackConfiguration) error { - for projectConfigKey, projectConfigType := range project.Config { - key := config.MustMakeKey(string(project.Name), projectConfigKey) - stackValue, found, _ := stackConfig.Config.Get(key, true) - hasDefault := projectConfigType.Default != nil - if !found && !hasDefault { - missingConfigError := fmt.Errorf( - "Stack '%v' missing configuration value '%v'", - stackName, - projectConfigKey) - return missingConfigError - } else if !found && hasDefault { - // not found at the stack level - // but has a default value at the project level - // assign the value to the stack - configValueJson, jsonError := json.Marshal(projectConfigType.Default) - if jsonError != nil { - return jsonError - } - configValue := config.NewValue(string(configValueJson)) - setError := stackConfig.Config.Set(key, configValue, true) - if setError != nil { - return setError - } - } else { - // found value on the stack level - // retrieve it and validate it against - // the config defined at the project level - content, contentError := stackValue.MarshalValue() - if contentError != nil { - return contentError - } - - if !workspace.ValidateConfigValue(projectConfigType.Type, projectConfigType.Items, content) { - typeName := workspace.InferFullTypeName(projectConfigType.Type, projectConfigType.Items) - validationError := fmt.Errorf( - "Stack '%v' with configuration key '%v' must of of type '%v'", - stackName, - projectConfigKey, - typeName) - - return validationError - } - } - } - - return nil -} diff --git a/pkg/cmd/pulumi/watch.go b/pkg/cmd/pulumi/watch.go index 65afc3e7371a..2042fae75d6a 100644 --- a/pkg/cmd/pulumi/watch.go +++ b/pkg/cmd/pulumi/watch.go @@ -111,7 +111,10 @@ func newWatchCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ + applyProjectConfig: true, + }) + if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } diff --git a/sdk/go/auto/local_workspace.go b/sdk/go/auto/local_workspace.go index 15aae0f6c44c..834299c2b3ab 100644 --- a/sdk/go/auto/local_workspace.go +++ b/sdk/go/auto/local_workspace.go @@ -75,11 +75,16 @@ func (l *LocalWorkspace) SaveProjectSettings(ctx context.Context, settings *work // StackSettings returns the settings object for the stack matching the specified stack name if any. // LocalWorkspace reads this from a Pulumi..yaml file in Workspace.WorkDir(). func (l *LocalWorkspace) StackSettings(ctx context.Context, stackName string) (*workspace.ProjectStack, error) { + project, err := l.ProjectSettings(ctx) + if err != nil { + return nil, err + } + name := getStackSettingsName(stackName) for _, ext := range settingsExtensions { stackPath := filepath.Join(l.WorkDir(), fmt.Sprintf("Pulumi.%s%s", name, ext)) if _, err := os.Stat(stackPath); err == nil { - proj, err := workspace.LoadProjectStack(stackPath) + proj, err := workspace.LoadProjectStack(project, stackPath) if err != nil { return nil, errors.Wrap(err, "found stack settings, but failed to load") } diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index c97583f3ea24..828e3c42f6c2 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -126,7 +126,7 @@ type projectStackLoader struct { } // Load a ProjectStack config file from the specified path. The configuration will be cached for subsequent loads. -func (singleton *projectStackLoader) load(path string) (*ProjectStack, error) { +func (singleton *projectStackLoader) load(project *Project, path string) (*ProjectStack, error) { singleton.Lock() defer singleton.Unlock() @@ -252,10 +252,10 @@ func LoadProject(path string) (*Project, error) { } // LoadProjectStack reads a stack definition from a file. -func LoadProjectStack(path string) (*ProjectStack, error) { +func LoadProjectStack(project *Project, path string) (*ProjectStack, error) { contract.Require(path != "", "path") - return projectStackSingleton.load(path) + return projectStackSingleton.load(project, path) } // LoadPluginProject reads a plugin project definition from a file. diff --git a/sdk/go/common/workspace/paths.go b/sdk/go/common/workspace/paths.go index 6de3556fb4f2..85409aaa7f53 100644 --- a/sdk/go/common/workspace/paths.go +++ b/sdk/go/common/workspace/paths.go @@ -93,19 +93,19 @@ func DetectProjectPath() (string, error) { // DetectProjectStackPath returns the name of the file to store stack specific project settings in. We place stack // specific settings next to the Pulumi.yaml file, named like: Pulumi..yaml -func DetectProjectStackPath(stackName tokens.QName) (string, error) { +func DetectProjectStackPath(stackName tokens.QName) (*Project, string, error) { proj, projPath, err := DetectProjectAndPath() if err != nil { - return "", err + return nil, "", err } fileName := fmt.Sprintf("%s.%s%s", ProjectFile, qnameFileName(stackName), filepath.Ext(projPath)) if proj.StackConfigDir != "" { - return filepath.Join(filepath.Dir(projPath), proj.StackConfigDir, fileName), nil + return proj, filepath.Join(filepath.Dir(projPath), proj.StackConfigDir, fileName), nil } - return filepath.Join(filepath.Dir(projPath), fileName), nil + return proj, filepath.Join(filepath.Dir(projPath), fileName), nil } var ErrProjectNotFound = errors.New("no project file found") @@ -144,12 +144,12 @@ func DetectProject() (*Project, error) { } func DetectProjectStack(stackName tokens.QName) (*ProjectStack, error) { - path, err := DetectProjectStackPath(stackName) + project, path, err := DetectProjectStackPath(stackName) if err != nil { return nil, err } - return LoadProjectStack(path) + return LoadProjectStack(project, path) } // DetectProjectAndPath loads the closest package from the current working directory, or an error if not found. It @@ -178,7 +178,7 @@ func SaveProject(proj *Project) error { } func SaveProjectStack(stackName tokens.QName, stack *ProjectStack) error { - path, err := DetectProjectStackPath(stackName) + _, path, err := DetectProjectStackPath(stackName) if err != nil { return err } diff --git a/sdk/go/common/workspace/paths_test.go b/sdk/go/common/workspace/paths_test.go index 3c8f5a025c84..64fc209647bf 100644 --- a/sdk/go/common/workspace/paths_test.go +++ b/sdk/go/common/workspace/paths_test.go @@ -110,7 +110,7 @@ func TestProjectStackPath(t *testing.T) { 0600) assert.NoError(t, err) - path, err := DetectProjectStackPath("my_stack") + _, path, err := DetectProjectStackPath("my_stack") tt.validate(t, tmpDir, path, err) }) } diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 6bb01d485738..b72aeeae3c4b 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -259,7 +259,13 @@ func ValidateConfigValue(typeName string, itemsType *ProjectConfigItemsType, val } if typeName == "boolean" { - _, ok := value.(bool) + // check to see if the value is a literal string "true" | "false" + literalValue, ok := value.(string) + if ok && (literalValue == "true" || literalValue == "false") { + return true + } + + _, ok = value.(bool) return ok } From aa648fdd547d12419aad75a3d961578d0c97fc70 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Wed, 21 Sep 2022 20:37:43 +0200 Subject: [PATCH 05/32] Implement short-hand configuration value support --- sdk/go/common/workspace/loaders.go | 11 ++++- sdk/go/common/workspace/project.go | 65 ++++++++++++++++++++++++++---- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index 828e3c42f6c2..34b32e9fbb0f 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -104,8 +104,17 @@ func (singleton *projectLoader) load(path string) (*Project, error) { return nil, err } + // just before marshalling, we will rewrite the config values + projectDef, err := SimplifyMarshalledProject(raw) + if err != nil { + return nil, err + } + + projectDef = RewriteShorthandConfigValues(projectDef) + modifiedProject, err := marshaller.Marshal(projectDef) + var project Project - err = marshaller.Unmarshal(b, &project) + err = marshaller.Unmarshal(modifiedProject, &project) if err != nil { return nil, err } diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index b72aeeae3c4b..4d91573528c9 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -155,8 +155,47 @@ type Project struct { AdditionalKeys map[string]interface{} `yaml:",inline"` } -func ValidateProject(raw interface{}) error { - // Cast any map[interface{}] from the yaml decoder to map[string] +// RewriteShorthandConfigValues rewrites short-hand version of configuration into a configuration type +// for example the following config block definition: +// +// config: +// instanceSize: t3.mirco +// +// will be rewritten into a typed value: +// +// config: +// type: string +// default: t3.mirco +// +func RewriteShorthandConfigValues(project map[string]interface{}) map[string]interface{} { + configMap, foundConfig := project["config"] + + if !foundConfig { + // no config defined, return as is + return project + } + + config, ok := configMap.(map[string]interface{}) + + if !ok { + return project + } + + for key, value := range config { + literalValue, isLiteral := value.(string) + if isLiteral { + configTypeDefinition := make(map[string]interface{}) + configTypeDefinition["type"] = "string" + configTypeDefinition["default"] = literalValue + config[key] = configTypeDefinition + } + } + + return project +} + +// Cast any map[interface{}] from the yaml decoder to map[string] +func SimplifyMarshalledProject(raw interface{}) (map[string]interface{}, error) { var cast func(value interface{}) (interface{}, error) cast = func(value interface{}) (interface{}, error) { if objMap, ok := value.(map[interface{}]interface{}); ok { @@ -188,29 +227,41 @@ func ValidateProject(raw interface{}) error { } result, err := cast(raw) if err != nil { - return err + return nil, err } var ok bool var obj map[string]interface{} if obj, ok = result.(map[string]interface{}); !ok { - return fmt.Errorf("expected an object") + return nil, fmt.Errorf("expected an object") + } + + return obj, nil +} + +func ValidateProject(raw interface{}) error { + + project, err := SimplifyMarshalledProject(raw) + if err != nil { + return err } // Couple of manual errors to match Validate - name, ok := obj["name"] + name, ok := project["name"] if !ok { return errors.New("project is missing a 'name' attribute") } if strName, ok := name.(string); !ok || strName == "" { return errors.New("project is missing a non-empty string 'name' attribute") } - if _, ok := obj["runtime"]; !ok { + if _, ok := project["runtime"]; !ok { return errors.New("project is missing a 'runtime' attribute") } + project = RewriteShorthandConfigValues(project) + // Let everything else be caught by jsonschema - if err = ProjectSchema.Validate(obj); err == nil { + if err = ProjectSchema.Validate(project); err == nil { return nil } validationError, ok := err.(*jsonschema.ValidationError) From dff4079d37d36d59f9d51e63b92377563e129942 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Wed, 21 Sep 2022 22:58:08 +0200 Subject: [PATCH 06/32] support string, int and bool in shorthand config definitions --- sdk/go/common/workspace/project.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 4d91573528c9..8c4c6ee54ea2 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -155,6 +155,25 @@ type Project struct { AdditionalKeys map[string]interface{} `yaml:",inline"` } +func isPrimitiveValue(value interface{}) (string, bool) { + _, isLiteralBoolean := value.(bool) + if isLiteralBoolean { + return "boolean", true + } + + _, isLiteralInt := value.(int) + if isLiteralInt { + return "integer", true + } + + _, isLiteralString := value.(string) + if isLiteralString { + return "string", true + } + + return "", false +} + // RewriteShorthandConfigValues rewrites short-hand version of configuration into a configuration type // for example the following config block definition: // @@ -182,11 +201,11 @@ func RewriteShorthandConfigValues(project map[string]interface{}) map[string]int } for key, value := range config { - literalValue, isLiteral := value.(string) + typeName, isLiteral := isPrimitiveValue(value) if isLiteral { configTypeDefinition := make(map[string]interface{}) - configTypeDefinition["type"] = "string" - configTypeDefinition["default"] = literalValue + configTypeDefinition["type"] = typeName + configTypeDefinition["default"] = value config[key] = configTypeDefinition } } From 6531b668fd50509db4ac9ea6afa3fe7f6d7cd5a1 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Thu, 22 Sep 2022 10:27:27 +0200 Subject: [PATCH 07/32] test config schemas that use short hand version --- sdk/go/common/workspace/project.go | 5 +- sdk/go/common/workspace/project_test.go | 133 ++++++++++++++++++++---- 2 files changed, 117 insertions(+), 21 deletions(-) diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 8c4c6ee54ea2..c256120e2fc0 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -183,8 +183,9 @@ func isPrimitiveValue(value interface{}) (string, bool) { // will be rewritten into a typed value: // // config: -// type: string -// default: t3.mirco +// instanceSize: +// type: string +// default: t3.mirco // func RewriteShorthandConfigValues(project map[string]interface{}) map[string]interface{} { configMap, foundConfig := project["config"] diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index 3b2ba26c6045..cc23cc77e112 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -3,6 +3,7 @@ package workspace import ( "encoding/json" "io/ioutil" + "os" "testing" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" @@ -205,44 +206,138 @@ func TestProjectLoadJSON(t *testing.T) { assert.Equal(t, "", proj.Main) } -func TestProjectLoadYAML(t *testing.T) { +func deleteFile(t *testing.T, file *os.File) { + if file != nil { + err := os.Remove(file.Name()) + assert.NoError(t, err, "Error while deleting file") + } +} + +func loadProjectFromText(t *testing.T, content string) (*Project, error) { + tmp, err := ioutil.TempFile("", "*.yaml") + assert.NoError(t, err) + path := tmp.Name() + err = ioutil.WriteFile(path, []byte(content), 0600) + assert.NoError(t, err) + defer deleteFile(t, tmp) + return LoadProject(path) +} + +func TestProjectLoadsConfigSchemas(t *testing.T) { t.Parallel() + projectContent := ` +name: test +runtime: dotnet +config: + integerSchemaFull: + type: integer + description: a very important value + default: 1 + integerSchemaSimple: 20 + textSchemaFull: + type: string + default: t3.micro + textSchemaSimple: t4.large + booleanSchemaFull: + type: boolean + default: true + booleanSchemaSimple: false + simpleArrayOfStrings: + type: array + items: + type: string + default: [hello] + arrayOfArrays: + type: array + items: + type: array + items: + type: string + ` + + project, err := loadProjectFromText(t, projectContent) + assert.NoError(t, err, "Should be able to load the project") + assert.Equal(t, 8, len(project.Config), "There are 8 config type definition") + // full integer config schema + integerSchemFull, ok := project.Config["integerSchemaFull"] + assert.True(t, ok, "should be able to read integerSchemaFull") + assert.Equal(t, "integer", integerSchemFull.Type) + assert.Equal(t, "a very important value", integerSchemFull.Description) + assert.Equal(t, 1, integerSchemFull.Default) + assert.Nil(t, integerSchemFull.Items, "Primtive config type doesn't have an items type") + + integerSchemaSimple, ok := project.Config["integerSchemaSimple"] + assert.True(t, ok, "should be able to read integerSchemaSimple") + assert.Equal(t, "integer", integerSchemaSimple.Type, "integer type is inferred correctly") + assert.Equal(t, 20, integerSchemaSimple.Default, "Default integer value is parsed correctly") + + textSchemaFull, ok := project.Config["textSchemaFull"] + assert.True(t, ok, "should be able to read textSchemaFull") + assert.Equal(t, "string", textSchemaFull.Type) + assert.Equal(t, "t3.micro", textSchemaFull.Default) + assert.Equal(t, "", textSchemaFull.Description) + + textSchemaSimple, ok := project.Config["textSchemaSimple"] + assert.True(t, ok, "should be able to read textSchemaSimple") + assert.Equal(t, "string", textSchemaSimple.Type) + assert.Equal(t, "t4.large", textSchemaSimple.Default) + + booleanSchemaFull, ok := project.Config["booleanSchemaFull"] + assert.True(t, ok, "should be able to read booleanSchemaFull") + assert.Equal(t, "boolean", booleanSchemaFull.Type) + assert.Equal(t, true, booleanSchemaFull.Default) + + booleanSchemaSimple, ok := project.Config["booleanSchemaSimple"] + assert.True(t, ok, "should be able to read booleanSchemaSimple") + assert.Equal(t, "boolean", booleanSchemaSimple.Type) + assert.Equal(t, false, booleanSchemaSimple.Default) + + simpleArrayOfStrings, ok := project.Config["simpleArrayOfStrings"] + assert.True(t, ok, "should be able to read simpleArrayOfStrings") + assert.Equal(t, "array", simpleArrayOfStrings.Type) + assert.NotNil(t, simpleArrayOfStrings.Items) + assert.Equal(t, "string", simpleArrayOfStrings.Items.Type) + arrayValues := simpleArrayOfStrings.Default.([]interface{}) + assert.Equal(t, "hello", arrayValues[0]) + + arrayOfArrays, ok := project.Config["arrayOfArrays"] + assert.True(t, ok, "should be able to read arrayOfArrays") + assert.Equal(t, "array", arrayOfArrays.Type) + assert.NotNil(t, arrayOfArrays.Items) + assert.Equal(t, "array", arrayOfArrays.Items.Type) + assert.NotNil(t, arrayOfArrays.Items.Items) + assert.Equal(t, "string", arrayOfArrays.Items.Items.Type) +} - writeAndLoad := func(str string) (*Project, error) { - tmp, err := ioutil.TempFile("", "*.yaml") - assert.NoError(t, err) - path := tmp.Name() - err = ioutil.WriteFile(path, []byte(str), 0600) - assert.NoError(t, err) - return LoadProject(path) - } +func TestProjectLoadYAML(t *testing.T) { + t.Parallel() // Test wrong type - _, err := writeAndLoad("\"hello\"") + _, err := loadProjectFromText(t, "\"hello\"") assert.Equal(t, "expected an object", err.Error()) // Test bad key - _, err = writeAndLoad("4: hello") + _, err = loadProjectFromText(t, "4: hello") assert.Equal(t, "expected only string keys, got '%!s(int=4)'", err.Error()) // Test nested bad key - _, err = writeAndLoad("hello:\n 6: bad") + _, err = loadProjectFromText(t, "hello:\n 6: bad") assert.Equal(t, "expected only string keys, got '%!s(int=6)'", err.Error()) // Test lack of name - _, err = writeAndLoad("{}") + _, err = loadProjectFromText(t, "{}") assert.Equal(t, "project is missing a 'name' attribute", err.Error()) // Test bad name - _, err = writeAndLoad("name:") + _, err = loadProjectFromText(t, "name:") assert.Equal(t, "project is missing a non-empty string 'name' attribute", err.Error()) // Test missing runtime - _, err = writeAndLoad("name: project") + _, err = loadProjectFromText(t, "name: project") assert.Equal(t, "project is missing a 'runtime' attribute", err.Error()) // Test other schema errors - _, err = writeAndLoad("name: project\nruntime: 4") + _, err = loadProjectFromText(t, "name: project\nruntime: 4") // These can vary in order, so contains not equals check expected := []string{ "3 errors occurred:", @@ -253,7 +348,7 @@ func TestProjectLoadYAML(t *testing.T) { assert.Contains(t, err.Error(), e) } - _, err = writeAndLoad("name: project\nruntime: test\nbackend: 4\nmain: {}") + _, err = loadProjectFromText(t, "name: project\nruntime: test\nbackend: 4\nmain: {}") expected = []string{ "2 errors occurred:", "* #/main: expected string or null, but got object", @@ -263,13 +358,13 @@ func TestProjectLoadYAML(t *testing.T) { } // Test success - proj, err := writeAndLoad("name: project\nruntime: test") + proj, err := loadProjectFromText(t, "name: project\nruntime: test") assert.NoError(t, err) assert.Equal(t, tokens.PackageName("project"), proj.Name) assert.Equal(t, "test", proj.Runtime.Name()) // Test null optionals should work - proj, err = writeAndLoad("name: project\nruntime: test\ndescription:\nmain: null\nbackend:\n") + proj, err = loadProjectFromText(t, "name: project\nruntime: test\ndescription:\nmain: null\nbackend:\n") assert.NoError(t, err) assert.Nil(t, proj.Description) assert.Equal(t, "", proj.Main) From d419605d1b3c890f30f75252e9ef5ce14b118876 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Thu, 22 Sep 2022 17:32:18 +0200 Subject: [PATCH 08/32] Moaar unit tests for config validation --- pkg/cmd/pulumi/config.go | 70 +------------ pkg/cmd/pulumi/destroy.go | 2 +- sdk/go/common/workspace/config.go | 72 +++++++++++++ sdk/go/common/workspace/project.go | 8 ++ sdk/go/common/workspace/project_test.go | 130 ++++++++++++++++++++++++ 5 files changed, 214 insertions(+), 68 deletions(-) create mode 100644 sdk/go/common/workspace/config.go diff --git a/pkg/cmd/pulumi/config.go b/pkg/cmd/pulumi/config.go index 3bc5ee518fa8..6676e6bb3a62 100644 --- a/pkg/cmd/pulumi/config.go +++ b/pkg/cmd/pulumi/config.go @@ -784,70 +784,6 @@ type configValueJSON struct { Secret bool `json:"secret"` } -func validateStackConfigAndApplyProjectConfig( - stackName string, - project *workspace.Project, - stackConfig config.Map) error { - for projectConfigKey, projectConfigType := range project.Config { - key := config.MustMakeKey(string(project.Name), projectConfigKey) - stackValue, found, _ := stackConfig.Get(key, true) - hasDefault := projectConfigType.Default != nil - if !found && !hasDefault { - missingConfigError := fmt.Errorf( - "Stack '%v' missing configuration value '%v'", - stackName, - projectConfigKey) - return missingConfigError - } else if !found && hasDefault { - // not found at the stack level - // but has a default value at the project level - // assign the value to the stack - var configValue config.Value - - if projectConfigType.Type == "array" { - // for array types, JSON-ify the default value - configValueJson, jsonError := json.Marshal(projectConfigType.Default) - if jsonError != nil { - return jsonError - } - configValue = config.NewObjectValue(string(configValueJson)) - - } else { - // for primitive types - // pass the values as is - configValueContent := fmt.Sprintf("%v", projectConfigType.Default) - configValue = config.NewValue(configValueContent) - } - - setError := stackConfig.Set(key, configValue, true) - if setError != nil { - return setError - } - } else { - // found value on the stack level - // retrieve it and validate it against - // the config defined at the project level - content, contentError := stackValue.MarshalValue() - if contentError != nil { - return contentError - } - - if !workspace.ValidateConfigValue(projectConfigType.Type, projectConfigType.Items, content) { - typeName := workspace.InferFullTypeName(projectConfigType.Type, projectConfigType.Items) - validationError := fmt.Errorf( - "Stack '%v' with configuration key '%v' must of of type '%v'", - stackName, - projectConfigKey, - typeName) - - return validationError - } - } - } - - return nil -} - func listConfig(ctx context.Context, project *workspace.Project, stack backend.Stack, @@ -862,7 +798,7 @@ func listConfig(ctx context.Context, stackName := stack.Ref().Name().String() // when listing configuration values // also show values coming from the project - configError := validateStackConfigAndApplyProjectConfig(stackName, project, ps.Config) + configError := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, project, ps.Config) if configError != nil { return configError } @@ -956,7 +892,7 @@ func getConfig(ctx context.Context, stack backend.Stack, key config.Key, path, j return err } stackName := stack.Ref().Name().String() - configError := validateStackConfigAndApplyProjectConfig(stackName, project, ps.Config) + configError := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, project, ps.Config) if configError != nil { return configError } @@ -1071,7 +1007,7 @@ func getStackConfiguration( } else { if options.applyProjectConfig { stackName := stack.Ref().Name().String() - configErr := validateStackConfigAndApplyProjectConfig(stackName, project, workspaceStack.Config) + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, project, workspaceStack.Config) if configErr != nil { return defaultStackConfig, configErr } diff --git a/pkg/cmd/pulumi/destroy.go b/pkg/cmd/pulumi/destroy.go index c7194c66d756..57adef0b2101 100644 --- a/pkg/cmd/pulumi/destroy.go +++ b/pkg/cmd/pulumi/destroy.go @@ -166,7 +166,7 @@ func newDestroyCmd() *cobra.Command { } stackName := s.Ref().Name().String() - configError := validateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config) + configError := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config) if configError != nil { return result.FromError(fmt.Errorf("validating stack config: %w", configError)) } diff --git a/sdk/go/common/workspace/config.go b/sdk/go/common/workspace/config.go new file mode 100644 index 000000000000..3c82051e419b --- /dev/null +++ b/sdk/go/common/workspace/config.go @@ -0,0 +1,72 @@ +package workspace + +import ( + "encoding/json" + "fmt" + + "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" +) + +func ValidateStackConfigAndApplyProjectConfig( + stackName string, + project *Project, + stackConfig config.Map) error { + for projectConfigKey, projectConfigType := range project.Config { + key := config.MustMakeKey(string(project.Name), projectConfigKey) + stackValue, found, _ := stackConfig.Get(key, true) + hasDefault := projectConfigType.Default != nil + if !found && !hasDefault { + missingConfigError := fmt.Errorf( + "Stack '%v' missing configuration value '%v'", + stackName, + projectConfigKey) + return missingConfigError + } else if !found && hasDefault { + // not found at the stack level + // but has a default value at the project level + // assign the value to the stack + var configValue config.Value + + if projectConfigType.Type == "array" { + // for array types, JSON-ify the default value + configValueJson, jsonError := json.Marshal(projectConfigType.Default) + if jsonError != nil { + return jsonError + } + configValue = config.NewObjectValue(string(configValueJson)) + + } else { + // for primitive types + // pass the values as is + configValueContent := fmt.Sprintf("%v", projectConfigType.Default) + configValue = config.NewValue(configValueContent) + } + + setError := stackConfig.Set(key, configValue, true) + if setError != nil { + return setError + } + } else { + // found value on the stack level + // retrieve it and validate it against + // the config defined at the project level + content, contentError := stackValue.MarshalValue() + if contentError != nil { + return contentError + } + + if !ValidateConfigValue(projectConfigType.Type, projectConfigType.Items, content) { + typeName := InferFullTypeName(projectConfigType.Type, projectConfigType.Items) + validationError := fmt.Errorf( + "Stack '%v' with configuration key '%v' must be of type '%v'", + stackName, + projectConfigKey, + typeName) + + return validationError + } + } + } + + return nil +} diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index c256120e2fc0..2380c2d6af4d 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "strings" "github.com/hashicorp/go-multierror" @@ -326,6 +327,13 @@ func ValidateConfigValue(typeName string, itemsType *ProjectConfigItemsType, val if typeName == "integer" { _, ok := value.(int) + if !ok { + valueAsText, isText := value.(string) + if isText { + _, integerParseError := strconv.Atoi(valueAsText) + return integerParseError == nil + } + } return ok } diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index cc23cc77e112..4970d17d8a4e 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" @@ -223,6 +224,16 @@ func loadProjectFromText(t *testing.T, content string) (*Project, error) { return LoadProject(path) } +func loadProjectStackFromText(t *testing.T, project *Project, content string) (*ProjectStack, error) { + tmp, err := ioutil.TempFile("", "*.yaml") + assert.NoError(t, err) + path := tmp.Name() + err = ioutil.WriteFile(path, []byte(content), 0600) + assert.NoError(t, err) + defer deleteFile(t, tmp) + return LoadProjectStack(project, path) +} + func TestProjectLoadsConfigSchemas(t *testing.T) { t.Parallel() projectContent := ` @@ -309,6 +320,125 @@ config: assert.Equal(t, "string", arrayOfArrays.Items.Items.Type) } +func getConfigValue(t *testing.T, stackConfig config.Map, key string) string { + parsedKey, err := config.ParseKey(key) + assert.NoErrorf(t, err, "There should be no error parsing the config key '%v'", key) + configValue, foundValue := stackConfig[parsedKey] + assert.Truef(t, foundValue, "Couldn't find a value for config key %v", key) + value, valueError := configValue.Value(config.NopDecrypter) + assert.NoErrorf(t, valueError, "Error while getting the value for key %v", key) + return value +} + +func TestStackConfigIsInheritedFromProjectConfig(t *testing.T) { + projectYaml := ` +name: test +runtime: dotnet +config: + instanceSize: t3.micro + instanceCount: 20 + protect: true` + + projectStackYaml := ` +config: + test:instanceSize: t4.large` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NoError(t, configError, "Config override should be valid") + + assert.Equal(t, 3, len(stack.Config), "Stack config now has three values") + // value of instanceSize is overwritten from the stack + assert.Equal(t, "t4.large", getConfigValue(t, stack.Config, "test:instanceSize")) + // instanceCount and protect are inherited from the project + assert.Equal(t, "20", getConfigValue(t, stack.Config, "test:instanceCount")) + assert.Equal(t, "true", getConfigValue(t, stack.Config, "test:protect")) +} + +func TestStackConfigErrorsWhenStackValueIsNotCorrectlyTyped(t *testing.T) { + projectYaml := ` +name: test +runtime: dotnet +config: + values: + type: array + items: + type: string + default: [value]` + + projectStackYaml := ` +config: + test:values: someValue +` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, configError.Error(), "Stack 'dev' with configuration key 'values' must be of type 'array'") +} + +func TestStackConfigIntegerTypeIsCorrectlyValidated(t *testing.T) { + projectYaml := ` +name: test +runtime: dotnet +config: + importantNumber: + type: integer +` + + projectStackYamlValid := ` +config: + test:importantNumber: 20 +` + + projectStackYamlInvalid := ` +config: + test:importantNumber: hello +` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYamlValid) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NoError(t, configError, "there should no config type error") + + invalidStackConfig, stackError := loadProjectStackFromText(t, project, projectStackYamlInvalid) + assert.NoError(t, stackError, "Should be able to read the stack") + configError = ValidateStackConfigAndApplyProjectConfig("dev", project, invalidStackConfig.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, + configError.Error(), + "Stack 'dev' with configuration key 'importantNumber' must be of type 'integer'") +} + +func TestStackConfigErrorsWhenMissingStackValueForConfigTypeWithNoDefault(t *testing.T) { + projectYaml := ` +name: test +runtime: dotnet +config: + values: + type: array + items: + type: string` + + projectStackYaml := `` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, configError.Error(), "Stack 'dev' missing configuration value 'values'") +} + func TestProjectLoadYAML(t *testing.T) { t.Parallel() From 02bdbd00eadf4ac8458fcc1b14829e7cc0eb068f Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Thu, 22 Sep 2022 17:40:14 +0200 Subject: [PATCH 09/32] changelog entry --- changelog/pending/20220922--cli-initial-mvp-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/pending/20220922--cli-initial-mvp-config.yaml diff --git a/changelog/pending/20220922--cli-initial-mvp-config.yaml b/changelog/pending/20220922--cli-initial-mvp-config.yaml new file mode 100644 index 000000000000..6f0307f94ad3 --- /dev/null +++ b/changelog/pending/20220922--cli-initial-mvp-config.yaml @@ -0,0 +1,4 @@ +changes: +- type: feat + scope: cli + description: Implement initial MVP for hierarchical and structured project configuration From 7975498b71d95589b371efefdae4a529bdd0a388 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Thu, 22 Sep 2022 18:27:08 +0200 Subject: [PATCH 10/32] lint --- sdk/go/common/workspace/config.go | 4 ++-- sdk/go/common/workspace/loaders.go | 2 +- sdk/go/common/workspace/project_test.go | 4 ++++ tests/integration/integration_test.go | 10 ++++++---- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/sdk/go/common/workspace/config.go b/sdk/go/common/workspace/config.go index 3c82051e419b..489c505ec1a7 100644 --- a/sdk/go/common/workspace/config.go +++ b/sdk/go/common/workspace/config.go @@ -29,11 +29,11 @@ func ValidateStackConfigAndApplyProjectConfig( if projectConfigType.Type == "array" { // for array types, JSON-ify the default value - configValueJson, jsonError := json.Marshal(projectConfigType.Default) + configValueJSON, jsonError := json.Marshal(projectConfigType.Default) if jsonError != nil { return jsonError } - configValue = config.NewObjectValue(string(configValueJson)) + configValue = config.NewObjectValue(string(configValueJSON)) } else { // for primitive types diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index 34b32e9fbb0f..e67977f2a4ca 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -111,7 +111,7 @@ func (singleton *projectLoader) load(path string) (*Project, error) { } projectDef = RewriteShorthandConfigValues(projectDef) - modifiedProject, err := marshaller.Marshal(projectDef) + modifiedProject, _ := marshaller.Marshal(projectDef) var project Project err = marshaller.Unmarshal(modifiedProject, &project) diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index 4970d17d8a4e..fbccd10dde78 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -331,6 +331,7 @@ func getConfigValue(t *testing.T, stackConfig config.Map, key string) string { } func TestStackConfigIsInheritedFromProjectConfig(t *testing.T) { + t.Parallel() projectYaml := ` name: test runtime: dotnet @@ -359,6 +360,7 @@ config: } func TestStackConfigErrorsWhenStackValueIsNotCorrectlyTyped(t *testing.T) { + t.Parallel() projectYaml := ` name: test runtime: dotnet @@ -384,6 +386,7 @@ config: } func TestStackConfigIntegerTypeIsCorrectlyValidated(t *testing.T) { + t.Parallel() projectYaml := ` name: test runtime: dotnet @@ -419,6 +422,7 @@ config: } func TestStackConfigErrorsWhenMissingStackValueForConfigTypeWithNoDefault(t *testing.T) { + t.Parallel() projectYaml := ` name: test runtime: dotnet diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 58358e667b9f..09e8f9a84380 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -163,10 +163,12 @@ func TestConfigSave(t *testing.T) { // Initialize an empty stack. path := filepath.Join(e.RootPath, "Pulumi.yaml") - err := (&workspace.Project{ + project := workspace.Project{ Name: "testing-config", Runtime: workspace.NewProjectRuntimeInfo("nodejs", nil), - }).Save(path) + } + + err := project.Save(path) assert.NoError(t, err) e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) e.RunCommand("pulumi", "stack", "init", "testing-2") @@ -209,9 +211,9 @@ func TestConfigSave(t *testing.T) { assert.Equal(t, v, dv) } - testStack1, err := workspace.LoadProjectStack(filepath.Join(e.CWD, "Pulumi.testing-1.yaml")) + testStack1, err := workspace.LoadProjectStack(&project, filepath.Join(e.CWD, "Pulumi.testing-1.yaml")) assert.NoError(t, err) - testStack2, err := workspace.LoadProjectStack(filepath.Join(e.CWD, "Pulumi.testing-2.yaml")) + testStack2, err := workspace.LoadProjectStack(&project, filepath.Join(e.CWD, "Pulumi.testing-2.yaml")) assert.NoError(t, err) assert.Equal(t, 2, len(testStack1.Config)) From 969e0e662c218658079bec79b176172471a9646a Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Sat, 24 Sep 2022 14:01:35 +0200 Subject: [PATCH 11/32] project.json should validate short-hand config syntax before rewriting input project --- sdk/go/common/workspace/project.json | 523 ++++++++++++++------------- 1 file changed, 268 insertions(+), 255 deletions(-) diff --git a/sdk/go/common/workspace/project.json b/sdk/go/common/workspace/project.json index 170c5355055b..2791526def7a 100644 --- a/sdk/go/common/workspace/project.json +++ b/sdk/go/common/workspace/project.json @@ -1,267 +1,262 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/pulumi/pulumi/blob/master/sdk/go/common/workspace/project.json", - "title": "Pulumi Project", - "description": "A schema for Pulumi project files.", - "type": "object", - "properties": { - "name": { - "description": "Name of the project containing alphanumeric characters, hyphens, underscores, and periods.", - "type": "string", - "minLength": 1 - }, - "description": { - "description": "Description of the project.", - "type": [ - "string", - "null" - ] - }, - "author": { - "description": "Author is an optional author that created this project.", - "type": [ - "string", - "null" - ] - }, - "website": { - "description": "Website is an optional website for additional info about this project.", - "type": [ - "string", - "null" - ] - }, - "license": { - "description": "License is the optional license governing this project's usage.", - "type": [ - "string", - "null" - ] - }, - "runtime": { - "title": "ProjectRuntimeInfo", - "oneOf": [ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/pulumi/pulumi/blob/master/sdk/go/common/workspace/project.json", + "title": "Pulumi Project", + "description": "A schema for Pulumi project files.", + "type": "object", + "properties": { + "name": { + "description": "Name of the project containing alphanumeric characters, hyphens, underscores, and periods.", + "type": "string", + "minLength": 1 + }, + "description": { + "description": "Description of the project.", + "type": [ + "string", + "null" + ] + }, + "author": { + "description": "Author is an optional author that created this project.", + "type": [ + "string", + "null" + ] + }, + "website": { + "description": "Website is an optional website for additional info about this project.", + "type": [ + "string", + "null" + ] + }, + "license": { + "description": "License is the optional license governing this project's usage.", + "type": [ + "string", + "null" + ] + }, + "runtime": { + "title": "ProjectRuntimeInfo", + "oneOf": [ + { + "title": "Name", + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string", + "minLength": 1 + }, + "options": { + "title": "Options", + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + } + ] + }, + "main": { + "description": "Path to the Pulumi program. The default is the working directory.", + "type": [ + "string", + "null" + ] + }, + "config": { + "description": "A map of configuration keys to their types. Using config directory location relative to the location of Pulumi.yaml is a deprecated use of this key. Use stackConfigDir instead.", + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "string" + }, { - "title": "Name", - "type": "string", - "minLength": 1 + "type": "integer" }, { - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string", - "minLength": 1 - }, - "options": { - "title": "Options", - "type": "object", - "additionalProperties": true - } - }, - "additionalProperties": false + "type": "boolean" + }, + { + "$ref": "#/$defs/configTypeDeclaration" } ] - }, - "main": { - "description": "Path to the Pulumi program. The default is the working directory.", - "type": [ - "string", - "null" - ] - }, - "config": { - "description": "A map of configuration keys to their types", - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/configTypeDeclaration" + } + }, + "stackConfigDir": { + "description": "Config directory location relative to the location of Pulumi.yaml.", + "type": [ + "string", + "null" + ] + }, + "backend": { + "description": "Backend of the project.", + "type": [ + "object", + "null" + ], + "properties": { + "url": { + "description": "URL is optional field to explicitly set backend url", + "type": "string" } }, - "stackConfigDir": { - "description": "Config directory location relative to the location of Pulumi.yaml.", - "type": [ - "string", - "null" - ] + "additionalProperties": false + }, + "options": { + "description": "Additional project options.", + "type": [ + "object", + "null" + ], + "properties": { + "refresh": { + "description": "Set to \"always\" to refresh the state before performing a Pulumi operation.", + "type": "string", + "const": "always" + } }, - "backend": { - "description": "Backend of the project.", - "type": [ - "object", - "null" - ], - "properties": { - "url": { - "description": "URL is optional field to explicitly set backend url", - "type": "string" - } + "additionalProperties": false + }, + "template": { + "title": "ProjectTemplate", + "description": "ProjectTemplate is a Pulumi project template manifest.", + "type": [ + "object", + "null" + ], + "properties": { + "description": { + "description": "Description of the template.", + "type": [ + "string", + "null" + ] }, - "additionalProperties": false - }, - "options": { - "description": "Additional project options.", - "type": [ - "object", - "null" - ], - "properties": { - "refresh": { - "description": "Set to \"always\" to refresh the state before performing a Pulumi operation.", - "type": "string", - "const": "always" - } + "quickstart": { + "description": "Quickstart contains optional text to be displayed after template creation.", + "type": [ + "string", + "null" + ] }, - "additionalProperties": false - }, - "template": { - "title": "ProjectTemplate", - "description": "ProjectTemplate is a Pulumi project template manifest.", - "type": [ - "object", - "null" - ], - "properties": { - "description": { - "description": "Description of the template.", - "type": [ - "string", - "null" - ] - }, - "quickstart": { - "description": "Quickstart contains optional text to be displayed after template creation.", - "type": [ - "string", - "null" - ] - }, - "important": { - "description": "Important indicates the template is important and should be listed by default.", - "type": [ - "boolean", - "null" - ] - }, - "config": { - "description": "Config to apply to each stack in the project.", - "type": [ - "object", - "null" - ], - "additionalProperties": { - "properties": { - "description": { - "description": "Description of the config.", - "type": [ - "string", - "null" - ] - }, - "default": { - "description": "Default value of the config." - }, - "secret": { - "description": "Boolean indicating if the configuration is labeled as a secret.", - "type": [ - "boolean", - "null" - ] - } + "important": { + "description": "Important indicates the template is important and should be listed by default.", + "type": [ + "boolean", + "null" + ] + }, + "config": { + "description": "Config to apply to each stack in the project.", + "type": [ + "object", + "null" + ], + "additionalProperties": { + "properties": { + "description": { + "description": "Description of the config.", + "type": [ + "string", + "null" + ] + }, + "default": { + "description": "Default value of the config." + }, + "secret": { + "description": "Boolean indicating if the configuration is labeled as a secret.", + "type": [ + "boolean", + "null" + ] } } } - }, - "additionalProperties": false + } }, - "plugins": { - "description": "Override for the plugin selection. Intended for use in developing pulumi plugins.", - "type": "object", - "properties": { - "providers": { - "description": "Plugins for resource providers.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" - } - }, - "analyzers": { - "description": "Plugins for policy analyzers.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" - } - }, - "languages": { - "description": "Plugins for languages.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" - } + "additionalProperties": false + }, + "plugins": { + "description": "Override for the plugin selection. Intended for use in developing pulumi plugins.", + "type": "object", + "properties": { + "providers": { + "description": "Plugins for resource providers.", + "type": "array", + "items": { + "$ref": "#/$defs/pluginOptions" + } + }, + "analyzers": { + "description": "Plugins for policy analyzers.", + "type": "array", + "items": { + "$ref": "#/$defs/pluginOptions" + } + }, + "languages": { + "description": "Plugins for languages.", + "type": "array", + "items": { + "$ref": "#/$defs/pluginOptions" } } } + } + }, + "required": [ + "name", + "runtime" + ], + "additionalProperties": true, + "$defs": { + "pluginOptions": { + "title": "PluginOptions", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the plugin" + }, + "path": { + "type": "string", + "description": "Path to the plugin folder" + }, + "version": { + "type": "string", + "description": "Version of the plugin, if not set, will match any version the engine requests." + } + } + }, + "simpleConfigType": { + "title": "SimpleConfigType", + "enum": ["string","integer", "boolean", "array"] }, - "required": [ - "name", - "runtime" - ], - "additionalProperties": true, - "$defs": { - "pluginOptions": { - "title": "PluginOptions", + "configItemsType": { "type": "object", + "required": ["type"], "properties": { - "name": { - "type": "string", - "description": "Name of the plugin" - }, - "path": { - "type": "string", - "description": "Path to the plugin folder" - }, - "version": { - "type": "string", - "description": "Version of the plugin, if not set, will match any version the engine requests." - } - } - }, - "simpleConfigType": { - "title": "SimpleConfigType", - "enum": ["string","integer", "boolean", "array"] - }, - "configItemsType": { - "type": "object", - "required": ["type"], - "properties": { - "type": { - "oneOf": [ - { "$ref": "#/$defs/simpleConfigType" }, - { "$ref": "#/$defs/configItemsType" } - ] - }, - "items": { - "$ref": "#/$defs/configItemsType" - } - }, - "if": { - "properties": { - "type": { - "const": "array" - } - } + "type": { + "oneOf": [ + { "$ref": "#/$defs/simpleConfigType" }, + { "$ref": "#/$defs/configItemsType" } + ] }, - "then": { - "required": ["items"] + "items": { + "$ref": "#/$defs/configItemsType" } - }, - "configTypeDeclaration": { - "title": "ConfigTypeDeclaration", - "type": "object", - "additionalProperties": false, - "required": [ - "type" - ], + }, "if": { "properties": { "type": { @@ -271,24 +266,42 @@ }, "then": { "required": ["items"] - }, + } + }, + "configTypeDeclaration": { + "title": "ConfigTypeDeclaration", + "type": "object", + "additionalProperties": false, + "required": [ + "type" + ], + "if": { "properties": { "type": { - "$ref": "#/$defs/simpleConfigType" - }, - "items": { - "$ref": "#/$defs/configItemsType" - }, - "description": { - "type": "string" - }, - "secret": { - "type": "boolean" - }, - "default": { - + "const": "array" } } + }, + "then": { + "required": ["items"] + }, + "properties": { + "type": { + "$ref": "#/$defs/simpleConfigType" + }, + "items": { + "$ref": "#/$defs/configItemsType" + }, + "description": { + "type": "string" + }, + "secret": { + "type": "boolean" + }, + "default": { + + } } } + } } \ No newline at end of file From 0498be116be896fa28b5af4746ad197666167d85 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Sat, 24 Sep 2022 14:32:11 +0200 Subject: [PATCH 12/32] Rewrite config to stackConfigDir when provided as string --- sdk/go/common/workspace/loaders.go | 5 +++ sdk/go/common/workspace/project.go | 32 ++++++++++++++++++ sdk/go/common/workspace/project.json | 2 +- sdk/go/common/workspace/project_test.go | 44 +++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index e67977f2a4ca..6c04102ee241 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -110,6 +110,11 @@ func (singleton *projectLoader) load(path string) (*Project, error) { return nil, err } + projectDef, rewriteError := RewriteConfigPathIntoStackConfigDir(projectDef) + if rewriteError != nil { + return nil, rewriteError + } + projectDef = RewriteShorthandConfigValues(projectDef) modifiedProject, _ := marshaller.Marshal(projectDef) diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 2380c2d6af4d..63485ee7ee6a 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -175,6 +175,31 @@ func isPrimitiveValue(value interface{}) (string, bool) { return "", false } +// RewriteConfigPathIntoStackConfigDir checks if the project is using the old "config" property +// to declare a path to the stack configuration directory. If that is the case, we rewrite it +// such that the value in config: {value} is moved to stackConfigDir: {value}. +// if the user defines both values as strings, we error out. +func RewriteConfigPathIntoStackConfigDir(project map[string]interface{}) (map[string]interface{}, error) { + config, hasConfig := project["config"] + _, hasStackConfigDir := project["stackConfigDir"] + + if hasConfig { + configText, configIsText := config.(string) + if configIsText && hasStackConfigDir { + return nil, errors.New("Should not use both config and stackConfigDir to define the stack directory. " + + "Use only stackConfigDir instead.") + } else if configIsText && !hasStackConfigDir { + // then we have config: {value}. Move this to stackConfigDir: {value} + project["stackConfigDir"] = configText + // reset the config property + project["config"] = nil + return project, nil + } + } + + return project, nil +} + // RewriteShorthandConfigValues rewrites short-hand version of configuration into a configuration type // for example the following config block definition: // @@ -279,6 +304,13 @@ func ValidateProject(raw interface{}) error { return errors.New("project is missing a 'runtime' attribute") } + project, rewriteError := RewriteConfigPathIntoStackConfigDir(project) + + // when defining both config and stackConfigDir as strings in the same project + if rewriteError != nil { + return rewriteError + } + project = RewriteShorthandConfigValues(project) // Let everything else be caught by jsonschema diff --git a/sdk/go/common/workspace/project.json b/sdk/go/common/workspace/project.json index 2791526def7a..1c6b103be313 100644 --- a/sdk/go/common/workspace/project.json +++ b/sdk/go/common/workspace/project.json @@ -73,7 +73,7 @@ }, "config": { "description": "A map of configuration keys to their types. Using config directory location relative to the location of Pulumi.yaml is a deprecated use of this key. Use stackConfigDir instead.", - "type": "object", + "type": ["object", "null"], "additionalProperties": { "oneOf": [ { diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index fbccd10dde78..22c885272986 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -385,6 +385,50 @@ config: assert.Contains(t, configError.Error(), "Stack 'dev' with configuration key 'values' must be of type 'array'") } +func TestLoadingConfigIsRewrittenToStackConfigDir(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: ./some/path` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + assert.Equal(t, "./some/path", project.StackConfigDir, "Stack config dir is read from the config property") + assert.Equal(t, 0, len(project.Config), "Config should be empty") +} + +func TestDefningBothConfigAndStackConfigDirErrorsOut(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: ./some/path +stackConfigDir: ./some/other/path` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.Nil(t, project, "Should NOT be able to load the project") + assert.NotNil(t, projectError, "There is a project error") + assert.Contains(t, projectError.Error(), "Should not use both config and stackConfigDir") +} + +func TestConfigObjectAndStackConfigDirSuccessfullyLoadProject(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +stackConfigDir: ./some/other/path +config: + value: hello +` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.Nil(t, projectError, "There is no error") + assert.NotNil(t, project, "The project can be loaded correctly") + assert.Equal(t, "./some/other/path", project.StackConfigDir) + assert.Equal(t, 1, len(project.Config), "there is one config value") +} + func TestStackConfigIntegerTypeIsCorrectlyValidated(t *testing.T) { t.Parallel() projectYaml := ` From 65cfee4a9599efa7e2cd3c2e00687d46df0c4490 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Sat, 24 Sep 2022 14:38:42 +0200 Subject: [PATCH 13/32] pulumi import uses hierarchical config --- pkg/cmd/pulumi/import.go | 3 +-- sdk/go/common/workspace/paths_test.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/cmd/pulumi/import.go b/pkg/cmd/pulumi/import.go index 32c253cdfe85..ed064dd69055 100644 --- a/pkg/cmd/pulumi/import.go +++ b/pkg/cmd/pulumi/import.go @@ -493,8 +493,7 @@ func newImportCmd() *cobra.Command { } cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ - // we don't need project config here - applyProjectConfig: false, + applyProjectConfig: true, }) if err != nil { diff --git a/sdk/go/common/workspace/paths_test.go b/sdk/go/common/workspace/paths_test.go index 64fc209647bf..2139f9e364a2 100644 --- a/sdk/go/common/workspace/paths_test.go +++ b/sdk/go/common/workspace/paths_test.go @@ -90,7 +90,7 @@ func TestProjectStackPath(t *testing.T) { "name: some_project\ndescription: Some project\nruntime: nodejs\nconfig: stacksA\nstackConfigDir: stacksB\n", func(t *testing.T, projectDir, path string, err error) { assert.Error(t, err) - assert.Equal(t, "can not set `config` and `stackConfigDir`, remove the `config` entry", err.Error()) + assert.Contains(t, err.Error(), "Should not use both config and stackConfigDir") }, }} From 5bd9b802564be372175cad215ac033a5e0b040bf Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Sat, 24 Sep 2022 14:51:08 +0200 Subject: [PATCH 14/32] namespaced config values don't need the project as root namespace --- sdk/go/common/workspace/config.go | 17 +++++++++++++++- sdk/go/common/workspace/project_test.go | 26 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/sdk/go/common/workspace/config.go b/sdk/go/common/workspace/config.go index 489c505ec1a7..2f32118dd4aa 100644 --- a/sdk/go/common/workspace/config.go +++ b/sdk/go/common/workspace/config.go @@ -3,6 +3,7 @@ package workspace import ( "encoding/json" "fmt" + "strings" "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" ) @@ -12,7 +13,21 @@ func ValidateStackConfigAndApplyProjectConfig( project *Project, stackConfig config.Map) error { for projectConfigKey, projectConfigType := range project.Config { - key := config.MustMakeKey(string(project.Name), projectConfigKey) + var key config.Key + if strings.Contains(projectConfigKey, ":") { + // key is already namespaced + parsedKey, parseError := config.ParseKey(projectConfigKey) + if parseError != nil { + return parseError + } + + key = parsedKey + } else { + // key is not namespaced + // use the project as namespace + key = config.MustMakeKey(string(project.Name), projectConfigKey) + } + stackValue, found, _ := stackConfig.Get(key, true) hasDefault := projectConfigType.Default != nil if !found && !hasDefault { diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index 22c885272986..90f8399be9e6 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -359,6 +359,32 @@ config: assert.Equal(t, "true", getConfigValue(t, stack.Config, "test:protect")) } +func TestNamespacedConfigValuesAreInheritedCorrectly(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + aws:region: us-west-1` + + projectStackYaml := ` +config: + test:instanceSize: t4.large` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NoError(t, configError, "Config override should be valid") + + assert.Equal(t, 2, len(stack.Config), "Stack config now has three values") + // value of instanceSize is overwritten from the stack + assert.Equal(t, "t4.large", getConfigValue(t, stack.Config, "test:instanceSize")) + // aws:region is namespaced and is inherited from the project + assert.Equal(t, "us-west-1", getConfigValue(t, stack.Config, "aws:region")) +} + func TestStackConfigErrorsWhenStackValueIsNotCorrectlyTyped(t *testing.T) { t.Parallel() projectYaml := ` From fbc68cb644b420b0d99428f9d1933501ae06cd78 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Mon, 26 Sep 2022 12:30:55 +0200 Subject: [PATCH 15/32] Project namespace is now optional when defining stack config --- sdk/go/common/workspace/loaders.go | 71 ++++++++++++++++++++++--- sdk/go/common/workspace/project_test.go | 26 +++++++++ 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index 6c04102ee241..d4e1ae4a98ea 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -15,8 +15,10 @@ package workspace import ( + "fmt" "io/ioutil" "os" + "strings" "sync" "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" @@ -139,6 +141,40 @@ type projectStackLoader struct { internal map[string]*ProjectStack } +// Rewrite config values to make them namespaced. Using the project name as the default namespace +// for example: +// config: +// instanceSize: t3.micro +// +// is valid configuration and will be rewritten in the form +// +// config: +// {projectName}:instanceSize:t3.micro +func stackConfigNamespacedWithProject(project *Project, projectStack map[string]interface{}) map[string]interface{} { + config, ok := projectStack["config"] + if ok { + configAsMap, isMap := config.(map[string]interface{}) + if isMap { + modifiedConfig := make(map[string]interface{}) + for key, value := range configAsMap { + if strings.Contains(key, ":") { + // key is already namespaced + // use it as is + modifiedConfig[key] = value + } else { + namespacedKey := fmt.Sprintf("%s:%s", project.Name, key) + modifiedConfig[namespacedKey] = value + } + } + + projectStack["config"] = modifiedConfig + return projectStack + } + } + + return projectStack +} + // Load a ProjectStack config file from the specified path. The configuration will be cached for subsequent loads. func (singleton *projectStackLoader) load(project *Project, path string) (*ProjectStack, error) { singleton.Lock() @@ -148,24 +184,47 @@ func (singleton *projectStackLoader) load(project *Project, path string) (*Proje return v, nil } - marshaler, err := marshallerForPath(path) + marshaller, err := marshallerForPath(path) if err != nil { return nil, err } - var projectStack ProjectStack b, err := readFileStripUTF8BOM(path) if os.IsNotExist(err) { - projectStack = ProjectStack{ + defaultProjectStack := ProjectStack{ Config: make(config.Map), } - singleton.internal[path] = &projectStack - return &projectStack, nil + singleton.internal[path] = &defaultProjectStack + return &defaultProjectStack, nil } else if err != nil { return nil, err } - err = marshaler.Unmarshal(b, &projectStack) + var projectStackRaw interface{} + err = marshaller.Unmarshal(b, &projectStackRaw) + if err != nil { + return nil, err + } + + simplifiedStackFormm, err := SimplifyMarshalledProject(projectStackRaw) + if err != nil { + return nil, err + } + + // rewrite config values to make them namespaced + // for example: + // config: + // instanceSize: t3.micro + // + // is valid configuration and will be rewritten in the form + // + // config: + // {projectName}:instanceSize:t3.micro + projectStackWithNamespacedConfig := stackConfigNamespacedWithProject(project, simplifiedStackFormm) + modifiedProjectStack, _ := marshaller.Marshal(projectStackWithNamespacedConfig) + + var projectStack ProjectStack + err = marshaller.Unmarshal(modifiedProjectStack, &projectStack) if err != nil { return nil, err } diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index 90f8399be9e6..d2646d502332 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -385,6 +385,32 @@ config: assert.Equal(t, "us-west-1", getConfigValue(t, stack.Config, "aws:region")) } +func TestLoadingStackConfigWithoutNamespacingTheProject(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + aws:region: us-west-1` + + projectStackYaml := ` +config: + instanceSize: t4.large` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NoError(t, configError, "Config override should be valid") + + assert.Equal(t, 2, len(stack.Config), "Stack config now has three values") + // value of instanceSize is overwritten from the stack + assert.Equal(t, "t4.large", getConfigValue(t, stack.Config, "test:instanceSize")) + // aws:region is namespaced and is inherited from the project + assert.Equal(t, "us-west-1", getConfigValue(t, stack.Config, "aws:region")) +} + func TestStackConfigErrorsWhenStackValueIsNotCorrectlyTyped(t *testing.T) { t.Parallel() projectYaml := ` From 77c7880e891d820f672d908db5cc77aa880d63fb Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Mon, 26 Sep 2022 12:49:07 +0200 Subject: [PATCH 16/32] Fix reading empty project stack file --- sdk/go/common/workspace/loaders.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index d4e1ae4a98ea..b4b81d8b39c5 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -206,7 +206,16 @@ func (singleton *projectStackLoader) load(project *Project, path string) (*Proje return nil, err } - simplifiedStackFormm, err := SimplifyMarshalledProject(projectStackRaw) + if projectStackRaw == nil { + // for example when reading an empty stack file + defaultProjectStack := ProjectStack{ + Config: make(config.Map), + } + singleton.internal[path] = &defaultProjectStack + return &defaultProjectStack, nil + } + + simplifiedStackForm, err := SimplifyMarshalledProject(projectStackRaw) if err != nil { return nil, err } @@ -220,7 +229,7 @@ func (singleton *projectStackLoader) load(project *Project, path string) (*Proje // // config: // {projectName}:instanceSize:t3.micro - projectStackWithNamespacedConfig := stackConfigNamespacedWithProject(project, simplifiedStackFormm) + projectStackWithNamespacedConfig := stackConfigNamespacedWithProject(project, simplifiedStackForm) modifiedProjectStack, _ := marshaller.Marshal(projectStackWithNamespacedConfig) var projectStack ProjectStack From ae03899b25f0237da1b8079fbb0edca521f80c24 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Mon, 26 Sep 2022 17:40:12 +0200 Subject: [PATCH 17/32] Fix TestSecretsProviderOverride which requires both stack and project files --- pkg/cmd/pulumi/crypto_cloud_test.go | 75 ++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/pkg/cmd/pulumi/crypto_cloud_test.go b/pkg/cmd/pulumi/crypto_cloud_test.go index 587bb9c64fa3..6f8a7aef4514 100644 --- a/pkg/cmd/pulumi/crypto_cloud_test.go +++ b/pkg/cmd/pulumi/crypto_cloud_test.go @@ -22,49 +22,76 @@ import ( "testing" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" + "github.com/stretchr/testify/assert" "gocloud.dev/secrets" "gocloud.dev/secrets/driver" ) +func deleteFile(t *testing.T, file *os.File) { + if file != nil { + err := os.Remove(file.Name()) + assert.NoError(t, err, "Error while deleting file") + } +} + +func deleteFiles(t *testing.T, files map[string]string) { + for file, _ := range files { + err := os.Remove(file) + assert.Nil(t, err, "Should be able to remove the file directory") + } +} + +func createTempFiles(t *testing.T, files map[string]string, f func()) { + for file, content := range files { + fileError := os.WriteFile(file, []byte(content), 0600) + assert.Nil(t, fileError, "should be able to write the file contents") + } + + defer deleteFiles(t, files) + f() +} + //nolint:paralleltest -func TestSecretsproviderOverride(t *testing.T) { +func TestSecretsProviderOverride(t *testing.T) { // Don't call t.Parallel because we temporarily modify // PULUMI_CLOUD_SECRET_OVERRIDE env var and it may interfere with other // tests. - const stackConfig = "Pulumi.TestSecretsproviderOverride.yaml" - var stackName = tokens.Name("TestSecretsproviderOverride") - // Cleanup the generated stack config after the test. - t.Cleanup(func() { os.Remove(stackConfig) }) + stackConfigFileName := "Pulumi.TestSecretsProviderOverride.yaml" + files := make(map[string]string) + files["Pulumi.yaml"] = "{\"name\":\"test\", \"runtime\":\"dotnet\"}" + files[stackConfigFileName] = "" + + var stackName = tokens.Name("TestSecretsProviderOverride") opener := &mockSecretsKeeperOpener{} secrets.DefaultURLMux().RegisterKeeper("test", opener) //nolint:paralleltest t.Run("without override", func(t *testing.T) { - opener.wantURL = "test://foo" - - if _, err := newCloudSecretsManager(stackName, stackConfig, "test://foo"); err != nil { - t.Fatalf("newCloudSecretsManager failed: %v", err) - } - if _, err := newCloudSecretsManager(stackName, stackConfig, "test://bar"); err == nil { - t.Fatal("newCloudSecretsManager with unexpected secretsProvider URL succeeded, expected an error") - } + createTempFiles(t, files, func() { + opener.wantURL = "test://foo" + _, createSecretsManagerError := newCloudSecretsManager(stackName, stackConfigFileName, "test://foo") + assert.Nil(t, createSecretsManagerError, "Creating the cloud secret manager should succeed") + + _, createSecretsManagerError = newCloudSecretsManager(stackName, stackConfigFileName, "test://bar") + assert.NotNil(t, createSecretsManagerError, "newCloudSecretsManager with unexpected secretsProvider URL succeeded, expected an error") + }) }) //nolint:paralleltest t.Run("with override", func(t *testing.T) { - opener.wantURL = "test://bar" - t.Setenv("PULUMI_CLOUD_SECRET_OVERRIDE", "test://bar") - - // Last argument here shouldn't matter anymore, since it gets overridden - // by the env var. Both calls should succeed. - if _, err := newCloudSecretsManager(stackName, stackConfig, "test://foo"); err != nil { - t.Fatalf("newCloudSecretsManager failed: %v", err) - } - if _, err := newCloudSecretsManager(stackName, stackConfig, "test://bar"); err != nil { - t.Fatalf("newCloudSecretsManager failed: %v", err) - } + createTempFiles(t, files, func() { + opener.wantURL = "test://bar" + t.Setenv("PULUMI_CLOUD_SECRET_OVERRIDE", "test://bar") + + // Last argument here shouldn't matter anymore, since it gets overridden + // by the env var. Both calls should succeed. + _, createSecretsManagerError := newCloudSecretsManager(stackName, stackConfigFileName, "test://foo") + assert.Nil(t, createSecretsManagerError, "creating the secrets manager should succeed regardless of secrets provider #1") + _, createSecretsManagerError = newCloudSecretsManager(stackName, stackConfigFileName, "test://bar") + assert.Nil(t, createSecretsManagerError, "creating the secrets manager should succeed regardless of secrets provider #2") + }) }) } From 46be886affc1186437133a050547a738b14eec1e Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Mon, 26 Sep 2022 17:49:57 +0200 Subject: [PATCH 18/32] lint --- pkg/cmd/pulumi/crypto_cloud_test.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pkg/cmd/pulumi/crypto_cloud_test.go b/pkg/cmd/pulumi/crypto_cloud_test.go index 6f8a7aef4514..d609ff58f78f 100644 --- a/pkg/cmd/pulumi/crypto_cloud_test.go +++ b/pkg/cmd/pulumi/crypto_cloud_test.go @@ -27,15 +27,8 @@ import ( "gocloud.dev/secrets/driver" ) -func deleteFile(t *testing.T, file *os.File) { - if file != nil { - err := os.Remove(file.Name()) - assert.NoError(t, err, "Error while deleting file") - } -} - func deleteFiles(t *testing.T, files map[string]string) { - for file, _ := range files { + for file := range files { err := os.Remove(file) assert.Nil(t, err, "Should be able to remove the file directory") } @@ -75,7 +68,8 @@ func TestSecretsProviderOverride(t *testing.T) { assert.Nil(t, createSecretsManagerError, "Creating the cloud secret manager should succeed") _, createSecretsManagerError = newCloudSecretsManager(stackName, stackConfigFileName, "test://bar") - assert.NotNil(t, createSecretsManagerError, "newCloudSecretsManager with unexpected secretsProvider URL succeeded, expected an error") + msg := "newCloudSecretsManager with unexpected secretsProvider URL succeeded, expected an error" + assert.NotNil(t, createSecretsManagerError, msg) }) }) @@ -87,10 +81,11 @@ func TestSecretsProviderOverride(t *testing.T) { // Last argument here shouldn't matter anymore, since it gets overridden // by the env var. Both calls should succeed. + msg := "creating the secrets manager should succeed regardless of secrets provider" _, createSecretsManagerError := newCloudSecretsManager(stackName, stackConfigFileName, "test://foo") - assert.Nil(t, createSecretsManagerError, "creating the secrets manager should succeed regardless of secrets provider #1") + assert.Nil(t, createSecretsManagerError, msg) _, createSecretsManagerError = newCloudSecretsManager(stackName, stackConfigFileName, "test://bar") - assert.Nil(t, createSecretsManagerError, "creating the secrets manager should succeed regardless of secrets provider #2") + assert.Nil(t, createSecretsManagerError, msg) }) }) } From 27556385d089f8a0d2fb98d7c07918eb8d4d46de Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Tue, 27 Sep 2022 13:11:37 +0200 Subject: [PATCH 19/32] Fix TestDestroyStackRef: don't change CWD and fix TestStackInitValidation (less rigid contains check) --- tests/integration/integration_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index dd580dea2e80..f5e8ae37a46b 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -146,9 +146,7 @@ func TestStackInitValidation(t *testing.T) { stdout, stderr := e.RunCommandExpectError("pulumi", "stack", "init", "valid-name") assert.Equal(t, "", stdout) - assert.Contains(t, stderr, - "error: could not get cloud url: could not load current project: "+ - "invalid YAML file: yaml: line 1: did not find expected key") + assert.Contains(t, stderr, "could not load current project: invalid YAML file") }) } @@ -988,7 +986,6 @@ func TestDestroyStackRef(t *testing.T) { e.RunCommand("pulumi", "up", "--skip-preview", "--yes") - e.CWD = os.TempDir() e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "-s", "dev") } From afeac6e9bec3dd8d1730aa0652a53e3149d5f080 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Tue, 27 Sep 2022 14:33:53 +0200 Subject: [PATCH 20/32] simplify test assert even more (attempt #3) --- tests/integration/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index f5e8ae37a46b..2a009e91daa5 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -146,7 +146,7 @@ func TestStackInitValidation(t *testing.T) { stdout, stderr := e.RunCommandExpectError("pulumi", "stack", "init", "valid-name") assert.Equal(t, "", stdout) - assert.Contains(t, stderr, "could not load current project: invalid YAML file") + assert.Contains(t, stderr, "invalid YAML file") }) } From 9a9a39bbf9e6a70615120dc8605106179d9fb8a9 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Tue, 4 Oct 2022 21:13:43 +0200 Subject: [PATCH 21/32] Make sure config defined by a stack are also defined by the project + a whole bunch of tests --- pkg/cmd/pulumi/config.go | 15 +-- pkg/cmd/pulumi/destroy.go | 4 +- pkg/cmd/pulumi/import.go | 11 ++- pkg/cmd/pulumi/logs.go | 18 +++- pkg/cmd/pulumi/preview.go | 13 ++- pkg/cmd/pulumi/refresh.go | 11 ++- pkg/cmd/pulumi/up.go | 20 ++-- pkg/cmd/pulumi/watch.go | 11 ++- pkg/testing/integration/pulumi.go | 9 ++ sdk/go/common/workspace/config.go | 99 +++++++++++++++++-- sdk/go/common/workspace/loaders.go | 2 +- sdk/go/common/workspace/project_test.go | 122 +++++++++++++++++++++++- tests/config_test.go | 25 +++++ 13 files changed, 307 insertions(+), 53 deletions(-) diff --git a/pkg/cmd/pulumi/config.go b/pkg/cmd/pulumi/config.go index 6676e6bb3a62..55bc254e9c03 100644 --- a/pkg/cmd/pulumi/config.go +++ b/pkg/cmd/pulumi/config.go @@ -39,10 +39,6 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) -type StackConfigOptions struct { - applyProjectConfig bool -} - func newConfigCmd() *cobra.Command { var stack string var showSecrets bool @@ -987,8 +983,7 @@ func looksLikeSecret(k config.Key, v string) bool { // it is uses instead of the default configuration file for the stack func getStackConfiguration( ctx context.Context, stack backend.Stack, - sm secrets.Manager, - options StackConfigOptions) (backend.StackConfiguration, error) { + sm secrets.Manager) (backend.StackConfiguration, error) { var cfg config.Map project, _, err := readProject() defaultStackConfig := backend.StackConfiguration{} @@ -1005,14 +1000,6 @@ func getStackConfiguration( "stack configuration could not be loaded from either Pulumi.yaml or the backend: %w", err) } } else { - if options.applyProjectConfig { - stackName := stack.Ref().Name().String() - configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, project, workspaceStack.Config) - if configErr != nil { - return defaultStackConfig, configErr - } - } - cfg = workspaceStack.Config } diff --git a/pkg/cmd/pulumi/destroy.go b/pkg/cmd/pulumi/destroy.go index 57adef0b2101..131bbe183fd5 100644 --- a/pkg/cmd/pulumi/destroy.go +++ b/pkg/cmd/pulumi/destroy.go @@ -157,9 +157,7 @@ func newDestroyCmd() *cobra.Command { sm = snap.SecretsManager } - cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ - applyProjectConfig: true, - }) + cfg, err := getStackConfiguration(ctx, s, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) diff --git a/pkg/cmd/pulumi/import.go b/pkg/cmd/pulumi/import.go index ed064dd69055..7ea98e6bbfc4 100644 --- a/pkg/cmd/pulumi/import.go +++ b/pkg/cmd/pulumi/import.go @@ -43,6 +43,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" javagen "github.com/pulumi/pulumi-java/pkg/codegen/java" yamlgen "github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/codegen" @@ -492,14 +493,16 @@ func newImportCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ - applyProjectConfig: true, - }) - + cfg, err := getStackConfiguration(ctx, s, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + opts.Engine = engine.UpdateOptions{ Parallel: parallel, Debug: debug, diff --git a/pkg/cmd/pulumi/logs.go b/pkg/cmd/pulumi/logs.go index cb7b435de4d3..989c57608726 100644 --- a/pkg/cmd/pulumi/logs.go +++ b/pkg/cmd/pulumi/logs.go @@ -27,6 +27,7 @@ import ( "github.com/pulumi/pulumi/pkg/v3/operations" "github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) // We use RFC 5424 timestamps with millisecond precision for displaying time stamps on log entries. Go does not @@ -57,6 +58,12 @@ func newLogsCmd() *cobra.Command { Color: cmdutil.GetGlobalColorization(), } + // Fetch the project. + proj, _, err := readProject() + if err != nil { + return err + } + s, err := requireStack(ctx, stack, false, opts, false /*setCurrent*/) if err != nil { return err @@ -67,15 +74,16 @@ func newLogsCmd() *cobra.Command { return fmt.Errorf("getting secrets manager: %w", err) } - cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ - // we don't need project config here - applyProjectConfig: false, - }) - + cfg, err := getStackConfiguration(ctx, s, sm) if err != nil { return fmt.Errorf("getting stack configuration: %w", err) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return fmt.Errorf("validating stack config: %w", configErr) + } + startTime, err := parseSince(since, time.Now()) if err != nil { return fmt.Errorf("failed to parse argument to '--since' as duration or timestamp: %w", err) diff --git a/pkg/cmd/pulumi/preview.go b/pkg/cmd/pulumi/preview.go index b082604fa20e..96a44cb35387 100644 --- a/pkg/cmd/pulumi/preview.go +++ b/pkg/cmd/pulumi/preview.go @@ -29,6 +29,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) func newPreviewCmd() *cobra.Command { @@ -148,9 +149,15 @@ func newPreviewCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ - applyProjectConfig: true, - }) + cfg, err := getStackConfiguration(ctx, s, sm) + if err != nil { + return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) + } + + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) diff --git a/pkg/cmd/pulumi/refresh.go b/pkg/cmd/pulumi/refresh.go index d3119ed48e01..252eb05de643 100644 --- a/pkg/cmd/pulumi/refresh.go +++ b/pkg/cmd/pulumi/refresh.go @@ -33,6 +33,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/common/resource" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) func newRefreshCmd() *cobra.Command { @@ -146,14 +147,16 @@ func newRefreshCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ - applyProjectConfig: true, - }) - + cfg, err := getStackConfiguration(ctx, s, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + if skipPendingCreates && clearPendingCreates { return result.FromError(fmt.Errorf( "cannot set both --skip-pending-creates and --clear-pending-creates")) diff --git a/pkg/cmd/pulumi/up.go b/pkg/cmd/pulumi/up.go index 37a99f7db407..7de2b8f7f28e 100644 --- a/pkg/cmd/pulumi/up.go +++ b/pkg/cmd/pulumi/up.go @@ -106,14 +106,16 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ - applyProjectConfig: true, - }) - + cfg, err := getStackConfiguration(ctx, s, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + targetURNs, replaceURNs := []resource.URN{}, []resource.URN{} if len(targets)+len(replaces)+len(targetReplaces) > 0 { @@ -341,14 +343,16 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ - applyProjectConfig: true, - }) - + cfg, err := getStackConfiguration(ctx, s, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + refreshOption, err := getRefreshOption(proj, refresh) if err != nil { return result.FromError(err) diff --git a/pkg/cmd/pulumi/watch.go b/pkg/cmd/pulumi/watch.go index 2042fae75d6a..32f45875430d 100644 --- a/pkg/cmd/pulumi/watch.go +++ b/pkg/cmd/pulumi/watch.go @@ -26,6 +26,7 @@ import ( "github.com/pulumi/pulumi/pkg/v3/engine" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" ) // intentionally disabling here for cleaner err declaration/assignment. @@ -111,14 +112,16 @@ func newWatchCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm, StackConfigOptions{ - applyProjectConfig: true, - }) - + cfg, err := getStackConfiguration(ctx, s, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } + configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config) + if configErr != nil { + return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) + } + opts.Engine = engine.UpdateOptions{ LocalPolicyPacks: engine.MakeLocalPolicyPacks(policyPackPaths, policyPackConfigPaths), Parallel: parallel, diff --git a/pkg/testing/integration/pulumi.go b/pkg/testing/integration/pulumi.go index 25ada9777352..dbe7c28e7217 100644 --- a/pkg/testing/integration/pulumi.go +++ b/pkg/testing/integration/pulumi.go @@ -38,6 +38,15 @@ func CreateBasicPulumiRepo(e *testing.Environment) { assert.NoError(e, err, "writing %s file", filePath) } +// CreateBasicPulumiRepo will initialize the environment with a basic Pulumi repository and +// project file definition. Returns the repo owner and name used. +func CreatePulumiRepo(e *testing.Environment, projectFileContent string) { + e.RunCommand("git", "init") + filePath := path.Join(e.CWD, fmt.Sprintf("%s.yaml", workspace.ProjectFile)) + err := ioutil.WriteFile(filePath, []byte(projectFileContent), os.ModePerm) + assert.NoError(e, err, "writing %s file", filePath) +} + // GetStacks returns the list of stacks and current stack by scraping `pulumi stack ls`. // Assumes .pulumi is in the current working directory. Fails the test on IO errors. func GetStacks(e *testing.Environment) ([]string, *string) { diff --git a/sdk/go/common/workspace/config.go b/sdk/go/common/workspace/config.go index 2f32118dd4aa..06a157bdce05 100644 --- a/sdk/go/common/workspace/config.go +++ b/sdk/go/common/workspace/config.go @@ -3,15 +3,94 @@ package workspace import ( "encoding/json" "fmt" + "sort" "strings" "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" ) +func formatMissingKeys(missingKeys []string) string { + if len(missingKeys) == 1 { + return fmt.Sprintf("'%v'", missingKeys[0]) + } + + sort.Strings(missingKeys) + + formattedMissingKeys := "" + for index, key := range missingKeys { + // if last index, then use and before the key + if index == len(missingKeys)-1 { + formattedMissingKeys += fmt.Sprintf("and '%s'", key) + } else if index == len(missingKeys)-2 { + // no comma before the last key + formattedMissingKeys += fmt.Sprintf("'%s' ", key) + } else { + formattedMissingKeys += fmt.Sprintf("'%s', ", key) + } + } + + return formattedMissingKeys +} + +func missingStackConfigurationKeysError(missingKeys []string, stackName string) error { + valueOrValues := "value" + if len(missingKeys) > 1 { + valueOrValues = "values" + } + + return fmt.Errorf( + "Stack '%v' is missing configuration %v %v", + stackName, + valueOrValues, + formatMissingKeys(missingKeys)) +} + +func missingProjectConfigurationKeysError(missingProjectKeys []string, stackName string) error { + valueOrValues := "value" + if len(missingProjectKeys) > 1 { + valueOrValues = "values" + } + + isOrAre := "is" + if len(missingProjectKeys) > 1 { + isOrAre = "are" + } + + return fmt.Errorf( + "Stack '%v' uses configuration %v %v which %v not defined by the project configuration", + stackName, + valueOrValues, + formatMissingKeys(missingProjectKeys), + isOrAre) +} + func ValidateStackConfigAndApplyProjectConfig( stackName string, project *Project, stackConfig config.Map) error { + + if len(project.Config) > 0 { + // only when the project defines config values, do we need to validate the stack config + // for each stack config key, check if it is in the project config + stackConfigKeysNotDefinedByProject := []string{} + for key := range stackConfig { + namespacedKey := fmt.Sprintf("%s:%s", key.Namespace(), key.Name()) + if key.Namespace() == string(project.Name) { + // then the namespace is implied and can be omitted + namespacedKey = key.Name() + } + + if _, ok := project.Config[namespacedKey]; !ok { + stackConfigKeysNotDefinedByProject = append(stackConfigKeysNotDefinedByProject, namespacedKey) + } + } + + if len(stackConfigKeysNotDefinedByProject) > 0 { + return missingProjectConfigurationKeysError(stackConfigKeysNotDefinedByProject, stackName) + } + } + + missingConfigurationKeys := make([]string, 0) for projectConfigKey, projectConfigType := range project.Config { var key config.Key if strings.Contains(projectConfigKey, ":") { @@ -28,14 +107,16 @@ func ValidateStackConfigAndApplyProjectConfig( key = config.MustMakeKey(string(project.Name), projectConfigKey) } - stackValue, found, _ := stackConfig.Get(key, true) + stackValue, found, err := stackConfig.Get(key, true) + if err != nil { + return fmt.Errorf("Error while getting stack config value for key '%v': %v", key.String(), err) + } + hasDefault := projectConfigType.Default != nil if !found && !hasDefault { - missingConfigError := fmt.Errorf( - "Stack '%v' missing configuration value '%v'", - stackName, - projectConfigKey) - return missingConfigError + // add it to the list to collect all missing configuration keys, + // then return them as a single error + missingConfigurationKeys = append(missingConfigurationKeys, projectConfigKey) } else if !found && hasDefault { // not found at the stack level // but has a default value at the project level @@ -83,5 +164,11 @@ func ValidateStackConfigAndApplyProjectConfig( } } + if len(missingConfigurationKeys) > 0 { + // there are missing configuration keys in the stack + // return them as a single error. + return missingStackConfigurationKeysError(missingConfigurationKeys, stackName) + } + return nil } diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index b4b81d8b39c5..114157fcd9ca 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -228,7 +228,7 @@ func (singleton *projectStackLoader) load(project *Project, path string) (*Proje // is valid configuration and will be rewritten in the form // // config: - // {projectName}:instanceSize:t3.micro + // {projectName}:instanceSize: t3.micro projectStackWithNamespacedConfig := stackConfigNamespacedWithProject(project, simplifiedStackForm) modifiedProjectStack, _ := marshaller.Marshal(projectStackWithNamespacedConfig) diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index d2646d502332..967aedbd3279 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -536,7 +536,127 @@ config: assert.NoError(t, stackError, "Should be able to read the stack") configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) assert.NotNil(t, configError, "there should be a config type error") - assert.Contains(t, configError.Error(), "Stack 'dev' missing configuration value 'values'") + assert.Contains(t, configError.Error(), "Stack 'dev' is missing configuration value 'values'") +} + +func TestStackConfigErrorsWhenMissingTwoStackValueForConfigTypeWithNoDefault(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + another: + type: string + values: + type: array + items: + type: string` + + projectStackYaml := `` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, configError.Error(), "Stack 'dev' is missing configuration values 'another' and 'values'") +} + +func TestStackConfigErrorsWhenMissingMultipleStackValueForConfigTypeWithNoDefault(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + hello: + type: integer + values: + type: array + items: + type: string + world: + type: string` + + projectStackYaml := `` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + assert.Contains(t, configError.Error(), "Stack 'dev' is missing configuration values 'hello', 'values' and 'world'") +} + +func TestStackConfigErrorsWhenUsingConfigValuesNotDefinedByProject(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + hello: + type: integer` + + projectStackYaml := ` +config: + hello: 21 + world: 42` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + expectedErrorMsg := "Stack 'dev' uses configuration value 'world' which is not defined by the project configuration" + assert.Contains(t, configError.Error(), expectedErrorMsg) +} + +func TestStackConfigErrorsWhenUsingMultipleConfigValuesNotDefinedByProject(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet +config: + hello: + type: integer` + + projectStackYaml := ` +config: + hello: 21 + world: 42 + another: 42` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.NotNil(t, configError, "there should be a config type error") + expectedErrorMsg := "Stack 'dev' uses configuration values 'another' and 'world'" + + " which are not defined by the project configuration" + assert.Contains(t, configError.Error(), expectedErrorMsg) +} + +func TestStackConfigDoesNotErrorWhenProjectHasNotDefinedConfig(t *testing.T) { + t.Parallel() + projectYaml := ` +name: test +runtime: dotnet` + + projectStackYaml := ` +config: + hello: 21 + world: 42 + another: 42` + + project, projectError := loadProjectFromText(t, projectYaml) + assert.NoError(t, projectError, "Shold be able to load the project") + stack, stackError := loadProjectStackFromText(t, project, projectStackYaml) + assert.NoError(t, stackError, "Should be able to read the stack") + configError := ValidateStackConfigAndApplyProjectConfig("dev", project, stack.Config) + assert.Nil(t, configError, "there should not be a config type error") } func TestProjectLoadYAML(t *testing.T) { diff --git a/tests/config_test.go b/tests/config_test.go index bc76b09a0d2a..5621803f166e 100644 --- a/tests/config_test.go +++ b/tests/config_test.go @@ -274,3 +274,28 @@ $` e.RunCommand("pulumi", "stack", "rm", "--yes") }) } + +func TestBasicConfigGetRetrievedValueFromProject(t *testing.T) { + t.Parallel() + + e := ptesting.NewEnvironment(t) + defer func() { + if !t.Failed() { + e.DeleteEnvironment() + } + }() + + pulumiProject := ` +name: pulumi-test +runtime: go +config: + first-value: + type: string + default: first` + + integration.CreatePulumiRepo(e, pulumiProject) + e.SetBackend(e.LocalURL()) + e.RunCommand("pulumi", "stack", "init", "test") + stdout, _ := e.RunCommand("pulumi", "config", "get", "first-value") + assert.Equal(t, "first", strings.Trim(stdout, "\r\n")) +} From c739adbe7ce1a906afb454ee7386ad079a585622 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Tue, 4 Oct 2022 21:29:42 +0200 Subject: [PATCH 22/32] Fix tests that have incomplete project config. Correct docs for CreatePulumiRepo --- pkg/testing/integration/pulumi.go | 5 +++-- sdk/go/common/workspace/project_test.go | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/testing/integration/pulumi.go b/pkg/testing/integration/pulumi.go index dbe7c28e7217..1c74600da636 100644 --- a/pkg/testing/integration/pulumi.go +++ b/pkg/testing/integration/pulumi.go @@ -38,8 +38,9 @@ func CreateBasicPulumiRepo(e *testing.Environment) { assert.NoError(e, err, "writing %s file", filePath) } -// CreateBasicPulumiRepo will initialize the environment with a basic Pulumi repository and -// project file definition. Returns the repo owner and name used. +// CreatePulumiRepo will initialize the environment with a basic Pulumi repository and +// project file definition based on the project file content. +// Returns the repo owner and name used. func CreatePulumiRepo(e *testing.Environment, projectFileContent string) { e.RunCommand("git", "init") filePath := path.Join(e.CWD, fmt.Sprintf("%s.yaml", workspace.ProjectFile)) diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index 967aedbd3279..739e91dbbdef 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -365,7 +365,8 @@ func TestNamespacedConfigValuesAreInheritedCorrectly(t *testing.T) { name: test runtime: dotnet config: - aws:region: us-west-1` + aws:region: us-west-1 + instanceSize: t3.micro` projectStackYaml := ` config: @@ -391,7 +392,8 @@ func TestLoadingStackConfigWithoutNamespacingTheProject(t *testing.T) { name: test runtime: dotnet config: - aws:region: us-west-1` + aws:region: us-west-1 + instanceSize: t3.micro` projectStackYaml := ` config: From 4cb2c19610fe0db9a061441df3c932c31db21b53 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Wed, 5 Oct 2022 15:10:19 +0200 Subject: [PATCH 23/32] Use "omit empty" for config types, add another integration test, checks that pulumi config goes through project config --- sdk/go/common/workspace/project.go | 15 ++++++++----- tests/config_test.go | 35 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 63485ee7ee6a..ecabeadce0ef 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -99,15 +99,15 @@ type Plugins struct { } type ProjectConfigItemsType struct { - Type string `json:"type" yaml:"type"` - Items *ProjectConfigItemsType `json:"items" yaml:"items"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Items *ProjectConfigItemsType `json:"items,omitempty" yaml:"items,omitempty"` } type ProjectConfigType struct { - Type string `json:"type" yaml:"type"` - Description string `json:"description" yaml:"description"` - Items *ProjectConfigItemsType `json:"items" yaml:"items"` - Default interface{} `json:"default" yaml:"default"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Items *ProjectConfigItemsType `json:"items,omitempty" yaml:"items,omitempty"` + Default interface{} `json:"default,omitempty" yaml:"default,omitempty"` } // Project is a Pulumi project manifest. @@ -350,6 +350,9 @@ func InferFullTypeName(typeName string, itemsType *ProjectConfigItemsType) strin return typeName } +// ValidateConfig validates the config value against its config type definition. +// We use this to validate the default config values alongside their type definition but +// also to validate config values coming from individual stacks. func ValidateConfigValue(typeName string, itemsType *ProjectConfigItemsType, value interface{}) bool { if typeName == "string" { diff --git a/tests/config_test.go b/tests/config_test.go index 5621803f166e..d53117664935 100644 --- a/tests/config_test.go +++ b/tests/config_test.go @@ -15,6 +15,7 @@ package tests import ( + "encoding/json" "os" "path/filepath" "regexp" @@ -299,3 +300,37 @@ config: stdout, _ := e.RunCommand("pulumi", "config", "get", "first-value") assert.Equal(t, "first", strings.Trim(stdout, "\r\n")) } + +func TestConfigGetRetrievedValueFromBothStackAndProjectUsingJson(t *testing.T) { + t.Parallel() + + e := ptesting.NewEnvironment(t) + defer func() { + if !t.Failed() { + e.DeleteEnvironment() + } + }() + + pulumiProject := ` +name: pulumi-test +runtime: go +config: + first-value: + type: string + default: first + second-value: + type: string` + + integration.CreatePulumiRepo(e, pulumiProject) + e.SetBackend(e.LocalURL()) + e.RunCommand("pulumi", "stack", "init", "test") + e.RunCommand("pulumi", "config", "set", "second-value", "second") + stdout, _ := e.RunCommand("pulumi", "config", "--json") + // check that stdout is an array containing 2 objects + var config map[string]interface{} + jsonError := json.Unmarshal([]byte(stdout), &config) + assert.Nil(t, jsonError) + assert.Equal(t, 2, len(config)) + assert.Equal(t, "first", config["pulumi-test:first-value"].(map[string]interface{})["value"]) + assert.Equal(t, "second", config["pulumi-test:second-value"].(map[string]interface{})["value"]) +} From 813ac76fecadf3e824a8d2c0f2423923978261d9 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Thu, 6 Oct 2022 12:33:28 +0200 Subject: [PATCH 24/32] Add array config to test that the array is marshalled/unmarshalled correctly --- tests/config_test.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/config_test.go b/tests/config_test.go index d53117664935..7232e28fe271 100644 --- a/tests/config_test.go +++ b/tests/config_test.go @@ -319,7 +319,12 @@ config: type: string default: first second-value: - type: string` + type: string + third-value: + type: array + items: + type: string + default: [third]` integration.CreatePulumiRepo(e, pulumiProject) e.SetBackend(e.LocalURL()) @@ -330,7 +335,10 @@ config: var config map[string]interface{} jsonError := json.Unmarshal([]byte(stdout), &config) assert.Nil(t, jsonError) - assert.Equal(t, 2, len(config)) + assert.Equal(t, 3, len(config)) assert.Equal(t, "first", config["pulumi-test:first-value"].(map[string]interface{})["value"]) assert.Equal(t, "second", config["pulumi-test:second-value"].(map[string]interface{})["value"]) + thirdValue := config["pulumi-test:third-value"].(map[string]interface{}) + assert.Equal(t, "[\"third\"]", thirdValue["value"]) + assert.Equal(t, []interface{}{"third"}, thirdValue["objectValue"]) } From 7c509d2346c13bfb0ba36609d51c8d318f0cd9e8 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Fri, 7 Oct 2022 14:03:20 +0200 Subject: [PATCH 25/32] Assume configFile is non-empty --- pkg/backend/filestate/crypto.go | 6 +----- pkg/backend/httpstate/crypto.go | 6 +----- pkg/cmd/pulumi/crypto_cloud.go | 5 +---- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/pkg/backend/filestate/crypto.go b/pkg/backend/filestate/crypto.go index b4b3c677f2eb..7379ea47dcbe 100644 --- a/pkg/backend/filestate/crypto.go +++ b/pkg/backend/filestate/crypto.go @@ -26,15 +26,11 @@ func NewPassphraseSecretsManager(stackName tokens.Name, configFile string, rotatePassphraseSecretsProvider bool) (secrets.Manager, error) { contract.Assertf(stackName != "", "stackName %s", "!= \"\"") - project, path, err := workspace.DetectProjectStackPath(stackName.Q()) + project, _, err := workspace.DetectProjectStackPath(stackName.Q()) if err != nil { return nil, err } - if configFile == "" { - configFile = path - } - info, err := workspace.LoadProjectStack(project, configFile) if err != nil { return nil, err diff --git a/pkg/backend/httpstate/crypto.go b/pkg/backend/httpstate/crypto.go index 4e8023ff2c05..1d3ad0296021 100644 --- a/pkg/backend/httpstate/crypto.go +++ b/pkg/backend/httpstate/crypto.go @@ -25,15 +25,11 @@ import ( func NewServiceSecretsManager(s Stack, stackName tokens.Name, configFile string) (secrets.Manager, error) { contract.Assertf(stackName != "", "stackName %s", "!= \"\"") - project, path, err := workspace.DetectProjectStackPath(stackName.Q()) + project, _, err := workspace.DetectProjectStackPath(stackName.Q()) if err != nil { return nil, err } - if configFile == "" { - configFile = path - } - info, err := workspace.LoadProjectStack(project, configFile) if err != nil { return nil, err diff --git a/pkg/cmd/pulumi/crypto_cloud.go b/pkg/cmd/pulumi/crypto_cloud.go index 961e1846bf55..8b49cf3b09c2 100644 --- a/pkg/cmd/pulumi/crypto_cloud.go +++ b/pkg/cmd/pulumi/crypto_cloud.go @@ -27,13 +27,10 @@ import ( func newCloudSecretsManager(stackName tokens.Name, configFile, secretsProvider string) (secrets.Manager, error) { contract.Assertf(stackName != "", "stackName %s", "!= \"\"") - proj, path, err := workspace.DetectProjectStackPath(stackName.Q()) + proj, _, err := workspace.DetectProjectStackPath(stackName.Q()) if err != nil { return nil, err } - if configFile == "" { - configFile = path - } info, err := workspace.LoadProjectStack(proj, configFile) if err != nil { From 12f399225e3ac6f3107d9745d5666e7d08d724df Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Wed, 12 Oct 2022 15:43:25 +0200 Subject: [PATCH 26/32] Revert changing the temp dir TestDestroyStackRef --- tests/integration/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 2a009e91daa5..996d287a6cbc 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -985,7 +985,7 @@ func TestDestroyStackRef(t *testing.T) { e.RunCommand("yarn", "install") e.RunCommand("pulumi", "up", "--skip-preview", "--yes") - + e.CWD = os.TempDir() e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "-s", "dev") } From c60675790e950fb43700b7bc0f3d9f112d49d661 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Wed, 12 Oct 2022 16:01:49 +0200 Subject: [PATCH 27/32] fix TestProjectLoadYAML expected error messages --- sdk/go/common/workspace/project_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/go/common/workspace/project_test.go b/sdk/go/common/workspace/project_test.go index f8d05b8b39db..3e20b9c35cbe 100644 --- a/sdk/go/common/workspace/project_test.go +++ b/sdk/go/common/workspace/project_test.go @@ -666,7 +666,7 @@ func TestProjectLoadYAML(t *testing.T) { // Test wrong type _, err := loadProjectFromText(t, "\"hello\"") - assert.Equal(t, "expected an object", err.Error()) + assert.Contains(t, err.Error(), "expected project to be an object") // Test bad key _, err = loadProjectFromText(t, "4: hello") From 42a51feb04574c8fa1b832154c2b4be53085293e Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Wed, 12 Oct 2022 16:23:16 +0200 Subject: [PATCH 28/32] Allow project to be nil when namespacing project stack config --- sdk/go/common/workspace/loaders.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sdk/go/common/workspace/loaders.go b/sdk/go/common/workspace/loaders.go index 920e6c92de17..c5e6f9ba5b2f 100644 --- a/sdk/go/common/workspace/loaders.go +++ b/sdk/go/common/workspace/loaders.go @@ -143,14 +143,20 @@ type projectStackLoader struct { // Rewrite config values to make them namespaced. Using the project name as the default namespace // for example: -// config: -// instanceSize: t3.micro +// +// config: +// instanceSize: t3.micro // // is valid configuration and will be rewritten in the form // -// config: -// {projectName}:instanceSize:t3.micro +// config: +// {projectName}:instanceSize:t3.micro func stackConfigNamespacedWithProject(project *Project, projectStack map[string]interface{}) map[string]interface{} { + if project == nil { + // return the original config if we don't have a project + return projectStack + } + config, ok := projectStack["config"] if ok { configAsMap, isMap := config.(map[string]interface{}) From 51b1329f5985f3bd017d37283c654b7c97fc6979 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Wed, 12 Oct 2022 16:29:41 +0200 Subject: [PATCH 29/32] remove readProject from getStackConfiguration and pass project from outside --- pkg/cmd/pulumi/config.go | 9 ++++----- pkg/cmd/pulumi/destroy.go | 2 +- pkg/cmd/pulumi/import.go | 2 +- pkg/cmd/pulumi/logs.go | 2 +- pkg/cmd/pulumi/preview.go | 2 +- pkg/cmd/pulumi/refresh.go | 2 +- pkg/cmd/pulumi/up.go | 4 ++-- pkg/cmd/pulumi/watch.go | 2 +- 8 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pkg/cmd/pulumi/config.go b/pkg/cmd/pulumi/config.go index 55bc254e9c03..507a1976c6b2 100644 --- a/pkg/cmd/pulumi/config.go +++ b/pkg/cmd/pulumi/config.go @@ -982,14 +982,13 @@ func looksLikeSecret(k config.Key, v string) bool { // getStackConfiguration loads configuration information for a given stack. If stackConfigFile is non empty, // it is uses instead of the default configuration file for the stack func getStackConfiguration( - ctx context.Context, stack backend.Stack, + ctx context.Context, + stack backend.Stack, + project *workspace.Project, sm secrets.Manager) (backend.StackConfiguration, error) { var cfg config.Map - project, _, err := readProject() + defaultStackConfig := backend.StackConfiguration{} - if err != nil { - return defaultStackConfig, err - } workspaceStack, err := loadProjectStack(project, stack) if err != nil || workspaceStack == nil { diff --git a/pkg/cmd/pulumi/destroy.go b/pkg/cmd/pulumi/destroy.go index 5f8711ff72c6..8ecc50ca5516 100644 --- a/pkg/cmd/pulumi/destroy.go +++ b/pkg/cmd/pulumi/destroy.go @@ -159,7 +159,7 @@ func newDestroyCmd() *cobra.Command { sm = snap.SecretsManager } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) diff --git a/pkg/cmd/pulumi/import.go b/pkg/cmd/pulumi/import.go index 7ea98e6bbfc4..81f6386ed2a0 100644 --- a/pkg/cmd/pulumi/import.go +++ b/pkg/cmd/pulumi/import.go @@ -493,7 +493,7 @@ func newImportCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } diff --git a/pkg/cmd/pulumi/logs.go b/pkg/cmd/pulumi/logs.go index 989c57608726..973fd85e2806 100644 --- a/pkg/cmd/pulumi/logs.go +++ b/pkg/cmd/pulumi/logs.go @@ -74,7 +74,7 @@ func newLogsCmd() *cobra.Command { return fmt.Errorf("getting secrets manager: %w", err) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return fmt.Errorf("getting stack configuration: %w", err) } diff --git a/pkg/cmd/pulumi/preview.go b/pkg/cmd/pulumi/preview.go index 33c7822b898c..f98dffd4700d 100644 --- a/pkg/cmd/pulumi/preview.go +++ b/pkg/cmd/pulumi/preview.go @@ -149,7 +149,7 @@ func newPreviewCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } diff --git a/pkg/cmd/pulumi/refresh.go b/pkg/cmd/pulumi/refresh.go index 252eb05de643..d06287f84cc7 100644 --- a/pkg/cmd/pulumi/refresh.go +++ b/pkg/cmd/pulumi/refresh.go @@ -147,7 +147,7 @@ func newRefreshCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } diff --git a/pkg/cmd/pulumi/up.go b/pkg/cmd/pulumi/up.go index 5253235e1ee2..5643a04add38 100644 --- a/pkg/cmd/pulumi/up.go +++ b/pkg/cmd/pulumi/up.go @@ -106,7 +106,7 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } @@ -345,7 +345,7 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } diff --git a/pkg/cmd/pulumi/watch.go b/pkg/cmd/pulumi/watch.go index 32f45875430d..f1de99ee8819 100644 --- a/pkg/cmd/pulumi/watch.go +++ b/pkg/cmd/pulumi/watch.go @@ -112,7 +112,7 @@ func newWatchCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting secrets manager: %w", err)) } - cfg, err := getStackConfiguration(ctx, s, sm) + cfg, err := getStackConfiguration(ctx, s, proj, sm) if err != nil { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } From fb113471c656a4ed62732cf9323f875314888294 Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Thu, 13 Oct 2022 19:52:09 +0200 Subject: [PATCH 30/32] No need to rewrite project when validating project schema and refactor test to assert full stackConfigDir error message --- sdk/go/common/resource/config/value.go | 11 +- sdk/go/common/workspace/paths_test.go | 4 +- sdk/go/common/workspace/project.go | 31 +- sdk/go/common/workspace/project.json | 590 +++++++++++++------------ 4 files changed, 318 insertions(+), 318 deletions(-) diff --git a/sdk/go/common/resource/config/value.go b/sdk/go/common/resource/config/value.go index 0be5451b61cd..613990013fec 100644 --- a/sdk/go/common/resource/config/value.go +++ b/sdk/go/common/resource/config/value.go @@ -139,13 +139,8 @@ func (c Value) ToObject() (interface{}, error) { return c.unmarshalObjectJSON() } -// MarshalValue returns the underlying content of the config value -func (c Value) MarshalValue() (interface{}, error) { - return c.marshalValue() -} - func (c Value) MarshalJSON() ([]byte, error) { - v, err := c.marshalValue() + v, err := c.MarshalValue() if err != nil { return nil, err } @@ -163,7 +158,7 @@ func (c *Value) UnmarshalJSON(b []byte) error { } func (c Value) MarshalYAML() (interface{}, error) { - return c.marshalValue() + return c.MarshalValue() } func (c *Value) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -207,7 +202,7 @@ func (c *Value) unmarshalValue(unmarshal func(interface{}) error, fix func(inter return nil } -func (c Value) marshalValue() (interface{}, error) { +func (c Value) MarshalValue() (interface{}, error) { if c.object { return c.unmarshalObjectJSON() } diff --git a/sdk/go/common/workspace/paths_test.go b/sdk/go/common/workspace/paths_test.go index 2139f9e364a2..be36e18a64c0 100644 --- a/sdk/go/common/workspace/paths_test.go +++ b/sdk/go/common/workspace/paths_test.go @@ -90,7 +90,9 @@ func TestProjectStackPath(t *testing.T) { "name: some_project\ndescription: Some project\nruntime: nodejs\nconfig: stacksA\nstackConfigDir: stacksB\n", func(t *testing.T, projectDir, path string, err error) { assert.Error(t, err) - assert.Contains(t, err.Error(), "Should not use both config and stackConfigDir") + errorMsg := "Should not use both config and stackConfigDir to define the stack directory. " + + "Use only stackConfigDir instead." + assert.Contains(t, err.Error(), errorMsg) }, }} diff --git a/sdk/go/common/workspace/project.go b/sdk/go/common/workspace/project.go index 3b2e38aa67e0..bcaffb8eb841 100644 --- a/sdk/go/common/workspace/project.go +++ b/sdk/go/common/workspace/project.go @@ -157,22 +157,16 @@ type Project struct { } func isPrimitiveValue(value interface{}) (string, bool) { - _, isLiteralBoolean := value.(bool) - if isLiteralBoolean { - return "boolean", true - } - - _, isLiteralInt := value.(int) - if isLiteralInt { - return "integer", true - } - - _, isLiteralString := value.(string) - if isLiteralString { + switch value.(type) { + case string: return "string", true + case int: + return "integer", true + case bool: + return "boolean", true + default: + return "", false } - - return "", false } // RewriteConfigPathIntoStackConfigDir checks if the project is using the old "config" property @@ -303,15 +297,6 @@ func ValidateProject(raw interface{}) error { return errors.New("project is missing a 'runtime' attribute") } - project, rewriteError := RewriteConfigPathIntoStackConfigDir(project) - - // when defining both config and stackConfigDir as strings in the same project - if rewriteError != nil { - return rewriteError - } - - project = RewriteShorthandConfigValues(project) - // Let everything else be caught by jsonschema if err = ProjectSchema.Validate(project); err == nil { return nil diff --git a/sdk/go/common/workspace/project.json b/sdk/go/common/workspace/project.json index 1c6b103be313..5af69a3323b2 100644 --- a/sdk/go/common/workspace/project.json +++ b/sdk/go/common/workspace/project.json @@ -1,307 +1,325 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/pulumi/pulumi/blob/master/sdk/go/common/workspace/project.json", - "title": "Pulumi Project", - "description": "A schema for Pulumi project files.", - "type": "object", - "properties": { - "name": { - "description": "Name of the project containing alphanumeric characters, hyphens, underscores, and periods.", - "type": "string", - "minLength": 1 - }, - "description": { - "description": "Description of the project.", - "type": [ - "string", - "null" - ] - }, - "author": { - "description": "Author is an optional author that created this project.", - "type": [ - "string", - "null" - ] - }, - "website": { - "description": "Website is an optional website for additional info about this project.", - "type": [ - "string", - "null" - ] - }, - "license": { - "description": "License is the optional license governing this project's usage.", - "type": [ - "string", - "null" - ] - }, - "runtime": { - "title": "ProjectRuntimeInfo", - "oneOf": [ - { - "title": "Name", - "type": "string", - "minLength": 1 + "$schema":"https://json-schema.org/draft/2020-12/schema", + "$id":"https://github.com/pulumi/pulumi/blob/master/sdk/go/common/workspace/project.json", + "title":"Pulumi Project", + "description":"A schema for Pulumi project files.", + "type":"object", + "properties":{ + "name":{ + "description":"Name of the project containing alphanumeric characters, hyphens, underscores, and periods.", + "type":"string", + "minLength":1 }, - { - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string", - "minLength": 1 - }, - "options": { - "title": "Options", - "type": "object", - "additionalProperties": true - } - }, - "additionalProperties": false - } - ] - }, - "main": { - "description": "Path to the Pulumi program. The default is the working directory.", - "type": [ - "string", - "null" - ] - }, - "config": { - "description": "A map of configuration keys to their types. Using config directory location relative to the location of Pulumi.yaml is a deprecated use of this key. Use stackConfigDir instead.", - "type": ["object", "null"], - "additionalProperties": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - }, - { - "type": "boolean" - }, - { - "$ref": "#/$defs/configTypeDeclaration" - } - ] - } - }, - "stackConfigDir": { - "description": "Config directory location relative to the location of Pulumi.yaml.", - "type": [ - "string", - "null" - ] - }, - "backend": { - "description": "Backend of the project.", - "type": [ - "object", - "null" - ], - "properties": { - "url": { - "description": "URL is optional field to explicitly set backend url", - "type": "string" - } - }, - "additionalProperties": false - }, - "options": { - "description": "Additional project options.", - "type": [ - "object", - "null" - ], - "properties": { - "refresh": { - "description": "Set to \"always\" to refresh the state before performing a Pulumi operation.", - "type": "string", - "const": "always" - } - }, - "additionalProperties": false - }, - "template": { - "title": "ProjectTemplate", - "description": "ProjectTemplate is a Pulumi project template manifest.", - "type": [ - "object", - "null" - ], - "properties": { - "description": { - "description": "Description of the template.", - "type": [ - "string", - "null" - ] + "description":{ + "description":"Description of the project.", + "type":[ + "string", + "null" + ] }, - "quickstart": { - "description": "Quickstart contains optional text to be displayed after template creation.", - "type": [ - "string", - "null" - ] + "author":{ + "description":"Author is an optional author that created this project.", + "type":[ + "string", + "null" + ] }, - "important": { - "description": "Important indicates the template is important and should be listed by default.", - "type": [ - "boolean", - "null" - ] + "website":{ + "description":"Website is an optional website for additional info about this project.", + "type":[ + "string", + "null" + ] }, - "config": { - "description": "Config to apply to each stack in the project.", - "type": [ - "object", - "null" - ], - "additionalProperties": { - "properties": { - "description": { - "description": "Description of the config.", - "type": [ - "string", - "null" - ] - }, - "default": { - "description": "Default value of the config." - }, - "secret": { - "description": "Boolean indicating if the configuration is labeled as a secret.", - "type": [ - "boolean", - "null" + "license":{ + "description":"License is the optional license governing this project's usage.", + "type":[ + "string", + "null" + ] + }, + "runtime":{ + "title":"ProjectRuntimeInfo", + "oneOf":[ + { + "title":"Name", + "type":"string", + "minLength":1 + }, + { + "type":"object", + "properties":{ + "name":{ + "title":"Name", + "type":"string", + "minLength":1 + }, + "options":{ + "title":"Options", + "type":"object", + "additionalProperties":true + } + }, + "additionalProperties":false + } + ] + }, + "main":{ + "description":"Path to the Pulumi program. The default is the working directory.", + "type":[ + "string", + "null" + ] + }, + "config":{ + "description":"A map of configuration keys to their types. Using config directory location relative to the location of Pulumi.yaml is a deprecated use of this key. Use stackConfigDir instead.", + "type":[ + "object", + "null" + ], + "additionalProperties":{ + "oneOf":[ + { + "type":"string" + }, + { + "type":"integer" + }, + { + "type":"boolean" + }, + { + "$ref":"#/$defs/configTypeDeclaration" + } ] - } } - } - } - }, - "additionalProperties": false - }, - "plugins": { - "description": "Override for the plugin selection. Intended for use in developing pulumi plugins.", - "type": "object", - "properties": { - "providers": { - "description": "Plugins for resource providers.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" - } }, - "analyzers": { - "description": "Plugins for policy analyzers.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" - } + "stackConfigDir":{ + "description":"Config directory location relative to the location of Pulumi.yaml.", + "type":[ + "string", + "null" + ] }, - "languages": { - "description": "Plugins for languages.", - "type": "array", - "items": { - "$ref": "#/$defs/pluginOptions" - } - } - } - } - }, - "required": [ - "name", - "runtime" - ], - "additionalProperties": true, - "$defs": { - "pluginOptions": { - "title": "PluginOptions", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "Name of the plugin" + "backend":{ + "description":"Backend of the project.", + "type":[ + "object", + "null" + ], + "properties":{ + "url":{ + "description":"URL is optional field to explicitly set backend url", + "type":"string" + } + }, + "additionalProperties":false }, - "path": { - "type": "string", - "description": "Path to the plugin folder" + "options":{ + "description":"Additional project options.", + "type":[ + "object", + "null" + ], + "properties":{ + "refresh":{ + "description":"Set to \"always\" to refresh the state before performing a Pulumi operation.", + "type":"string", + "const":"always" + } + }, + "additionalProperties":false }, - "version": { - "type": "string", - "description": "Version of the plugin, if not set, will match any version the engine requests." - } - } - }, - "simpleConfigType": { - "title": "SimpleConfigType", - "enum": ["string","integer", "boolean", "array"] - }, - "configItemsType": { - "type": "object", - "required": ["type"], - "properties": { - "type": { - "oneOf": [ - { "$ref": "#/$defs/simpleConfigType" }, - { "$ref": "#/$defs/configItemsType" } - ] - }, - "items": { - "$ref": "#/$defs/configItemsType" - } + "template":{ + "title":"ProjectTemplate", + "description":"ProjectTemplate is a Pulumi project template manifest.", + "type":[ + "object", + "null" + ], + "properties":{ + "description":{ + "description":"Description of the template.", + "type":[ + "string", + "null" + ] + }, + "quickstart":{ + "description":"Quickstart contains optional text to be displayed after template creation.", + "type":[ + "string", + "null" + ] + }, + "important":{ + "description":"Important indicates the template is important and should be listed by default.", + "type":[ + "boolean", + "null" + ] + }, + "config":{ + "description":"Config to apply to each stack in the project.", + "type":[ + "object", + "null" + ], + "additionalProperties":{ + "properties":{ + "description":{ + "description":"Description of the config.", + "type":[ + "string", + "null" + ] + }, + "default":{ + "description":"Default value of the config." + }, + "secret":{ + "description":"Boolean indicating if the configuration is labeled as a secret.", + "type":[ + "boolean", + "null" + ] + } + } + } + } + }, + "additionalProperties":false }, - "if": { - "properties": { - "type": { - "const": "array" + "plugins":{ + "description":"Override for the plugin selection. Intended for use in developing pulumi plugins.", + "type":"object", + "properties":{ + "providers":{ + "description":"Plugins for resource providers.", + "type":"array", + "items":{ + "$ref":"#/$defs/pluginOptions" + } + }, + "analyzers":{ + "description":"Plugins for policy analyzers.", + "type":"array", + "items":{ + "$ref":"#/$defs/pluginOptions" + } + }, + "languages":{ + "description":"Plugins for languages.", + "type":"array", + "items":{ + "$ref":"#/$defs/pluginOptions" + } + } } - } - }, - "then": { - "required": ["items"] } }, - "configTypeDeclaration": { - "title": "ConfigTypeDeclaration", - "type": "object", - "additionalProperties": false, - "required": [ - "type" - ], - "if": { - "properties": { - "type": { - "const": "array" - } - } - }, - "then": { - "required": ["items"] - }, - "properties": { - "type": { - "$ref": "#/$defs/simpleConfigType" - }, - "items": { - "$ref": "#/$defs/configItemsType" + "required":[ + "name", + "runtime" + ], + "additionalProperties":true, + "$defs":{ + "pluginOptions":{ + "title":"PluginOptions", + "type":"object", + "properties":{ + "name":{ + "type":"string", + "description":"Name of the plugin" + }, + "path":{ + "type":"string", + "description":"Path to the plugin folder" + }, + "version":{ + "type":"string", + "description":"Version of the plugin, if not set, will match any version the engine requests." + } + } }, - "description": { - "type": "string" + "simpleConfigType":{ + "title":"SimpleConfigType", + "enum":[ + "string", + "integer", + "boolean", + "array" + ] }, - "secret": { - "type": "boolean" + "configItemsType":{ + "type":"object", + "required":[ + "type" + ], + "properties":{ + "type":{ + "oneOf":[ + { + "$ref":"#/$defs/simpleConfigType" + }, + { + "$ref":"#/$defs/configItemsType" + } + ] + }, + "items":{ + "$ref":"#/$defs/configItemsType" + } + }, + "if":{ + "properties":{ + "type":{ + "const":"array" + } + } + }, + "then":{ + "required":[ + "items" + ] + } }, - "default": { - + "configTypeDeclaration":{ + "title":"ConfigTypeDeclaration", + "type":"object", + "additionalProperties":false, + "required":[ + "type" + ], + "if":{ + "properties":{ + "type":{ + "const":"array" + } + } + }, + "then":{ + "required":[ + "items" + ] + }, + "properties":{ + "type":{ + "$ref":"#/$defs/simpleConfigType" + }, + "items":{ + "$ref":"#/$defs/configItemsType" + }, + "description":{ + "type":"string" + }, + "secret":{ + "type":"boolean" + }, + "default":{ + + } + } } - } } - } } \ No newline at end of file From 58de26d7d8b6d5b1fdd3c9da494a09c52f9a3dbb Mon Sep 17 00:00:00 2001 From: Zaid Ajaj Date: Thu, 13 Oct 2022 20:09:32 +0200 Subject: [PATCH 31/32] Don't know why the change in the project schema was reverted but here is it: allow config to be string for validation --- sdk/go/common/workspace/project.json | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/go/common/workspace/project.json b/sdk/go/common/workspace/project.json index 5af69a3323b2..c1aa08d8e427 100644 --- a/sdk/go/common/workspace/project.json +++ b/sdk/go/common/workspace/project.json @@ -75,6 +75,7 @@ "description":"A map of configuration keys to their types. Using config directory location relative to the location of Pulumi.yaml is a deprecated use of this key. Use stackConfigDir instead.", "type":[ "object", + "string", "null" ], "additionalProperties":{ From f2ac21f38b55de44b0cfee5aaaf210f5a7ec4f41 Mon Sep 17 00:00:00 2001 From: Fraser Waters Date: Mon, 17 Oct 2022 16:03:49 +0100 Subject: [PATCH 32/32] Update changelog/pending/20220922--cli-initial-mvp-config.yaml --- changelog/pending/20220922--cli-initial-mvp-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/pending/20220922--cli-initial-mvp-config.yaml b/changelog/pending/20220922--cli-initial-mvp-config.yaml index 6f0307f94ad3..233135232623 100644 --- a/changelog/pending/20220922--cli-initial-mvp-config.yaml +++ b/changelog/pending/20220922--cli-initial-mvp-config.yaml @@ -1,4 +1,4 @@ changes: - type: feat scope: cli - description: Implement initial MVP for hierarchical and structured project configuration + description: Implement initial MVP for hierarchical and structured project configuration.