Merge #11192

11192: Hierarchical config: optional typing extended short-hand syntax r=Zaid-Ajaj a=Zaid-Ajaj

### Description

This PR extends the specs of the hierarchical configuration rules in the following manner:
 - The `type` property/attribute is made _optional_ for project configuration which means stack values overriding this config block will not be validated against the type (integer stack value will override string project value)
 - Stacks can now use non-project config values that don't have to be defined at the project level (because they are not namespaced by the project, i.e. `aws:region`)
 - Non-project config values (i.e. `aws:region`) defined at the project level _cannot_ have a `type` nor `default` properties, only `value`
 - Project config block using short-hand syntax now accept arrays: `pulumi:disable-default-providers: ["*"]`
 - Project config block when defined using `value` can be anything (objects, arrays, primitives)
 - Project config blocks cannot have both `value` and `default` defined at the same time

Fixes #11127
Fixes #11128

See added tests for more details

## Checklist

<!--- Please provide details if the checkbox below is to be left unchecked. -->
- [x] I have added tests that prove my fix is effective or that my feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the Pulumi Service,
then the service should honor older versions of the CLI where this change would not exist.
You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version
  <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. -->


Co-authored-by: Zaid Ajaj <zaid.naom@gmail.com>

changelog/pending/20221029--cli-config--optional-typing-and-extended-short-syntax.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: cli/config
+  description: Typing made optional, extended short-hand values to arrays and correctly pass stack name to config validator

pkg/cmd/pulumi/import.go

@@ -503,7 +503,8 @@ func newImportCmd() *cobra.Command {
 				return result.FromError(fmt.Errorf("getting stack decrypter: %w", err))
 			}
 
-			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config, decrypter)
+			stackName := s.Ref().Name().String()
+			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config, decrypter)
 			if configErr != nil {
 				return result.FromError(fmt.Errorf("validating stack config: %w", configErr))
 			}

pkg/cmd/pulumi/logs.go

@@ -84,7 +84,8 @@ func newLogsCmd() *cobra.Command {
 				return fmt.Errorf("getting stack decrypter: %w", err)
 			}
 
-			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config, decrypter)
+			stackName := s.Ref().Name().String()
+			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config, decrypter)
 			if configErr != nil {
 				return fmt.Errorf("validating stack config: %w", configErr)
 			}

pkg/cmd/pulumi/preview.go

@@ -185,7 +185,8 @@ func newPreviewCmd() *cobra.Command {
 				return result.FromError(fmt.Errorf("getting stack decrypter: %w", err))
 			}
 
-			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config, decrypter)
+			stackName := s.Ref().Name().String()
+			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config, decrypter)
 			if configErr != nil {
 				return result.FromError(fmt.Errorf("validating stack config: %w", configErr))
 			}

pkg/cmd/pulumi/refresh.go

@@ -188,7 +188,8 @@ func newRefreshCmd() *cobra.Command {
 				return result.FromError(fmt.Errorf("getting stack decrypter: %w", err))
 			}
 
-			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config, decrypter)
+			stackName := s.Ref().Name().String()
+			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config, decrypter)
 			if configErr != nil {
 				return result.FromError(fmt.Errorf("validating stack config: %w", configErr))
 			}

pkg/cmd/pulumi/up.go

@@ -119,7 +119,8 @@ func newUpCmd() *cobra.Command {
 			return result.FromError(fmt.Errorf("getting stack decrypter: %w", err))
 		}
 
-		configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config, decrypter)
+		stackName := s.Ref().Name().String()
+		configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config, decrypter)
 		if configErr != nil {
 			return result.FromError(fmt.Errorf("validating stack config: %w", configErr))
 		}
@@ -342,7 +343,8 @@ func newUpCmd() *cobra.Command {
 			return result.FromError(fmt.Errorf("getting stack decrypter: %w", err))
 		}
 
-		configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config, decrypter)
+		stackName := s.Ref().String()
+		configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config, decrypter)
 		if configErr != nil {
 			return result.FromError(fmt.Errorf("validating stack config: %w", configErr))
 		}

pkg/cmd/pulumi/watch.go

@@ -122,7 +122,8 @@ func newWatchCmd() *cobra.Command {
 				return result.FromError(fmt.Errorf("getting stack decrypter: %w", err))
 			}
 
-			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stack, proj, cfg.Config, decrypter)
+			stackName := s.Ref().Name().String()
+			configErr := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, proj, cfg.Config, decrypter)
 			if configErr != nil {
 				return result.FromError(fmt.Errorf("validating stack config: %w", configErr))
 			}

sdk/go/common/workspace/config.go

@@ -45,25 +45,6 @@ func missingStackConfigurationKeysError(missingKeys []string, stackName string)
 		formatMissingKeys(missingKeys))
 }
 
-func missingProjectConfigurationKeysError(missingProjectKeys []string, stackName string) error {
-	valueOrValues := "value"
-	if len(missingProjectKeys) > 1 {
-		valueOrValues = "values"
-	}
-
-	isOrAre := "is"
-	if len(missingProjectKeys) > 1 {
-		isOrAre = "are"
-	}
-
-	return fmt.Errorf(
-		"Stack '%v' uses configuration %v %v which %v not defined by the project configuration",
-		stackName,
-		valueOrValues,
-		formatMissingKeys(missingProjectKeys),
-		isOrAre)
-}
-
 type StackName = string
 type ProjectConfigKey = string
 type StackConfigValidator = func(StackName, ProjectConfigKey, ProjectConfigType, config.Value, config.Decrypter) error
@@ -97,8 +78,8 @@ func DefaultStackConfigValidator(
 		}
 	}
 
-	if !ValidateConfigValue(projectConfigType.Type, projectConfigType.Items, content) {
-		typeName := InferFullTypeName(projectConfigType.Type, projectConfigType.Items)
+	if !ValidateConfigValue(*projectConfigType.Type, projectConfigType.Items, content) {
+		typeName := InferFullTypeName(*projectConfigType.Type, projectConfigType.Items)
 		validationError := fmt.Errorf(
 			"Stack '%v' with configuration key '%v' must be of type '%v'",
 			stackName,
@@ -111,36 +92,44 @@ func DefaultStackConfigValidator(
 	return nil
 }
 
+// The validator which does not validate anything
+// used when we only want to merge the project config onto the stack config
+func NoopStackConfigValidator(
+	stackName string,
+	projectConfigKey string,
+	projectConfigType ProjectConfigType,
+	stackValue config.Value,
+	dec config.Decrypter) error {
+	return nil
+}
+
+func createConfigValue(rawValue interface{}) (config.Value, error) {
+	if isPrimitiveValue(rawValue) {
+		configValueContent := fmt.Sprintf("%v", rawValue)
+		return config.NewValue(configValueContent), nil
+	}
+	value, err := SimplifyMarshalledValue(rawValue)
+	if err != nil {
+		return config.Value{}, err
+	}
+	configValueJSON, jsonError := json.Marshal(value)
+	if jsonError != nil {
+		return config.Value{}, jsonError
+	}
+	return config.NewObjectValue(string(configValueJSON)), nil
+
+}
+
 func ValidateStackConfigAndMergeProjectConfig(
 	stackName string,
 	project *Project,
 	stackConfig config.Map,
 	lazyDecrypter func() config.Decrypter,
 	validate StackConfigValidator) error {
 
-	if len(project.Config) > 0 {
-		// only when the project defines config values, do we need to validate the stack config
-		// for each stack config key, check if it is in the project config
-		stackConfigKeysNotDefinedByProject := []string{}
-		for key := range stackConfig {
-			namespacedKey := fmt.Sprintf("%s:%s", key.Namespace(), key.Name())
-			if key.Namespace() == string(project.Name) {
-				// then the namespace is implied and can be omitted
-				namespacedKey = key.Name()
-			}
-
-			if _, ok := project.Config[namespacedKey]; !ok {
-				stackConfigKeysNotDefinedByProject = append(stackConfigKeysNotDefinedByProject, namespacedKey)
-			}
-		}
-
-		if len(stackConfigKeysNotDefinedByProject) > 0 {
-			return missingProjectConfigurationKeysError(stackConfigKeysNotDefinedByProject, stackName)
-		}
-	}
-
 	var decrypter config.Decrypter
 	missingConfigurationKeys := make([]string, 0)
+	projectName := project.Name.String()
 	for projectConfigKey, projectConfigType := range project.Config {
 		var key config.Key
 		if strings.Contains(projectConfigKey, ":") {
@@ -153,59 +142,61 @@ func ValidateStackConfigAndMergeProjectConfig(
 			key = parsedKey
 		} else {
 			// key is not namespaced
-			// use the project as namespace
-			key = config.MustMakeKey(string(project.Name), projectConfigKey)
+			// use the project as default namespace
+			key = config.MustMakeKey(projectName, projectConfigKey)
 		}
 
-		stackValue, found, err := stackConfig.Get(key, true)
+		stackValue, foundOnStack, err := stackConfig.Get(key, true)
 		if err != nil {
 			return fmt.Errorf("Error while getting stack config value for key '%v': %v", key.String(), err)
 		}
 
 		hasDefault := projectConfigType.Default != nil
-		if !found && !hasDefault {
-			// add it to the list to collect all missing configuration keys,
+		hasValue := projectConfigType.Value != nil
+
+		if !foundOnStack && !hasValue && !hasDefault && key.Namespace() == projectName {
+			// add it to the list of missing project configuration keys in the stack
+			// which are required by the project
 			// then return them as a single error
 			missingConfigurationKeys = append(missingConfigurationKeys, projectConfigKey)
-		} else if !found && hasDefault {
-			// not found at the stack level
-			// but has a default value at the project level
-			// assign the value to the stack
-			var configValue config.Value
-
-			if projectConfigType.Type == "array" {
-				// for array types, JSON-ify the default value
-				configValueJSON, jsonError := json.Marshal(projectConfigType.Default)
-				if jsonError != nil {
-					return jsonError
-				}
-				configValue = config.NewObjectValue(string(configValueJSON))
+			continue
+		}
 
-			} else {
-				// for primitive types
-				// pass the values as is
-				configValueContent := fmt.Sprintf("%v", projectConfigType.Default)
-				configValue = config.NewValue(configValueContent)
+		if !foundOnStack && (hasValue || hasDefault) {
+			// either value or default value is provided
+			var value interface{}
+			if hasValue {
+				value = projectConfigType.Value
+			}
+			if hasDefault {
+				value = projectConfigType.Default
+			}
+			// it is not found on the stack we are currently validating / merging values with
+			// then we assign the value to that stack whatever that value is
+			configValue, err := createConfigValue(value)
+			if err != nil {
+				return err
 			}
-
 			setError := stackConfig.Set(key, configValue, true)
 			if setError != nil {
 				return setError
 			}
-		} else {
-			// Validate stack level value against the config defined at the project level
-			if validate != nil {
-				// we have a validator
-				if decrypter == nil && lazyDecrypter != nil {
-					// initialize the decrypter once
-					decrypter = lazyDecrypter()
-				}
 
-				if decrypter != nil {
-					validationError := validate(stackName, projectConfigKey, projectConfigType, stackValue, decrypter)
-					if validationError != nil {
-						return validationError
-					}
+			continue
+		}
+
+		// Validate stack level value against the config defined at the project level
+		if projectConfigType.IsExplicitlyTyped() {
+			// we have a validator
+			if decrypter == nil {
+				// initialize the decrypter once
+				decrypter = lazyDecrypter()
+			}
+
+			if decrypter != nil {
+				validationError := validate(stackName, projectConfigKey, projectConfigType, stackValue, decrypter)
+				if validationError != nil {
+					return validationError
 				}
 			}
 		}
@@ -238,5 +229,10 @@ func ValidateStackConfigAndApplyProjectConfig(
 // This is because sometimes during pulumi config ls and pulumi config get, if users are
 // using PassphraseDecrypter, we don't want to always prompt for the values when not necessary
 func ApplyProjectConfig(stackName string, project *Project, stackConfig config.Map) error {
-	return ValidateStackConfigAndMergeProjectConfig(stackName, project, stackConfig, nil, nil)
+	emptyDecrypter := func() config.Decrypter {
+		return nil
+	}
+
+	return ValidateStackConfigAndMergeProjectConfig(stackName, project, stackConfig,
+		emptyDecrypter, NoopStackConfigValidator)
 }