Merge #10832

10832: Hierarchical and structured config implementation: the initial pass r=Zaid-Ajaj a=Zaid-Ajaj

# Description

<!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. -->

This implements the initial pass of hierarchical and structured config which fixes #10602.

This changes the CLI such that configuration can now be defined at the _project_ level using a `config` block. The configuration values defined here are inherited by all the stacks and made available to the Pulumi program without having to duplicate values in every stack (hence hierarchical) and the values are also typed / structured. 

Example Project.yaml syntax:
```yaml
name: config-test
runtime: dotnet
config:
  instanceSize:
    type: string
    default: t3.micro
  instanceCount: 
    type: integer
    default: 5
```
This can also be rewritten using short-hand syntax and will be equivalent to the above
```yaml
name: config-test
runtime: dotnet
config:
  instanceSize: t3.micro
  instanceCount: 5
```
The complex types allowed for now are only arrays and nested arrays:
```yaml
name: config-test
runtime: dotnet
config:
  availabilityZones:
    type: array
    items: 
      type: string
    default: [us-east-1-atl-1a, us-east-1-chi-1a]
```


- Project-level configuration values that do not have a default value _MUST_ be defined at the stack level
- Stack configuration values are type-checked against their defined type in the project file i.e. Pulumi.yaml
- Short-hand syntax only accepts primitive values (no arrays for now)
- Accepted config types are a subset of a JSON schema where the property `type: string | integer | boolean | array` is expected. When `type: array` then a config block must also have property `items` which defines the type of array elements (can be nested)
- Running `pulumi config` will list the configuration values from the selected stack _AND_ the values inherited from the project
- After a successful `pulumi up` run using hierarchical config from the project, `pulumi config refresh` will write _ALL_ the used config back to the refreshed stack
- `pulumi config set/rm` only applies to the selected stack

## Checklist

<!--- Please provide details if the checkbox below is to be left unchecked. -->
- [x] I have added tests that prove my fix is effective or that my feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the Pulumi Service,
then the service should honor older versions of the CLI where this change would not exist.
You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version
  <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. -->


Co-authored-by: Zaid Ajaj <zaid.naom@gmail.com>

.gitignore

@@ -16,7 +16,7 @@ venv/
 
 **/.idea/
 *.iml
-
+.yarn
 # VSCode creates this binary when running tests in the debugger
 **/debug.test
 

changelog/pending/20220922--cli-initial-mvp-config.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: cli
+  description: Implement initial MVP for hierarchical and structured project configuration.

pkg/backend/filestate/crypto.go

@@ -26,7 +26,12 @@ func NewPassphraseSecretsManager(stackName tokens.Name, configFile string,
 	rotatePassphraseSecretsProvider bool) (secrets.Manager, error) {
 	contract.Assertf(stackName != "", "stackName %s", "!= \"\"")
 
-	info, err := workspace.LoadProjectStack(configFile)
+	project, _, err := workspace.DetectProjectStackPath(stackName.Q())
+	if err != nil {
+		return nil, err
+	}
+
+	info, err := workspace.LoadProjectStack(project, configFile)
 	if err != nil {
 		return nil, err
 	}

pkg/backend/httpstate/crypto.go

@@ -25,7 +25,12 @@ import (
 func NewServiceSecretsManager(s Stack, stackName tokens.Name, configFile string) (secrets.Manager, error) {
 	contract.Assertf(stackName != "", "stackName %s", "!= \"\"")
 
-	info, err := workspace.LoadProjectStack(configFile)
+	project, _, err := workspace.DetectProjectStackPath(stackName.Q())
+	if err != nil {
+		return nil, err
+	}
+
+	info, err := workspace.LoadProjectStack(project, configFile)
 	if err != nil {
 		return nil, err
 	}

pkg/cmd/pulumi/config.go

@@ -57,12 +57,18 @@ func newConfigCmd() *cobra.Command {
 				Color: cmdutil.GetGlobalColorization(),
 			}
 
+			project, _, err := readProject()
+
+			if err != nil {
+				return err
+			}
+
 			stack, err := requireStack(ctx, stack, true, opts, true /*setCurrent*/)
 			if err != nil {
 				return err
 			}
 
-			return listConfig(ctx, stack, showSecrets, jsonOut)
+			return listConfig(ctx, project, stack, showSecrets, jsonOut)
 		}),
 	}
 
@@ -106,6 +112,12 @@ func newConfigCopyCmd(stack *string) *cobra.Command {
 				Color: cmdutil.GetGlobalColorization(),
 			}
 
+			project, _, err := readProject()
+
+			if err != nil {
+				return err
+			}
+
 			// Get current stack and ensure that it is a different stack to the destination stack
 			currentStack, err := requireStack(ctx, *stack, false, opts, true /*setCurrent*/)
 			if err != nil {
@@ -114,7 +126,7 @@ func newConfigCopyCmd(stack *string) *cobra.Command {
 			if currentStack.Ref().Name().String() == destinationStackName {
 				return errors.New("current stack and destination stack are the same")
 			}
-			currentProjectStack, err := loadProjectStack(currentStack)
+			currentProjectStack, err := loadProjectStack(project, currentStack)
 			if err != nil {
 				return err
 			}
@@ -124,7 +136,7 @@ func newConfigCopyCmd(stack *string) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			destinationProjectStack, err := loadProjectStack(destinationStack)
+			destinationProjectStack, err := loadProjectStack(project, destinationStack)
 			if err != nil {
 				return err
 			}
@@ -302,7 +314,12 @@ func newConfigRmCmd(stack *string) *cobra.Command {
 				Color: cmdutil.GetGlobalColorization(),
 			}
 
-			s, err := requireStack(ctx, *stack, true, opts, true /*setCurrent*/)
+			project, _, err := readProject()
+			if err != nil {
+				return err
+			}
+
+			stack, err := requireStack(ctx, *stack, true, opts, true /*setCurrent*/)
 			if err != nil {
 				return err
 			}
@@ -312,7 +329,7 @@ func newConfigRmCmd(stack *string) *cobra.Command {
 				return fmt.Errorf("invalid configuration key: %w", err)
 			}
 
-			ps, err := loadProjectStack(s)
+			ps, err := loadProjectStack(project, stack)
 			if err != nil {
 				return err
 			}
@@ -322,7 +339,7 @@ func newConfigRmCmd(stack *string) *cobra.Command {
 				return err
 			}
 
-			return saveProjectStack(s, ps)
+			return saveProjectStack(stack, ps)
 		}),
 	}
 	rmCmd.PersistentFlags().BoolVar(
@@ -351,12 +368,17 @@ func newConfigRmAllCmd(stack *string) *cobra.Command {
 				Color: cmdutil.GetGlobalColorization(),
 			}
 
-			s, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/)
+			project, _, err := readProject()
 			if err != nil {
 				return err
 			}
 
-			ps, err := loadProjectStack(s)
+			stack, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/)
+			if err != nil {
+				return err
+			}
+
+			ps, err := loadProjectStack(project, stack)
 			if err != nil {
 				return err
 			}
@@ -373,7 +395,7 @@ func newConfigRmAllCmd(stack *string) *cobra.Command {
 				}
 			}
 
-			return saveProjectStack(s, ps)
+			return saveProjectStack(stack, ps)
 		}),
 	}
 	rmAllCmd.PersistentFlags().BoolVar(
@@ -395,6 +417,11 @@ func newConfigRefreshCmd(stack *string) *cobra.Command {
 				Color: cmdutil.GetGlobalColorization(),
 			}
 
+			project, _, err := readProject()
+			if err != nil {
+				return err
+			}
+
 			// Ensure the stack exists.
 			s, err := requireStack(ctx, *stack, false, opts, false /*setCurrent*/)
 			if err != nil {
@@ -411,7 +438,7 @@ func newConfigRefreshCmd(stack *string) *cobra.Command {
 				return err
 			}
 
-			ps, err := workspace.LoadProjectStack(configPath)
+			ps, err := workspace.LoadProjectStack(project, configPath)
 			if err != nil {
 				return err
 			}
@@ -480,6 +507,12 @@ func newConfigSetCmd(stack *string) *cobra.Command {
 				Color: cmdutil.GetGlobalColorization(),
 			}
 
+			project, _, err := readProject()
+
+			if err != nil {
+				return err
+			}
+
 			// Ensure the stack exists.
 			s, err := requireStack(ctx, *stack, true, opts, true /*setCurrent*/)
 			if err != nil {
@@ -538,7 +571,7 @@ func newConfigSetCmd(stack *string) *cobra.Command {
 				}
 			}
 
-			ps, err := loadProjectStack(s)
+			ps, err := loadProjectStack(project, s)
 			if err != nil {
 				return err
 			}
@@ -591,13 +624,18 @@ func newConfigSetAllCmd(stack *string) *cobra.Command {
 				Color: cmdutil.GetGlobalColorization(),
 			}
 
+			project, _, err := readProject()
+			if err != nil {
+				return err
+			}
+
 			// Ensure the stack exists.
-			s, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/)
+			stack, err := requireStack(ctx, *stack, true, opts, false /*setCurrent*/)
 			if err != nil {
 				return err
 			}
 
-			ps, err := loadProjectStack(s)
+			ps, err := loadProjectStack(project, stack)
 			if err != nil {
 				return err
 			}
@@ -620,7 +658,7 @@ func newConfigSetAllCmd(stack *string) *cobra.Command {
 				if err != nil {
 					return err
 				}
-				c, cerr := getStackEncrypter(s)
+				c, cerr := getStackEncrypter(stack)
 				if cerr != nil {
 					return cerr
 				}
@@ -636,7 +674,7 @@ func newConfigSetAllCmd(stack *string) *cobra.Command {
 				}
 			}
 
-			return saveProjectStack(s, ps)
+			return saveProjectStack(stack, ps)
 		}),
 	}
 
@@ -680,16 +718,17 @@ var stackConfigFile string
 
 func getProjectStackPath(stack backend.Stack) (string, error) {
 	if stackConfigFile == "" {
-		return workspace.DetectProjectStackPath(stack.Ref().Name().Q())
+		_, path, err := workspace.DetectProjectStackPath(stack.Ref().Name().Q())
+		return path, err
 	}
 	return stackConfigFile, nil
 }
 
-func loadProjectStack(stack backend.Stack) (*workspace.ProjectStack, error) {
+func loadProjectStack(project *workspace.Project, stack backend.Stack) (*workspace.ProjectStack, error) {
 	if stackConfigFile == "" {
 		return workspace.DetectProjectStack(stack.Ref().Name().Q())
 	}
-	return workspace.LoadProjectStack(stackConfigFile)
+	return workspace.LoadProjectStack(project, stackConfigFile)
 }
 
 func saveProjectStack(stack backend.Stack, ps *workspace.ProjectStack) error {
@@ -741,12 +780,25 @@ type configValueJSON struct {
 	Secret      bool        `json:"secret"`
 }
 
-func listConfig(ctx context.Context, stack backend.Stack, showSecrets bool, jsonOut bool) error {
-	ps, err := loadProjectStack(stack)
+func listConfig(ctx context.Context,
+	project *workspace.Project,
+	stack backend.Stack,
+	showSecrets bool,
+	jsonOut bool) error {
+
+	ps, err := loadProjectStack(project, stack)
 	if err != nil {
 		return err
 	}
 
+	stackName := stack.Ref().Name().String()
+	// when listing configuration values
+	// also show values coming from the project
+	configError := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, project, ps.Config)
+	if configError != nil {
+		return configError
+	}
+
 	cfg := ps.Config
 
 	// By default, we will use a blinding decrypter to show "[secret]". If requested, display secrets in plaintext.
@@ -827,10 +879,19 @@ func listConfig(ctx context.Context, stack backend.Stack, showSecrets bool, json
 }
 
 func getConfig(ctx context.Context, stack backend.Stack, key config.Key, path, jsonOut bool) error {
-	ps, err := loadProjectStack(stack)
+	project, _, err := readProject()
 	if err != nil {
 		return err
 	}
+	ps, err := loadProjectStack(project, stack)
+	if err != nil {
+		return err
+	}
+	stackName := stack.Ref().Name().String()
+	configError := workspace.ValidateStackConfigAndApplyProjectConfig(stackName, project, ps.Config)
+	if configError != nil {
+		return configError
+	}
 
 	cfg := ps.Config
 
@@ -921,15 +982,20 @@ func looksLikeSecret(k config.Key, v string) bool {
 // getStackConfiguration loads configuration information for a given stack. If stackConfigFile is non empty,
 // it is uses instead of the default configuration file for the stack
 func getStackConfiguration(
-	ctx context.Context, stack backend.Stack,
+	ctx context.Context,
+	stack backend.Stack,
+	project *workspace.Project,
 	sm secrets.Manager) (backend.StackConfiguration, error) {
 	var cfg config.Map
-	workspaceStack, err := loadProjectStack(stack)
+
+	defaultStackConfig := backend.StackConfiguration{}
+
+	workspaceStack, err := loadProjectStack(project, stack)
 	if err != nil || workspaceStack == nil {
 		// On first run or the latest configuration is unavailable, fallback to check the project's configuration
 		cfg, err = backend.GetLatestConfiguration(ctx, stack)
 		if err != nil {
-			return backend.StackConfiguration{}, fmt.Errorf(
+			return defaultStackConfig, fmt.Errorf(
 				"stack configuration could not be loaded from either Pulumi.yaml or the backend: %w", err)
 		}
 	} else {
@@ -948,8 +1014,9 @@ func getStackConfiguration(
 
 	crypter, err := sm.Decrypter()
 	if err != nil {
-		return backend.StackConfiguration{}, fmt.Errorf("getting configuration decrypter: %w", err)
+		return defaultStackConfig, fmt.Errorf("getting configuration decrypter: %w", err)
 	}
+
 	return backend.StackConfiguration{
 		Config:    cfg,
 		Decrypter: crypter,

pkg/cmd/pulumi/crypto.go

@@ -45,7 +45,13 @@ func getStackDecrypter(s backend.Stack) (config.Decrypter, error) {
 }
 
 func getStackSecretsManager(s backend.Stack) (secrets.Manager, error) {
-	ps, err := loadProjectStack(s)
+	project, _, err := readProject()
+
+	if err != nil {
+		return nil, err
+	}
+
+	ps, err := loadProjectStack(project, s)
 	if err != nil {
 		return nil, err
 	}

pkg/cmd/pulumi/crypto_cloud.go

@@ -27,8 +27,12 @@ import (
 
 func newCloudSecretsManager(stackName tokens.Name, configFile, secretsProvider string) (secrets.Manager, error) {
 	contract.Assertf(stackName != "", "stackName %s", "!= \"\"")
+	proj, _, err := workspace.DetectProjectStackPath(stackName.Q())
+	if err != nil {
+		return nil, err
+	}
 
-	info, err := workspace.LoadProjectStack(configFile)
+	info, err := workspace.LoadProjectStack(proj, configFile)
 	if err != nil {
 		return nil, err
 	}