Eject with missing nodes (#393)

* Eject with missing nodes

* Allow ejecting nodes like foo:bar

* Inject missing config nodes

* Provide raw config names to PCL

* CL

* Fix tests

* Support config logcal names

CHANGELOG_PENDING.md

@@ -1,34 +1,6 @@
 ### Improvements
 
-- Ensure that constant values passed to enums are valid.
-  [#357](https://github.com/pulumi/pulumi-yaml/pull/357)
-
-- Warn on non camelCase names.
-  [#362](https://github.com/pulumi/pulumi-yaml/pull/362)
-
-- Recognize the new core project-level `config` block.
-  [#369](https://github.com/pulumi/pulumi-yaml/pull/369)
+- Allow ejecting when relying on `config` nodes.
+  [#393](https://github.com/pulumi/pulumi-yaml/pull/393)
 
 ### Bug Fixes
-
-- Allow interpolations for `AssetOrArchive` function values
-  [#341](https://github.com/pulumi/pulumi-yaml/pull/341)
-
-- Clarify the lifetimes when calling `codegen.Eject`. This is a breaking change to the
-  `codegen.Eject` API.
-  [#358](https://github.com/pulumi/pulumi-yaml/pull/358)
-
-- Quote generated strings that could be numbers.
-  [#363](https://github.com/pulumi/pulumi-yaml/issues/363)
-
-- Respect import option on resource.
-  [#367](https://github.com/pulumi/pulumi-yaml/issues/367)
-
-- Discover Invokes during `GetReferencedPlugins`.
-  [#381](https://github.com/pulumi/pulumi-yaml/pull/381)
-
-- Escaped interpolated strings now remove one extra dollar sign.
-  [#382](https://github.com/pulumi/pulumi-yaml/pull/382)
-
-- Only insert "id" in ejected resource refs when the receiver type is a string.
-  [#389](https://github.com/pulumi/pulumi-yaml/pull/389)

pkg/pulumiyaml/codegen/convert.go

@@ -53,56 +53,60 @@ func ConvertTemplateIL(template *ast.TemplateDecl, loader schema.ReferenceLoader
 
 	pkgLoader := pulumiyaml.NewPackageLoaderFromSchemaLoader(loader)
 	// nil runner passed in since template is not executed and we can use pkgLoader
-	_, tdiags, err := pulumiyaml.PrepareTemplate(template, nil, pkgLoader)
+	r, tdiags, err := pulumiyaml.PrepareTemplate(template, nil, pkgLoader)
 	if err != nil {
 		return "", diags, err
 	}
 	diags = diags.Extend(tdiags.HCL())
-	if diags.HasErrors() {
-		return "", diags, nil
-	}
+
+	pulumiyaml.InjectMissingNodes(r, template)
 
 	templateBody, tdiags := ImportTemplate(template, pkgLoader)
 	diags = diags.Extend(tdiags.HCL())
-	if diags.HasErrors() {
-		return "", diags, nil
+	if templateBody == nil {
+		// This is a irrecoverable error, so we make sure the error field is non-nil
+		return "", diags, diags
 	}
 	programText := fmt.Sprintf("%v", templateBody)
 
-	return programText, nil, nil
+	if programText == "" {
+		return "", diags, diags
+	}
+
+	return programText, diags, nil
 }
 
 func EjectProgram(template *ast.TemplateDecl, loader schema.ReferenceLoader) (*pcl.Program, hcl.Diagnostics, error) {
-	programText, diags, err := ConvertTemplateIL(template, loader)
-	if err != nil {
-		return nil, diags, err
-	}
-	if diags.HasErrors() {
-		return nil, diags, fmt.Errorf("internal error: %w", diags)
+	programText, yamlDiags, err := ConvertTemplateIL(template, loader)
+	if err != nil || programText == "" {
+		return nil, yamlDiags, err
 	}
+
 	parser := hclsyntax.NewParser()
 	if err := parser.ParseFile(strings.NewReader(programText), "program.pp"); err != nil {
-		return nil, diags, err
+		return nil, yamlDiags, err
 	}
-	diags = diags.Extend(parser.Diagnostics)
+	diags := parser.Diagnostics
 	if diags.HasErrors() {
-		return nil, diags, nil
+		return nil, append(yamlDiags, diags...), diags
 	}
+
 	bindOpts := []pcl.BindOption{
 		pcl.SkipResourceTypechecking,
 		pcl.AllowMissingProperties,
 		pcl.AllowMissingVariables,
 	}
 	bindOpts = append(bindOpts, pcl.Loader(loader))
 	program, pdiags, err := pcl.BindProgram(parser.Files, bindOpts...)
+	diags = diags.Extend(pdiags)
 	if err != nil {
-		return nil, diags, err
+		return nil, append(yamlDiags, diags...), err
 	}
-	if pdiags.HasErrors() {
-		return nil, diags, fmt.Errorf("internal error: %w", pdiags)
+	if pdiags.HasErrors() || program == nil {
+		return nil, append(yamlDiags, diags...), fmt.Errorf("internal error: %w", pdiags)
 	}
 
-	return program, diags, nil
+	return program, append(yamlDiags, diags...), nil
 }
 
 // ConvertTemplate converts a Pulumi YAML template to a target language using PCL as an intermediate representation.

pkg/pulumiyaml/codegen/load.go

@@ -78,7 +78,7 @@ func (imp *importer) importRef(node ast.Expr, name string, environment map[strin
 	}
 
 	return &model.ScopeTraversalExpression{
-		Traversal: hcl.Traversal{hcl.TraverseRoot{Name: name}},
+		Traversal: hcl.Traversal{hcl.TraverseRoot{Name: camel(makeLegalIdentifier(name))}},
 		Parts:     []model.Traversable{model.DynamicType},
 	}, syntax.Diagnostics{ast.ExprError(node, fmt.Sprintf("unknown config, variable, or resource '%v'", name), "")}
 }
@@ -511,7 +511,7 @@ func (imp *importer) importConfig(kvp ast.ConfigMapEntry) (model.BodyItem, synta
 		typeExpr = "string"
 	}
 
-	configVar, ok := imp.configuration[name]
+	configName, ok := imp.configuration[name]
 	contract.Assert(ok)
 
 	var defaultValue model.Expression
@@ -527,8 +527,15 @@ func (imp *importer) importConfig(kvp ast.ConfigMapEntry) (model.BodyItem, synta
 
 	configDef := &model.Block{
 		Type:   "config",
-		Labels: []string{configVar.Name, typeExpr},
-		Body:   &model.Body{},
+		Labels: []string{configName.Name, typeExpr},
+		Body: &model.Body{
+			Items: []model.BodyItem{
+				&model.Attribute{
+					Name:  pcl.LogicalNamePropertyKey,
+					Value: quotedLit(kvp.Key.GetValue()),
+				},
+			},
+		},
 	}
 	if defaultValue != nil {
 		configDef.Body.Items = append(configDef.Body.Items, &model.Attribute{
@@ -922,6 +929,7 @@ func (imp *importer) assignNames() {
 
 	assign := func(name, suffix string) *model.Variable {
 		assignName := func(name, suffix string) string {
+
 			name = camel(makeLegalIdentifier(name))
 			if !assigned.Has(name) {
 				assigned.Add(name)

pkg/pulumiyaml/run.go

@@ -180,7 +180,12 @@ func HasDiagnostics(err error) (syntax.Diagnostics, bool) {
 	}
 }
 
-func (r *Runner) setDefaultProviders() error {
+// Set default providers for resources and invokes.
+//
+// This function communicates errors by appending to the internal diags field of `r`.
+// It is the responsibility of the caller to verify that no err diags were appended if
+// that should prevent proceeding.
+func (r *Runner) setDefaultProviders() {
 	defaultProviderInfoMap := make(map[string]*providerInfo)
 	for _, resource := range r.t.Resources.Entries {
 		v := resource.Value
@@ -196,7 +201,9 @@ func (r *Runner) setDefaultProviders() error {
 				}
 			}
 		} else if v.DefaultProvider != nil {
-			return errors.New("cannot set defaultProvider on non-provider resource")
+			r.sdiags.Extend(syntax.NodeError(
+				v.DefaultProvider.Syntax(),
+				"cannot set defaultProvider on non-provider resource", ""))
 		}
 	}
 
@@ -282,10 +289,10 @@ func (r *Runner) setDefaultProviders() error {
 		},
 	})
 
-	if diags.HasErrors() {
-		return diags
-	}
-	return nil
+	// This function communicates errors by appending to the internal diags field of `r`.
+	// It is the responsibility of the caller to verify that no err diags were appended if
+	// that should prevent proceeding.
+	contract.IgnoreError(diags)
 }
 
 // PrepareTemplate prepares a template for converting or running
@@ -295,25 +302,27 @@ func PrepareTemplate(t *ast.TemplateDecl, r *Runner, loader PackageLoader) (*Run
 	if r == nil {
 		r = newRunner(t, loader)
 	}
+
+	// We are preemptively calling r.setIntermediates. We are forcing tolerating missing
+	// nodes, ensuring the process can continue even for invalid templates. Diags will
+	// still be reported normally.
+	//
+	// r.setDefaultProviders uses r.setIntermediates, so this line need to precede calls
+	// to r.setDefaultProviders.
+	r.setIntermediates(nil, true /*force*/)
+
 	// runner hooks up default providers
-	err := r.setDefaultProviders()
-	if err != nil {
-		return nil, nil, err
-	}
+	r.setDefaultProviders()
 
 	// runner type checks nodes
 	_, diags := TypeCheck(r)
-	if diags.HasErrors() {
-		return nil, diags, nil
-	}
-
 	return r, diags, nil
 }
 
 // RunTemplate runs the programEvaluator against a template using the given request/settings.
 func RunTemplate(ctx *pulumi.Context, t *ast.TemplateDecl, config map[string]string, loader PackageLoader) error {
 	r := newRunner(t, loader)
-	r.setIntermediates(config)
+	r.setIntermediates(config, false)
 	if r.sdiags.HasErrors() {
 		return &r.sdiags
 	}
@@ -722,21 +731,24 @@ func getPulumiConfNodes(config map[string]string) ([]configNode, error) {
 }
 
 // setIntermediates is called for convert and runtime evaluation
-func (r *Runner) setIntermediates(config map[string]string) {
+//
+// If force is true, set intermediates even if errors were encountered
+// Errors will always be reflected in r.sdiags.
+func (r *Runner) setIntermediates(config map[string]string, force bool) {
 	if r.intermediates != nil {
 		return
 	}
 
 	r.intermediates = []graphNode{}
 	confNodes, err := getPulumiConfNodes(config)
-	if err != nil {
+	if err != nil && !force {
 		r.sdiags.Extend(syntax.Error(nil, err.Error(), ""))
 		return
 	}
 	// Topologically sort the intermediates based on implicit and explicit dependencies
 	intermediates, rdiags := topologicallySortedResources(r.t, confNodes)
 	r.sdiags.Extend(rdiags...)
-	if rdiags.HasErrors() {
+	if rdiags.HasErrors() && !force {
 		return
 	}
 	if intermediates != nil {
@@ -747,7 +759,7 @@ func (r *Runner) setIntermediates(config map[string]string) {
 // ensureSetup is called at runtime evaluation
 func (r *Runner) ensureSetup(ctx *pulumi.Context) {
 	// Our tests need to set intermediates, even though they don't have runtime config
-	r.setIntermediates(nil)
+	r.setIntermediates(nil, false)
 
 	cwd, err := os.Getwd()
 	if err != nil {
@@ -842,6 +854,8 @@ func (r *Runner) Run(e Evaluator) syntax.Diagnostics {
 			if !e.EvalResource(r, kvp) {
 				return returnDiags()
 			}
+		case missingNode:
+			// We ignore this intentionally
 		}
 	}
 
@@ -2166,3 +2180,24 @@ func listStrings(v *ast.StringListDecl) []string {
 	}
 	return a
 }
+
+// Edit the template with the goal of making it valid.
+//
+// This is a best effort function, and does not guarantee a valid template.
+func InjectMissingNodes(r *Runner, template *ast.TemplateDecl) {
+	for _, node := range r.intermediates {
+		if node, ok := node.(missingNode); ok {
+			if isGlobalConfigName(node.name.GetValue()) {
+				template.Configuration.Entries = append(template.Configuration.Entries,
+					ast.ConfigMapEntry{
+						Key:   node.key(),
+						Value: &ast.ConfigParamDecl{},
+					})
+			}
+		}
+	}
+}
+
+func isGlobalConfigName(s string) bool {
+	return strings.Count(s, ":") == 1
+}

pkg/pulumiyaml/run_options_test.go

@@ -160,7 +160,8 @@ variables:
 	}
 	err := pulumi.RunErr(func(ctx *pulumi.Context) error {
 		runner := newRunner(template, newMockPackageMap())
-		assert.Equal(t, runner.setDefaultProviders(), nil)
+		runner.setDefaultProviders()
+		requireNoErrors(t, template, runner.sdiags.diags)
 		diags := runner.Evaluate(ctx)
 		requireNoErrors(t, template, diags)
 		return nil

pkg/pulumiyaml/sort.go

@@ -75,6 +75,18 @@ func (e configNodeEnv) value() interface{} {
 	return e.Value
 }
 
+type missingNode struct {
+	name *ast.StringExpr
+}
+
+func (e missingNode) key() *ast.StringExpr {
+	return e.name
+}
+
+func (missingNode) valueKind() string {
+	return "missing node"
+}
+
 func topologicallySortedResources(t *ast.TemplateDecl, externalConfig []configNode) ([]graphNode, syntax.Diagnostics) {
 	var diags syntax.Diagnostics
 
@@ -167,7 +179,8 @@ func topologicallySortedResources(t *ast.TemplateDecl, externalConfig []configNo
 		e, ok := intermediates[name.Value]
 		if !ok {
 			diags.Extend(ast.ExprError(name, fmt.Sprintf("resource %q not found", name.Value), ""))
-			return false
+			e = missingNode{name}
+			addIntermediate(name.Value, e)
 		}
 		kind := e.valueKind()
 

pkg/tests/transpiled_examples/azure-app-service-pp/azure-app-service.pp

@@ -1,4 +1,5 @@
 config sqlAdmin string {
+	__logicalName = "sqlAdmin"
 	default = "pulumi"
 }
 

pkg/tests/transpiled_examples/azure-container-apps-pp/azure-container-apps.pp

@@ -1,8 +1,10 @@
 config sqlAdmin string {
+	__logicalName = "sqlAdmin"
 	default = "pulumi"
 }
 
 config retentionInDays int {
+	__logicalName = "retentionInDays"
 	default = 30
 }
 

pkg/tests/transpiled_examples/kubernetes-pp/kubernetes.pp

@@ -1,4 +1,5 @@
 config hostname string {
+	__logicalName = "hostname"
 	default = "example.com"
 }
 

pkg/tests/transpiled_examples/webserver-json-pp/webserver-json.pp

@@ -1,4 +1,5 @@
 config instanceType string {
+	__logicalName = "InstanceType"
 	default = "t3.micro"
 }
 

pkg/tests/transpiled_examples/webserver-pp/webserver.pp

@@ -1,4 +1,5 @@
 config instanceType string {
+	__logicalName = "InstanceType"
 	default = "t3.micro"
 }