Merge #10913

10913: Allow more flexible parsing when extracting examples from descriptions r=iwahbe a=iwahbe

<!--- 
Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation.
-->

# Description

<!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. -->

Fixes #10614

## Checklist

<!--- Please provide details if the checkbox below is to be left unchecked. -->
- [X] I have added tests that prove my fix is effective or that my feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [X] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the Pulumi Service,
then the service should honor older versions of the CLI where this change would not exist.
You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version
  <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. -->


Co-authored-by: Ian Wahbe <ian@wahbe.com>

changelog/pending/20221004--docs--more-flexible-doc-parsing.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: fix
+  scope: docs
+  description: Allow more flexible parsing when extracting examples from doc comments

pkg/codegen/docs/examples.go

@@ -55,10 +55,41 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo {
 
 	var examplesShortcode *schema.Shortcode
 	var exampleShortcode *schema.Shortcode
-	var title string
-	var snippets map[string]string
 	var examples []exampleSection
+	currentSection := exampleSection{
+		Snippets: map[string]string{},
+	}
+	var nextTitle string
+	var nextInferredTitle string
+	// Push any examples we have found. Since `pushExamples` is called between sections,
+	// it needs to behave correctly when no examples were found.
+	pushExamples := func() {
+		if len(currentSection.Snippets) > 0 {
+			for _, l := range dctx.snippetLanguages {
+				if _, ok := currentSection.Snippets[l]; !ok {
+					currentSection.Snippets[l] = defaultMissingExampleSnippetPlaceholder
+				}
+			}
+
+			examples = append(examples, currentSection)
+		}
+		if nextTitle == "" {
+			nextTitle = nextInferredTitle
+		}
+		currentSection = exampleSection{
+			Snippets: map[string]string{},
+			Title:    nextTitle,
+		}
+		nextTitle = ""
+		nextInferredTitle = ""
+	}
 	err := ast.Walk(parsed, func(n ast.Node, enter bool) (ast.WalkStatus, error) {
+		// ast.Walk visits each node twice. The first time descending and the second time
+		// ascending. We only want to view the nodes while descending, so we skip when
+		// `enter` is false.
+		if !enter {
+			return ast.WalkContinue, nil
+		}
 		if shortcode, ok := n.(*schema.Shortcode); ok {
 			name := string(shortcode.Name)
 			switch name {
@@ -68,49 +99,62 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo {
 				}
 			case schema.ExampleShortcode:
 				if exampleShortcode == nil {
-					exampleShortcode, title, snippets = shortcode, "", map[string]string{}
+					exampleShortcode = shortcode
+					currentSection.Title, currentSection.Snippets = "", map[string]string{}
 				} else if !enter && shortcode == exampleShortcode {
-					for _, l := range dctx.snippetLanguages {
-						if _, ok := snippets[l]; !ok {
-							snippets[l] = defaultMissingExampleSnippetPlaceholder
-						}
-					}
-
-					examples = append(examples, exampleSection{
-						Title:    title,
-						Snippets: snippets,
-					})
-
+					pushExamples()
 					exampleShortcode = nil
 				}
 			}
 			return ast.WalkContinue, nil
 		}
+
+		// We check to make sure we are in an examples section.
 		if exampleShortcode == nil {
 			return ast.WalkContinue, nil
 		}
 
 		switch n := n.(type) {
 		case *ast.Heading:
-			if n.Level == 3 && title == "" {
-				title = strings.TrimSpace(schema.RenderDocsToString(source, n))
+			if n.Level == 3 {
+				title := strings.TrimSpace(schema.RenderDocsToString(source, n))
+				if currentSection.Title == "" && len(currentSection.Snippets) == 0 {
+					currentSection.Title = title
+				} else {
+					nextTitle = title
+				}
 			}
+			return ast.WalkSkipChildren, nil
+
 		case *ast.FencedCodeBlock:
 			language := string(n.Language(source))
-			if !languages.Has(language) {
+			snippet := schema.RenderDocsToString(source, n)
+			if !languages.Has(language) || len(snippet) == 0 {
 				return ast.WalkContinue, nil
 			}
-			if _, ok := snippets[language]; ok {
-				return ast.WalkContinue, nil
+			if _, ok := currentSection.Snippets[language]; ok {
+				// We have the same language appearing multiple times in a {{% examples
+				// %}} without an {{% example %}} to break them up. We are going to just
+				// pretend there was an {{% example %}}
+				pushExamples()
+			}
+			currentSection.Snippets[language] = snippet
+		case *ast.Text:
+			// We only want to change the title before we collect any snippets
+			title := strings.TrimSuffix(string(n.Text(source)), ":")
+			if currentSection.Title == "" && len(currentSection.Snippets) == 0 {
+				currentSection.Title = title
+			} else {
+				// Since we might find out we are done with the previous section only
+				// after we have consumed the next title, we store the title.
+				nextInferredTitle = title
 			}
-
-			snippet := schema.RenderDocsToString(source, n)
-			snippets[language] = snippet
 		}
 
 		return ast.WalkContinue, nil
 	})
 	contract.AssertNoError(err)
+	pushExamples()
 
 	if examplesShortcode != nil {
 		p := examplesShortcode.Parent()

pkg/codegen/docs/gen_test.go

@@ -526,3 +526,126 @@ func TestGeneratePackage(t *testing.T) {
 		TestCases:  test.PulumiPulumiSDKTests,
 	})
 }
+
+func TestDecomposeDocstring(t *testing.T) {
+	t.Parallel()
+	awsVpcDocs := "Provides a VPC resource.\n" +
+		"\n" +
+		"{{% examples %}}\n" +
+		"## Example Usage\n" +
+		"{{% example %}}\n" +
+		"\n" +
+		"Basic usage:\n" +
+		"\n" +
+		"```typescript\n" +
+		"Basic usage: typescript\n" +
+		"```\n" +
+		"```python\n" +
+		"Basic usage: python\n" +
+		"```\n" +
+		"```csharp\n" +
+		"Basic usage: csharp\n" +
+		"```\n" +
+		"```go\n" +
+		"Basic usage: go\n" +
+		"```\n" +
+		"```java\n" +
+		"Basic usage: java\n" +
+		"```\n" +
+		"```yaml\n" +
+		"Basic usage: yaml\n" +
+		"```\n" +
+		"\n" +
+		"Basic usage with tags:\n" +
+		"\n" +
+		"```typescript\n" +
+		"Basic usage with tags: typescript\n" +
+		"```\n" +
+		"```python\n" +
+		"Basic usage with tags: python\n" +
+		"```\n" +
+		"```csharp\n" +
+		"Basic usage with tags: csharp\n" +
+		"```\n" +
+		"```go\n" +
+		"Basic usage with tags: go\n" +
+		"```\n" +
+		"```java\n" +
+		"Basic usage with tags: java\n" +
+		"```\n" +
+		"```yaml\n" +
+		"Basic usage with tags: yaml\n" +
+		"```\n" +
+		"\n" +
+		"VPC with CIDR from AWS IPAM:\n" +
+		"\n" +
+		"```typescript\n" +
+		"VPC with CIDR from AWS IPAM: typescript\n" +
+		"```\n" +
+		"```python\n" +
+		"VPC with CIDR from AWS IPAM: python\n" +
+		"```\n" +
+		"```csharp\n" +
+		"VPC with CIDR from AWS IPAM: csharp\n" +
+		"```\n" +
+		"```java\n" +
+		"VPC with CIDR from AWS IPAM: java\n" +
+		"```\n" +
+		"```yaml\n" +
+		"VPC with CIDR from AWS IPAM: yaml\n" +
+		"```\n" +
+		"{{% /example %}}\n" +
+		"{{% /examples %}}\n" +
+		"\n" +
+		"## Import\n" +
+		"\n" +
+		"VPCs can be imported using the `vpc id`, e.g.,\n" +
+		"\n" +
+		"```sh\n" +
+		" $ pulumi import aws:ec2/vpc:Vpc test_vpc vpc-a01106c2\n" +
+		"```\n" +
+		"\n" +
+		" "
+	dctx := newDocGenContext()
+
+	info := dctx.decomposeDocstring(awsVpcDocs)
+	assert.Equal(t, docInfo{
+		description: "Provides a VPC resource.\n",
+		examples: []exampleSection{
+			{
+				Title: "Basic usage",
+				Snippets: map[string]string{
+					"csharp":     "```csharp\nBasic usage: csharp\n```\n",
+					"go":         "```go\nBasic usage: go\n```\n",
+					"java":       "```java\nBasic usage: java\n```\n",
+					"python":     "```python\nBasic usage: python\n```\n",
+					"typescript": "\n```typescript\nBasic usage: typescript\n```\n",
+					"yaml":       "```yaml\nBasic usage: yaml\n```\n",
+				},
+			},
+			{
+				Title: "Basic usage with tags",
+				Snippets: map[string]string{
+					"csharp":     "```csharp\nBasic usage with tags: csharp\n```\n",
+					"go":         "```go\nBasic usage with tags: go\n```\n",
+					"java":       "```java\nBasic usage with tags: java\n```\n",
+					"python":     "```python\nBasic usage with tags: python\n```\n",
+					"typescript": "\n```typescript\nBasic usage with tags: typescript\n```\n",
+					"yaml":       "```yaml\nBasic usage with tags: yaml\n```\n",
+				},
+			},
+			{
+				Title: "VPC with CIDR from AWS IPAM",
+				Snippets: map[string]string{
+					"csharp":     "```csharp\nVPC with CIDR from AWS IPAM: csharp\n```\n",
+					"go":         "Coming soon!",
+					"java":       "```java\nVPC with CIDR from AWS IPAM: java\n```\n",
+					"python":     "```python\nVPC with CIDR from AWS IPAM: python\n```\n",
+					"typescript": "\n```typescript\nVPC with CIDR from AWS IPAM: typescript\n```\n",
+					"yaml":       "```yaml\nVPC with CIDR from AWS IPAM: yaml\n```\n",
+				},
+			},
+		},
+		importDetails: "\n\nVPCs can be imported using the `vpc id`, e.g.,\n\n```sh\n $ pulumi import aws:ec2/vpc:Vpc test_vpc vpc-a01106c2\n```\n"},
+		info)
+}