Allow more flexible parsing when extracting examples from descriptions

changelog/pending/20221004--docs--more-flexible-doc-parsing.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: fix
+  scope: docs
+  description: Allow more flexible parsing when extracting examples from doc comments

pkg/codegen/docs/examples.go

@@ -55,10 +55,41 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo {
 
 	var examplesShortcode *schema.Shortcode
 	var exampleShortcode *schema.Shortcode
-	var title string
-	var snippets map[string]string
 	var examples []exampleSection
+	currentSection := exampleSection{
+		Snippets: map[string]string{},
+	}
+	var nextTitle string
+	var nextInferredTitle string
+	// Push any examples we have found. Since `pushExamples` is called between sections,
+	// it needs to behave correctly when no examples were found.
+	pushExamples := func() {
+		if len(currentSection.Snippets) > 0 {
+			for _, l := range dctx.snippetLanguages {
+				if _, ok := currentSection.Snippets[l]; !ok {
+					currentSection.Snippets[l] = defaultMissingExampleSnippetPlaceholder
+				}
+			}
+
+			examples = append(examples, currentSection)
+		}
+		if nextTitle == "" {
+			nextTitle = nextInferredTitle
+		}
+		currentSection = exampleSection{
+			Snippets: map[string]string{},
+			Title:    nextTitle,
+		}
+		nextTitle = ""
+		nextInferredTitle = ""
+	}
 	err := ast.Walk(parsed, func(n ast.Node, enter bool) (ast.WalkStatus, error) {
+		// ast.Walk visits each node twice. The first time descending and the second time
+		// ascending. We only want to view the nodes while descending, so we skip when
+		// `enter` is false.
+		if !enter {
+			return ast.WalkContinue, nil
+		}
 		if shortcode, ok := n.(*schema.Shortcode); ok {
 			name := string(shortcode.Name)
 			switch name {
@@ -68,49 +99,62 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo {
 				}
 			case schema.ExampleShortcode:
 				if exampleShortcode == nil {
-					exampleShortcode, title, snippets = shortcode, "", map[string]string{}
+					exampleShortcode = shortcode
+					currentSection.Title, currentSection.Snippets = "", map[string]string{}
 				} else if !enter && shortcode == exampleShortcode {
-					for _, l := range dctx.snippetLanguages {
-						if _, ok := snippets[l]; !ok {
-							snippets[l] = defaultMissingExampleSnippetPlaceholder
-						}
-					}
-
-					examples = append(examples, exampleSection{
-						Title:    title,
-						Snippets: snippets,
-					})
-
+					pushExamples()
 					exampleShortcode = nil
 				}
 			}
 			return ast.WalkContinue, nil
 		}
+
+		// We check to make sure we are in an examples section.
 		if exampleShortcode == nil {
 			return ast.WalkContinue, nil
 		}
 
 		switch n := n.(type) {
 		case *ast.Heading:
-			if n.Level == 3 && title == "" {
-				title = strings.TrimSpace(schema.RenderDocsToString(source, n))
+			if n.Level == 3 {
+				title := strings.TrimSpace(schema.RenderDocsToString(source, n))
+				if currentSection.Title == "" && len(currentSection.Snippets) == 0 {
+					currentSection.Title = title
+				} else {
+					nextTitle = title
+				}
 			}
+			return ast.WalkSkipChildren, nil
+
 		case *ast.FencedCodeBlock:
 			language := string(n.Language(source))
-			if !languages.Has(language) {
+			snippet := schema.RenderDocsToString(source, n)
+			if !languages.Has(language) || len(snippet) == 0 {
 				return ast.WalkContinue, nil
 			}
-			if _, ok := snippets[language]; ok {
-				return ast.WalkContinue, nil
+			if _, ok := currentSection.Snippets[language]; ok {
+				// We have the same language appearing multiple times in a {{% examples
+				// %}} without an {{% example %}} to break them up. We are going to just
+				// pretend there was an {{% example %}}
+				pushExamples()
+			}
+			currentSection.Snippets[language] = snippet
+		case *ast.Text:
+			// We only want to change the title before we collect any snippets
+			title := strings.TrimSuffix(string(n.Text(source)), ":")
+			if currentSection.Title == "" && len(currentSection.Snippets) == 0 {
+				currentSection.Title = title
+			} else {
+				// Since we might find out we are done with the previous section only
+				// after we have consumed the next title, we store the title.
+				nextInferredTitle = title
 			}
-
-			snippet := schema.RenderDocsToString(source, n)
-			snippets[language] = snippet
 		}
 
 		return ast.WalkContinue, nil
 	})
 	contract.AssertNoError(err)
+	pushExamples()
 
 	if examplesShortcode != nil {
 		p := examplesShortcode.Parent()

pkg/codegen/docs/gen_test.go

@@ -526,3 +526,126 @@ func TestGeneratePackage(t *testing.T) {
 		TestCases:  test.PulumiPulumiSDKTests,
 	})
 }
+
+func TestDecomposeDocstring(t *testing.T) {
+	t.Parallel()
+	awsVpcDocs := "Provides a VPC resource.\n" +
+		"\n" +
+		"{{% examples %}}\n" +
+		"## Example Usage\n" +
+		"{{% example %}}\n" +
+		"\n" +
+		"Basic usage:\n" +
+		"\n" +
+		"```typescript\n" +
+		"Basic usage: typescript\n" +
+		"```\n" +
+		"```python\n" +
+		"Basic usage: python\n" +
+		"```\n" +
+		"```csharp\n" +
+		"Basic usage: csharp\n" +
+		"```\n" +
+		"```go\n" +
+		"Basic usage: go\n" +
+		"```\n" +
+		"```java\n" +
+		"Basic usage: java\n" +
+		"```\n" +
+		"```yaml\n" +
+		"Basic usage: yaml\n" +
+		"```\n" +
+		"\n" +
+		"Basic usage with tags:\n" +
+		"\n" +
+		"```typescript\n" +
+		"Basic usage with tags: typescript\n" +
+		"```\n" +
+		"```python\n" +
+		"Basic usage with tags: python\n" +
+		"```\n" +
+		"```csharp\n" +
+		"Basic usage with tags: csharp\n" +
+		"```\n" +
+		"```go\n" +
+		"Basic usage with tags: go\n" +
+		"```\n" +
+		"```java\n" +
+		"Basic usage with tags: java\n" +
+		"```\n" +
+		"```yaml\n" +
+		"Basic usage with tags: yaml\n" +
+		"```\n" +
+		"\n" +
+		"VPC with CIDR from AWS IPAM:\n" +
+		"\n" +
+		"```typescript\n" +
+		"VPC with CIDR from AWS IPAM: typescript\n" +
+		"```\n" +
+		"```python\n" +
+		"VPC with CIDR from AWS IPAM: python\n" +
+		"```\n" +
+		"```csharp\n" +
+		"VPC with CIDR from AWS IPAM: csharp\n" +
+		"```\n" +
+		"```java\n" +
+		"VPC with CIDR from AWS IPAM: java\n" +
+		"```\n" +
+		"```yaml\n" +
+		"VPC with CIDR from AWS IPAM: yaml\n" +
+		"```\n" +
+		"{{% /example %}}\n" +
+		"{{% /examples %}}\n" +
+		"\n" +
+		"## Import\n" +
+		"\n" +
+		"VPCs can be imported using the `vpc id`, e.g.,\n" +
+		"\n" +
+		"```sh\n" +
+		" $ pulumi import aws:ec2/vpc:Vpc test_vpc vpc-a01106c2\n" +
+		"```\n" +
+		"\n" +
+		" "
+	dctx := newDocGenContext()
+
+	info := dctx.decomposeDocstring(awsVpcDocs)
+	assert.Equal(t, docInfo{
+		description: "Provides a VPC resource.\n",
+		examples: []exampleSection{
+			{
+				Title: "Basic usage",
+				Snippets: map[string]string{
+					"csharp":     "```csharp\nBasic usage: csharp\n```\n",
+					"go":         "```go\nBasic usage: go\n```\n",
+					"java":       "```java\nBasic usage: java\n```\n",
+					"python":     "```python\nBasic usage: python\n```\n",
+					"typescript": "\n```typescript\nBasic usage: typescript\n```\n",
+					"yaml":       "```yaml\nBasic usage: yaml\n```\n",
+				},
+			},
+			{
+				Title: "Basic usage with tags",
+				Snippets: map[string]string{
+					"csharp":     "```csharp\nBasic usage with tags: csharp\n```\n",
+					"go":         "```go\nBasic usage with tags: go\n```\n",
+					"java":       "```java\nBasic usage with tags: java\n```\n",
+					"python":     "```python\nBasic usage with tags: python\n```\n",
+					"typescript": "\n```typescript\nBasic usage with tags: typescript\n```\n",
+					"yaml":       "```yaml\nBasic usage with tags: yaml\n```\n",
+				},
+			},
+			{
+				Title: "VPC with CIDR from AWS IPAM",
+				Snippets: map[string]string{
+					"csharp":     "```csharp\nVPC with CIDR from AWS IPAM: csharp\n```\n",
+					"go":         "Coming soon!",
+					"java":       "```java\nVPC with CIDR from AWS IPAM: java\n```\n",
+					"python":     "```python\nVPC with CIDR from AWS IPAM: python\n```\n",
+					"typescript": "\n```typescript\nVPC with CIDR from AWS IPAM: typescript\n```\n",
+					"yaml":       "```yaml\nVPC with CIDR from AWS IPAM: yaml\n```\n",
+				},
+			},
+		},
+		importDetails: "\n\nVPCs can be imported using the `vpc id`, e.g.,\n\n```sh\n $ pulumi import aws:ec2/vpc:Vpc test_vpc vpc-a01106c2\n```\n"},
+		info)
+}