diff --git a/changelog/pending/20221004--docs--more-flexible-doc-parsing.yaml b/changelog/pending/20221004--docs--more-flexible-doc-parsing.yaml new file mode 100644 index 000000000000..ebca6d1310a6 --- /dev/null +++ b/changelog/pending/20221004--docs--more-flexible-doc-parsing.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: docs + description: Allow more flexible parsing when extracting examples from doc comments diff --git a/pkg/codegen/docs/examples.go b/pkg/codegen/docs/examples.go index a0d29c81366b..01f0af73054a 100644 --- a/pkg/codegen/docs/examples.go +++ b/pkg/codegen/docs/examples.go @@ -55,10 +55,41 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { var examplesShortcode *schema.Shortcode var exampleShortcode *schema.Shortcode - var title string - var snippets map[string]string var examples []exampleSection + currentSection := exampleSection{ + Snippets: map[string]string{}, + } + var nextTitle string + var nextInferredTitle string + // Push any examples we have found. Since `pushExamples` is called between sections, + // it needs to behave correctly when no examples were found. + pushExamples := func() { + if len(currentSection.Snippets) > 0 { + for _, l := range dctx.snippetLanguages { + if _, ok := currentSection.Snippets[l]; !ok { + currentSection.Snippets[l] = defaultMissingExampleSnippetPlaceholder + } + } + + examples = append(examples, currentSection) + } + if nextTitle == "" { + nextTitle = nextInferredTitle + } + currentSection = exampleSection{ + Snippets: map[string]string{}, + Title: nextTitle, + } + nextTitle = "" + nextInferredTitle = "" + } err := ast.Walk(parsed, func(n ast.Node, enter bool) (ast.WalkStatus, error) { + // ast.Walk visits each node twice. The first time descending and the second time + // ascending. We only want to view the nodes while descending, so we skip when + // `enter` is false. + if !enter { + return ast.WalkContinue, nil + } if shortcode, ok := n.(*schema.Shortcode); ok { name := string(shortcode.Name) switch name { @@ -68,49 +99,62 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { } case schema.ExampleShortcode: if exampleShortcode == nil { - exampleShortcode, title, snippets = shortcode, "", map[string]string{} + exampleShortcode = shortcode + currentSection.Title, currentSection.Snippets = "", map[string]string{} } else if !enter && shortcode == exampleShortcode { - for _, l := range dctx.snippetLanguages { - if _, ok := snippets[l]; !ok { - snippets[l] = defaultMissingExampleSnippetPlaceholder - } - } - - examples = append(examples, exampleSection{ - Title: title, - Snippets: snippets, - }) - + pushExamples() exampleShortcode = nil } } return ast.WalkContinue, nil } + + // We check to make sure we are in an examples section. if exampleShortcode == nil { return ast.WalkContinue, nil } switch n := n.(type) { case *ast.Heading: - if n.Level == 3 && title == "" { - title = strings.TrimSpace(schema.RenderDocsToString(source, n)) + if n.Level == 3 { + title := strings.TrimSpace(schema.RenderDocsToString(source, n)) + if currentSection.Title == "" && len(currentSection.Snippets) == 0 { + currentSection.Title = title + } else { + nextTitle = title + } } + return ast.WalkSkipChildren, nil + case *ast.FencedCodeBlock: language := string(n.Language(source)) - if !languages.Has(language) { + snippet := schema.RenderDocsToString(source, n) + if !languages.Has(language) || len(snippet) == 0 { return ast.WalkContinue, nil } - if _, ok := snippets[language]; ok { - return ast.WalkContinue, nil + if _, ok := currentSection.Snippets[language]; ok { + // We have the same language appearing multiple times in a {{% examples + // %}} without an {{% example %}} to break them up. We are going to just + // pretend there was an {{% example %}} + pushExamples() + } + currentSection.Snippets[language] = snippet + case *ast.Text: + // We only want to change the title before we collect any snippets + title := strings.TrimSuffix(string(n.Text(source)), ":") + if currentSection.Title == "" && len(currentSection.Snippets) == 0 { + currentSection.Title = title + } else { + // Since we might find out we are done with the previous section only + // after we have consumed the next title, we store the title. + nextInferredTitle = title } - - snippet := schema.RenderDocsToString(source, n) - snippets[language] = snippet } return ast.WalkContinue, nil }) contract.AssertNoError(err) + pushExamples() if examplesShortcode != nil { p := examplesShortcode.Parent() diff --git a/pkg/codegen/docs/gen_test.go b/pkg/codegen/docs/gen_test.go index 67d8ac537ada..dd00bf2d1ce4 100644 --- a/pkg/codegen/docs/gen_test.go +++ b/pkg/codegen/docs/gen_test.go @@ -526,3 +526,126 @@ func TestGeneratePackage(t *testing.T) { TestCases: test.PulumiPulumiSDKTests, }) } + +func TestDecomposeDocstring(t *testing.T) { + t.Parallel() + awsVpcDocs := "Provides a VPC resource.\n" + + "\n" + + "{{% examples %}}\n" + + "## Example Usage\n" + + "{{% example %}}\n" + + "\n" + + "Basic usage:\n" + + "\n" + + "```typescript\n" + + "Basic usage: typescript\n" + + "```\n" + + "```python\n" + + "Basic usage: python\n" + + "```\n" + + "```csharp\n" + + "Basic usage: csharp\n" + + "```\n" + + "```go\n" + + "Basic usage: go\n" + + "```\n" + + "```java\n" + + "Basic usage: java\n" + + "```\n" + + "```yaml\n" + + "Basic usage: yaml\n" + + "```\n" + + "\n" + + "Basic usage with tags:\n" + + "\n" + + "```typescript\n" + + "Basic usage with tags: typescript\n" + + "```\n" + + "```python\n" + + "Basic usage with tags: python\n" + + "```\n" + + "```csharp\n" + + "Basic usage with tags: csharp\n" + + "```\n" + + "```go\n" + + "Basic usage with tags: go\n" + + "```\n" + + "```java\n" + + "Basic usage with tags: java\n" + + "```\n" + + "```yaml\n" + + "Basic usage with tags: yaml\n" + + "```\n" + + "\n" + + "VPC with CIDR from AWS IPAM:\n" + + "\n" + + "```typescript\n" + + "VPC with CIDR from AWS IPAM: typescript\n" + + "```\n" + + "```python\n" + + "VPC with CIDR from AWS IPAM: python\n" + + "```\n" + + "```csharp\n" + + "VPC with CIDR from AWS IPAM: csharp\n" + + "```\n" + + "```java\n" + + "VPC with CIDR from AWS IPAM: java\n" + + "```\n" + + "```yaml\n" + + "VPC with CIDR from AWS IPAM: yaml\n" + + "```\n" + + "{{% /example %}}\n" + + "{{% /examples %}}\n" + + "\n" + + "## Import\n" + + "\n" + + "VPCs can be imported using the `vpc id`, e.g.,\n" + + "\n" + + "```sh\n" + + " $ pulumi import aws:ec2/vpc:Vpc test_vpc vpc-a01106c2\n" + + "```\n" + + "\n" + + " " + dctx := newDocGenContext() + + info := dctx.decomposeDocstring(awsVpcDocs) + assert.Equal(t, docInfo{ + description: "Provides a VPC resource.\n", + examples: []exampleSection{ + { + Title: "Basic usage", + Snippets: map[string]string{ + "csharp": "```csharp\nBasic usage: csharp\n```\n", + "go": "```go\nBasic usage: go\n```\n", + "java": "```java\nBasic usage: java\n```\n", + "python": "```python\nBasic usage: python\n```\n", + "typescript": "\n```typescript\nBasic usage: typescript\n```\n", + "yaml": "```yaml\nBasic usage: yaml\n```\n", + }, + }, + { + Title: "Basic usage with tags", + Snippets: map[string]string{ + "csharp": "```csharp\nBasic usage with tags: csharp\n```\n", + "go": "```go\nBasic usage with tags: go\n```\n", + "java": "```java\nBasic usage with tags: java\n```\n", + "python": "```python\nBasic usage with tags: python\n```\n", + "typescript": "\n```typescript\nBasic usage with tags: typescript\n```\n", + "yaml": "```yaml\nBasic usage with tags: yaml\n```\n", + }, + }, + { + Title: "VPC with CIDR from AWS IPAM", + Snippets: map[string]string{ + "csharp": "```csharp\nVPC with CIDR from AWS IPAM: csharp\n```\n", + "go": "Coming soon!", + "java": "```java\nVPC with CIDR from AWS IPAM: java\n```\n", + "python": "```python\nVPC with CIDR from AWS IPAM: python\n```\n", + "typescript": "\n```typescript\nVPC with CIDR from AWS IPAM: typescript\n```\n", + "yaml": "```yaml\nVPC with CIDR from AWS IPAM: yaml\n```\n", + }, + }, + }, + importDetails: "\n\nVPCs can be imported using the `vpc id`, e.g.,\n\n```sh\n $ pulumi import aws:ec2/vpc:Vpc test_vpc vpc-a01106c2\n```\n"}, + info) +}