From 1d8762c4d1522a507dc45465e1338a5e0e226e76 Mon Sep 17 00:00:00 2001 From: Ian Wahbe Date: Mon, 3 Oct 2022 18:06:38 -0700 Subject: [PATCH 1/5] Loosen our ast.Walk function --- pkg/codegen/docs/examples.go | 74 ++++++++++++++++----- pkg/codegen/docs/gen_test.go | 125 +++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 18 deletions(-) diff --git a/pkg/codegen/docs/examples.go b/pkg/codegen/docs/examples.go index a0d29c81366b..7ffaf8269848 100644 --- a/pkg/codegen/docs/examples.go +++ b/pkg/codegen/docs/examples.go @@ -55,10 +55,30 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { var examplesShortcode *schema.Shortcode var exampleShortcode *schema.Shortcode - var title string - var snippets map[string]string var examples []exampleSection + currentSection := exampleSection{ + Snippets: map[string]string{}, + } + var nextTitle string + // Push any examples we have found. Since `pushExamples` is called between sections, + // it needs to behave correctly when no examples were found. + pushExamples := func() { + if len(currentSection.Snippets) > 0 { + examples = append(examples, currentSection) + } + currentSection = exampleSection{ + Snippets: map[string]string{}, + Title: nextTitle, + } + nextTitle = "" + } err := ast.Walk(parsed, func(n ast.Node, enter bool) (ast.WalkStatus, error) { + // ast.Walk visits each node twice. The first time descending and the second time + // ascending. We only want to view the nodes while descending, so we skip when + // `enter` is false. + if !enter { + return ast.WalkContinue, nil + } if shortcode, ok := n.(*schema.Shortcode); ok { name := string(shortcode.Name) switch name { @@ -68,49 +88,67 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { } case schema.ExampleShortcode: if exampleShortcode == nil { - exampleShortcode, title, snippets = shortcode, "", map[string]string{} + exampleShortcode = shortcode + currentSection.Title, currentSection.Snippets = "", map[string]string{} } else if !enter && shortcode == exampleShortcode { for _, l := range dctx.snippetLanguages { - if _, ok := snippets[l]; !ok { - snippets[l] = defaultMissingExampleSnippetPlaceholder + if _, ok := currentSection.Snippets[l]; !ok { + currentSection.Snippets[l] = defaultMissingExampleSnippetPlaceholder } } - examples = append(examples, exampleSection{ - Title: title, - Snippets: snippets, - }) - + pushExamples() exampleShortcode = nil } } return ast.WalkContinue, nil } + + // We check to make sure we are in an examples section. if exampleShortcode == nil { return ast.WalkContinue, nil } switch n := n.(type) { case *ast.Heading: - if n.Level == 3 && title == "" { - title = strings.TrimSpace(schema.RenderDocsToString(source, n)) + if n.Level == 3 { + title := strings.TrimSpace(schema.RenderDocsToString(source, n)) + if currentSection.Title == "" && len(currentSection.Snippets) == 0 { + currentSection.Title = title + } else { + nextTitle = title + } } + case *ast.FencedCodeBlock: language := string(n.Language(source)) - if !languages.Has(language) { + snippet := schema.RenderDocsToString(source, n) + if !languages.Has(language) || len(snippet) == 0 { return ast.WalkContinue, nil } - if _, ok := snippets[language]; ok { - return ast.WalkContinue, nil + if _, ok := currentSection.Snippets[language]; ok { + // We have the same language appearing multiple times in a {{% examples + // %}} without an {{% example %}} to break them up. We are going to just + // pretend there was an {{% example %}} + pushExamples() + } + currentSection.Snippets[language] = snippet + case *ast.Text: + // We only want to change the title before we collect any snippets + title := strings.TrimSuffix(string(n.Text([]byte(source))), ":") + if currentSection.Title == "" && len(currentSection.Snippets) == 0 { + currentSection.Title = title + } else { + // Since we might find out we are done with the previous section only + // after we have consumed the next title, we store the title. + nextTitle = title } - - snippet := schema.RenderDocsToString(source, n) - snippets[language] = snippet } return ast.WalkContinue, nil }) contract.AssertNoError(err) + pushExamples() if examplesShortcode != nil { p := examplesShortcode.Parent() diff --git a/pkg/codegen/docs/gen_test.go b/pkg/codegen/docs/gen_test.go index 67d8ac537ada..f03905901d2f 100644 --- a/pkg/codegen/docs/gen_test.go +++ b/pkg/codegen/docs/gen_test.go @@ -526,3 +526,128 @@ func TestGeneratePackage(t *testing.T) { TestCases: test.PulumiPulumiSDKTests, }) } + +func TestDecomposeDocstring(t *testing.T) { + awsVpcDocs := "Provides a VPC resource.\n" + + "\n" + + "{{% examples %}}\n" + + "## Example Usage\n" + + "{{% example %}}\n" + + "\n" + + "Basic usage:\n" + + "\n" + + "```typescript\n" + + "Basic usage: typescript\n" + + "```\n" + + "```python\n" + + "Basic usage: python\n" + + "```\n" + + "```csharp\n" + + "Basic usage: csharp\n" + + "```\n" + + "```go\n" + + "Basic usage: go\n" + + "```\n" + + "```java\n" + + "Basic usage: java\n" + + "```\n" + + "```yaml\n" + + "Basic usage: yaml\n" + + "```\n" + + "\n" + + "Basic usage with tags:\n" + + "\n" + + "```typescript\n" + + "Basic usage with tags: typescript\n" + + "```\n" + + "```python\n" + + "Basic usage with tags: python\n" + + "```\n" + + "```csharp\n" + + "Basic usage with tags: csharp\n" + + "```\n" + + "```go\n" + + "Basic usage with tags: go\n" + + "```\n" + + "```java\n" + + "Basic usage with tags: java\n" + + "```\n" + + "```yaml\n" + + "Basic usage with tags: yaml\n" + + "```\n" + + "\n" + + "VPC with CIDR from AWS IPAM:\n" + + "\n" + + "```typescript\n" + + "VPC with CIDR from AWS IPAM: typescript\n" + + "```\n" + + "```python\n" + + "VPC with CIDR from AWS IPAM: python\n" + + "```\n" + + "```csharp\n" + + "VPC with CIDR from AWS IPAM: csharp\n" + + "```\n" + + "```go\n" + + "VPC with CIDR from AWS IPAM: go\n" + + "```\n" + + "```java\n" + + "VPC with CIDR from AWS IPAM: java\n" + + "```\n" + + "```yaml\n" + + "VPC with CIDR from AWS IPAM: yaml\n" + + "```\n" + + "{{% /example %}}\n" + + "{{% /examples %}}\n" + + "\n" + + "## Import\n" + + "\n" + + "VPCs can be imported using the `vpc id`, e.g.,\n" + + "\n" + + "```sh\n" + + " $ pulumi import aws:ec2/vpc:Vpc test_vpc vpc-a01106c2\n" + + "```\n" + + "\n" + + " " + dctx := newDocGenContext() + + info := dctx.decomposeDocstring(awsVpcDocs) + assert.Equal(t, docInfo{ + description: "Provides a VPC resource.\n", + examples: []exampleSection{ + { + Title: "Basic usage", + Snippets: map[string]string{ + "csharp": "```csharp\nBasic usage: csharp\n```\n", + "go": "```go\nBasic usage: go\n```\n", + "java": "```java\nBasic usage: java\n```\n", + "python": "```python\nBasic usage: python\n```\n", + "typescript": "\n```typescript\nBasic usage: typescript\n```\n", + "yaml": "```yaml\nBasic usage: yaml\n```\n", + }, + }, + { + Title: "Basic usage with tags", + Snippets: map[string]string{ + "csharp": "```csharp\nBasic usage with tags: csharp\n```\n", + "go": "```go\nBasic usage with tags: go\n```\n", + "java": "```java\nBasic usage with tags: java\n```\n", + "python": "```python\nBasic usage with tags: python\n```\n", + "typescript": "\n```typescript\nBasic usage with tags: typescript\n```\n", + "yaml": "```yaml\nBasic usage with tags: yaml\n```\n", + }, + }, + { + Title: "VPC with CIDR from AWS IPAM", + Snippets: map[string]string{ + "csharp": "```csharp\nVPC with CIDR from AWS IPAM: csharp\n```\n", + "go": "```go\nVPC with CIDR from AWS IPAM: go\n```\n", + "java": "```java\nVPC with CIDR from AWS IPAM: java\n```\n", + "python": "```python\nVPC with CIDR from AWS IPAM: python\n```\n", + "typescript": "\n```typescript\nVPC with CIDR from AWS IPAM: typescript\n```\n", + "yaml": "```yaml\nVPC with CIDR from AWS IPAM: yaml\n```\n", + }, + }, + }, + importDetails: "\n\nVPCs can be imported using the `vpc id`, e.g.,\n\n```sh\n $ pulumi import aws:ec2/vpc:Vpc test_vpc vpc-a01106c2\n```\n"}, + info) +} From 044e781b3437fdc2d69cf27fd9ee04230c4ae0cc Mon Sep 17 00:00:00 2001 From: Ian Wahbe Date: Mon, 3 Oct 2022 18:07:48 -0700 Subject: [PATCH 2/5] Satisfy linter --- pkg/codegen/docs/examples.go | 2 +- pkg/codegen/docs/gen_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/codegen/docs/examples.go b/pkg/codegen/docs/examples.go index 7ffaf8269848..1b178d82cbe8 100644 --- a/pkg/codegen/docs/examples.go +++ b/pkg/codegen/docs/examples.go @@ -135,7 +135,7 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { currentSection.Snippets[language] = snippet case *ast.Text: // We only want to change the title before we collect any snippets - title := strings.TrimSuffix(string(n.Text([]byte(source))), ":") + title := strings.TrimSuffix(string(n.Text(source)), ":") if currentSection.Title == "" && len(currentSection.Snippets) == 0 { currentSection.Title = title } else { diff --git a/pkg/codegen/docs/gen_test.go b/pkg/codegen/docs/gen_test.go index f03905901d2f..9df8f9fd08ec 100644 --- a/pkg/codegen/docs/gen_test.go +++ b/pkg/codegen/docs/gen_test.go @@ -528,6 +528,7 @@ func TestGeneratePackage(t *testing.T) { } func TestDecomposeDocstring(t *testing.T) { + t.Parallel() awsVpcDocs := "Provides a VPC resource.\n" + "\n" + "{{% examples %}}\n" + From 4f8f7c2ac51acf6316a41802e65e5158d408e257 Mon Sep 17 00:00:00 2001 From: Ian Wahbe Date: Mon, 3 Oct 2022 18:10:00 -0700 Subject: [PATCH 3/5] CL --- .../pending/20221004--docs--more-flexible-doc-parsing.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/pending/20221004--docs--more-flexible-doc-parsing.yaml diff --git a/changelog/pending/20221004--docs--more-flexible-doc-parsing.yaml b/changelog/pending/20221004--docs--more-flexible-doc-parsing.yaml new file mode 100644 index 000000000000..ebca6d1310a6 --- /dev/null +++ b/changelog/pending/20221004--docs--more-flexible-doc-parsing.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: docs + description: Allow more flexible parsing when extracting examples from doc comments From 0097c7247f1a3564778b6cc98854bc4f289a3029 Mon Sep 17 00:00:00 2001 From: Ian Wahbe Date: Tue, 4 Oct 2022 10:24:06 -0700 Subject: [PATCH 4/5] Support missing snippets correctly --- pkg/codegen/docs/examples.go | 12 ++++++------ pkg/codegen/docs/gen_test.go | 5 +---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/pkg/codegen/docs/examples.go b/pkg/codegen/docs/examples.go index 1b178d82cbe8..2dc8f9b35426 100644 --- a/pkg/codegen/docs/examples.go +++ b/pkg/codegen/docs/examples.go @@ -64,6 +64,12 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { // it needs to behave correctly when no examples were found. pushExamples := func() { if len(currentSection.Snippets) > 0 { + for _, l := range dctx.snippetLanguages { + if _, ok := currentSection.Snippets[l]; !ok { + currentSection.Snippets[l] = defaultMissingExampleSnippetPlaceholder + } + } + examples = append(examples, currentSection) } currentSection = exampleSection{ @@ -91,12 +97,6 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { exampleShortcode = shortcode currentSection.Title, currentSection.Snippets = "", map[string]string{} } else if !enter && shortcode == exampleShortcode { - for _, l := range dctx.snippetLanguages { - if _, ok := currentSection.Snippets[l]; !ok { - currentSection.Snippets[l] = defaultMissingExampleSnippetPlaceholder - } - } - pushExamples() exampleShortcode = nil } diff --git a/pkg/codegen/docs/gen_test.go b/pkg/codegen/docs/gen_test.go index 9df8f9fd08ec..dd00bf2d1ce4 100644 --- a/pkg/codegen/docs/gen_test.go +++ b/pkg/codegen/docs/gen_test.go @@ -588,9 +588,6 @@ func TestDecomposeDocstring(t *testing.T) { "```csharp\n" + "VPC with CIDR from AWS IPAM: csharp\n" + "```\n" + - "```go\n" + - "VPC with CIDR from AWS IPAM: go\n" + - "```\n" + "```java\n" + "VPC with CIDR from AWS IPAM: java\n" + "```\n" + @@ -641,7 +638,7 @@ func TestDecomposeDocstring(t *testing.T) { Title: "VPC with CIDR from AWS IPAM", Snippets: map[string]string{ "csharp": "```csharp\nVPC with CIDR from AWS IPAM: csharp\n```\n", - "go": "```go\nVPC with CIDR from AWS IPAM: go\n```\n", + "go": "Coming soon!", "java": "```java\nVPC with CIDR from AWS IPAM: java\n```\n", "python": "```python\nVPC with CIDR from AWS IPAM: python\n```\n", "typescript": "\n```typescript\nVPC with CIDR from AWS IPAM: typescript\n```\n", From 49dd8db47a9aacf3c662277e5199f58027f91650 Mon Sep 17 00:00:00 2001 From: Ian Wahbe Date: Tue, 4 Oct 2022 11:21:57 -0700 Subject: [PATCH 5/5] Special case headings --- pkg/codegen/docs/examples.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/codegen/docs/examples.go b/pkg/codegen/docs/examples.go index 2dc8f9b35426..01f0af73054a 100644 --- a/pkg/codegen/docs/examples.go +++ b/pkg/codegen/docs/examples.go @@ -60,6 +60,7 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { Snippets: map[string]string{}, } var nextTitle string + var nextInferredTitle string // Push any examples we have found. Since `pushExamples` is called between sections, // it needs to behave correctly when no examples were found. pushExamples := func() { @@ -72,11 +73,15 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { examples = append(examples, currentSection) } + if nextTitle == "" { + nextTitle = nextInferredTitle + } currentSection = exampleSection{ Snippets: map[string]string{}, Title: nextTitle, } nextTitle = "" + nextInferredTitle = "" } err := ast.Walk(parsed, func(n ast.Node, enter bool) (ast.WalkStatus, error) { // ast.Walk visits each node twice. The first time descending and the second time @@ -119,6 +124,7 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { nextTitle = title } } + return ast.WalkSkipChildren, nil case *ast.FencedCodeBlock: language := string(n.Language(source)) @@ -141,7 +147,7 @@ func (dctx *docGenContext) decomposeDocstring(docstring string) docInfo { } else { // Since we might find out we are done with the previous section only // after we have consumed the next title, we store the title. - nextTitle = title + nextInferredTitle = title } }