Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
11032: Add standardized program-gen reporting helpers r=iwahbe a=iwahbe This PR introduces a shared format for reporting codegen problems. <!--- Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation. --> # Description <!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. --> Fixes #10988 ## Checklist <!--- Please provide details if the checkbox below is to be left unchecked. --> - [ ] I have added tests that prove my fix is effective or that my feature works <!--- User-facing changes require a CHANGELOG entry. --> - [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change <!-- If the change(s) in this PR is a modification of an existing call to the Pulumi Service, then the service should honor older versions of the CLI where this change would not exist. You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add it to the service. --> - [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. --> Co-authored-by: Ian Wahbe <ian@wahbe.com>
- Loading branch information
Showing
4 changed files
with
352 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
changes: | ||
- type: feat | ||
scope: programgen | ||
description: Add error reporting infrastructure |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,244 @@ | ||
// Copyright 2016-2022, Pulumi Corporation. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package report | ||
|
||
import ( | ||
"encoding/json" | ||
"fmt" | ||
"io" | ||
"io/ioutil" | ||
"os" | ||
"path/filepath" | ||
"sync" | ||
"time" | ||
|
||
"github.com/hashicorp/hcl/v2" | ||
|
||
"github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax" | ||
hcl2 "github.com/pulumi/pulumi/pkg/v3/codegen/pcl" | ||
"github.com/pulumi/pulumi/pkg/v3/version" | ||
) | ||
|
||
const ExportTargetDir = "PULUMI_CODEGEN_REPORT_DIR" | ||
|
||
type GenerateProgramFn func(*hcl2.Program) (map[string][]byte, hcl.Diagnostics, error) | ||
|
||
type Reporter interface { | ||
io.Closer | ||
// Report a call to GenerateProgram. | ||
Report(title, language string, files []*syntax.File, diags hcl.Diagnostics, err error) | ||
Summary() Summary | ||
} | ||
|
||
func New(name, version string) Reporter { | ||
return &reporter{ | ||
data: Summary{ | ||
Name: name, | ||
Version: version, | ||
}, | ||
} | ||
} | ||
|
||
type Summary struct { | ||
Stats | ||
Name string `json:"name"` | ||
Version string `json:"version"` | ||
ReportVersion string `json:"reportVersion"` | ||
Languages map[string]*Language `json:"languages"` | ||
} | ||
|
||
type Stats struct { | ||
NumConversions int | ||
Successes int | ||
} | ||
|
||
type Language struct { | ||
Stats | ||
|
||
// A mapping from Error:(title:occurrences) | ||
Warnings map[string]map[string]int `json:"warning,omitempty"` | ||
Errors map[string]map[string]int `json:"errors,omitempty"` | ||
|
||
// A mapping from between titles and Go errors (as opposed to diag errors) | ||
GoErrors map[string]string `json:"goerrors,omitempty"` | ||
|
||
// A mapping from title:files | ||
Files map[string][]File `json:"files,omitempty"` | ||
} | ||
|
||
type File struct { | ||
Name string `json:"name,omitempty"` | ||
Body string `json:"body,omitempty"` | ||
} | ||
|
||
type reporter struct { | ||
data Summary | ||
reported bool | ||
m sync.Mutex | ||
} | ||
|
||
func (s *Stats) update(succeed bool) { | ||
s.NumConversions++ | ||
if succeed { | ||
s.Successes++ | ||
} | ||
} | ||
|
||
func (r *reporter) getLanguage(lang string) *Language { | ||
if r.data.Languages == nil { | ||
r.data.Languages = map[string]*Language{} | ||
} | ||
l, ok := r.data.Languages[lang] | ||
if !ok { | ||
l = new(Language) | ||
r.data.Languages[lang] = l | ||
} | ||
return l | ||
} | ||
|
||
func WrapGen(reporter Reporter, title, language string, files []*syntax.File, f GenerateProgramFn) GenerateProgramFn { | ||
return func(p *hcl2.Program) (m map[string][]byte, diags hcl.Diagnostics, err error) { | ||
defer func() { | ||
reporter.Report(title, language, files, diags, err) | ||
}() | ||
m, diags, err = f(p) | ||
return m, diags, err | ||
} | ||
} | ||
|
||
func (r *reporter) Report(title, language string, files []*syntax.File, diags hcl.Diagnostics, err error) { | ||
r.m.Lock() | ||
defer r.m.Unlock() | ||
if panicErr := recover(); panicErr != nil { | ||
if panic, ok := panicErr.(error); ok { | ||
err = fmt.Errorf("panic: %w", panic) | ||
} else { | ||
err = fmt.Errorf("panic: %v", panicErr) | ||
} | ||
} | ||
failed := diags.HasErrors() || err != nil | ||
r.data.Stats.update(!failed) | ||
lang := r.getLanguage(language) | ||
lang.Stats.update(!failed) | ||
|
||
if failed { | ||
var txts []File | ||
for _, file := range files { | ||
txts = append(txts, File{ | ||
Name: file.Name, | ||
Body: string(file.Bytes), | ||
}) | ||
} | ||
if lang.Files == nil { | ||
lang.Files = map[string][]File{} | ||
} | ||
lang.Files[title] = txts | ||
} | ||
if err != nil { | ||
err := fmt.Sprintf("error: %v", err) | ||
if lang.GoErrors == nil { | ||
lang.GoErrors = map[string]string{} | ||
} | ||
lang.GoErrors[title] = err | ||
} | ||
|
||
incr := func(m *map[string]map[string]int, key string) { | ||
if (*m) == nil { | ||
*m = map[string]map[string]int{} | ||
} | ||
if (*m)[key] == nil { | ||
(*m)[key] = map[string]int{} | ||
} | ||
(*m)[key][title]++ | ||
} | ||
|
||
for _, diag := range diags { | ||
switch diag.Severity { | ||
case hcl.DiagError: | ||
incr(&lang.Errors, diag.Error()) | ||
case hcl.DiagWarning: | ||
incr(&lang.Warnings, diag.Error()) | ||
case hcl.DiagInvalid: | ||
msg := fmt.Sprintf("invalid diag: %v", diag) | ||
incr(&lang.Errors, msg) | ||
} | ||
} | ||
} | ||
|
||
// Fetch the summary to report on. | ||
// | ||
// Calling this function disables automatic reporting. | ||
func (r *reporter) Summary() Summary { | ||
if r == nil { | ||
return Summary{ReportVersion: version.Version} | ||
} | ||
r.m.Lock() | ||
defer r.m.Unlock() | ||
r.reported = true | ||
return r.summary() | ||
} | ||
|
||
func (r *reporter) summary() Summary { | ||
r.data.ReportVersion = version.Version | ||
return r.data | ||
} | ||
|
||
// If an env var is set to specify where we should write our results to, and if no other | ||
// program has looked at our results, we write out our results to a file. | ||
func (r *reporter) Close() error { | ||
return r.DefaultExport() | ||
} | ||
|
||
// Run the default export behavior on the current report. | ||
func (r *reporter) DefaultExport() error { | ||
r.m.Lock() | ||
defer r.m.Unlock() | ||
dir, ok := os.LookupEnv(ExportTargetDir) | ||
if !ok || r.reported { | ||
return nil | ||
} | ||
r.reported = true | ||
return r.defaultExport(dir) | ||
} | ||
|
||
func (r *reporter) defaultExport(dir string) error { | ||
if dir == "" { | ||
err := fmt.Errorf("%q set to the empty string", ExportTargetDir) | ||
fmt.Fprintln(os.Stderr, err.Error()) | ||
return err | ||
} | ||
|
||
if info, err := os.Stat(dir); os.IsNotExist(err) { | ||
err := os.MkdirAll(dir, 0700) | ||
if err != nil { | ||
return err | ||
} | ||
} else if err != nil { | ||
return err | ||
} else if !info.IsDir() { | ||
err := fmt.Errorf("expected %q to be a directory or empty, found a file", dir) | ||
fmt.Fprintln(os.Stderr, err.Error()) | ||
return err | ||
} | ||
|
||
name := fmt.Sprintf("%s-%s.json", r.data.Name, time.Now().Format("2006-01-02-15:04:05")) | ||
path := filepath.Join(dir, name) | ||
data, err := json.MarshalIndent(r.summary(), "", " ") | ||
if err != nil { | ||
return err | ||
} | ||
return ioutil.WriteFile(path, data, 0600) | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
package report_test | ||
|
||
import ( | ||
"bytes" | ||
"path/filepath" | ||
"testing" | ||
|
||
"github.com/pulumi/pulumi/pkg/v3/codegen/dotnet" | ||
"github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax" | ||
"github.com/pulumi/pulumi/pkg/v3/codegen/nodejs" | ||
"github.com/pulumi/pulumi/pkg/v3/codegen/pcl" | ||
"github.com/pulumi/pulumi/pkg/v3/codegen/report" | ||
"github.com/pulumi/pulumi/pkg/v3/codegen/testing/utils" | ||
"github.com/pulumi/pulumi/pkg/v3/version" | ||
"github.com/stretchr/testify/assert" | ||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
var testdataPath = filepath.Join("..", "testing", "test", "testdata") | ||
|
||
func TestReportExample(t *testing.T) { | ||
t.Parallel() | ||
|
||
reporter := report.New("example", version.Version) | ||
defer reporter.Close() | ||
|
||
examples := []struct { | ||
title string | ||
body string | ||
}{ | ||
{"Our basic bucket", `resource bucket "aws:s3:BucketV2" { }`}, | ||
{"A resource group", `resource group "azure:core:ResourceGroup" { location: "westus2" }`}, | ||
{"Might not bind", `resource foo "not:a:Resource" { foo: "bar" }`}, | ||
} | ||
for _, example := range examples { | ||
parser := syntax.NewParser() | ||
err := parser.ParseFile(bytes.NewReader([]byte(example.body)), example.title) | ||
require.NoError(t, err, "parse failed") | ||
program, diags, err := pcl.BindProgram(parser.Files, pcl.PluginHost(utils.NewHost(testdataPath))) | ||
if err != nil || diags.HasErrors() { | ||
reporter.Report(example.title, "", parser.Files, diags, err) | ||
continue | ||
} | ||
|
||
langs := []string{"dotnet", "nodejs"} | ||
for i, genFn := range []report.GenerateProgramFn{dotnet.GenerateProgram, nodejs.GenerateProgram} { | ||
program, diags, err := report.WrapGen(reporter, example.title, langs[i], parser.Files, genFn)(program) | ||
handleAsNormal(program, diags, err) | ||
} | ||
} | ||
|
||
assert.Equal(t, report.Summary{ | ||
Name: "example", | ||
Stats: report.Stats{ | ||
NumConversions: 5, | ||
Successes: 4, | ||
}, | ||
Languages: map[string]*report.Language{ | ||
"": { | ||
Stats: report.Stats{ | ||
NumConversions: 1, | ||
Successes: 0, | ||
}, | ||
GoErrors: map[string]string{ | ||
"Might not bind": "error: could not locate a compatible plugin in " + | ||
"deploytest, the makefile and & constructor of the plugin host " + | ||
"must define the location of the schema: failed " + | ||
"to locate compatible plugin", | ||
}, | ||
Files: map[string][]report.File{ | ||
"Might not bind": {{Name: "Might not bind", Body: "resource foo \"not:a:Resource\" { foo: \"bar\" }"}}, | ||
}, | ||
}, | ||
"dotnet": { | ||
Stats: report.Stats{ | ||
NumConversions: 2, | ||
Successes: 2, | ||
}, | ||
}, | ||
"nodejs": { | ||
Stats: report.Stats{ | ||
NumConversions: 2, | ||
Successes: 2, | ||
}, | ||
}, | ||
}, | ||
}, reporter.Summary()) | ||
} | ||
|
||
func handleAsNormal(args ...interface{}) {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters