diff --git a/go.mod b/go.mod index 70e6f039a92..d38e02ebd5e 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-version v1.3.0 github.com/henvic/httpretty v0.0.6 + github.com/imdario/mergo v0.3.12 github.com/itchyny/gojq v0.12.7 github.com/joho/godotenv v1.4.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 @@ -70,4 +71,7 @@ require ( gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) -replace golang.org/x/crypto => github.com/cli/crypto v0.0.0-20210929142629-6be313f59b03 +replace ( + github.com/imdario/mergo => ../mergo + golang.org/x/crypto => github.com/cli/crypto v0.0.0-20210929142629-6be313f59b03 +) diff --git a/go.sum b/go.sum index f4957528d25..8e6cbed7015 100644 --- a/go.sum +++ b/go.sum @@ -498,6 +498,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= diff --git a/pkg/cmd/api/api.go b/pkg/cmd/api/api.go index 2d92173ab54..4454722c77e 100644 --- a/pkg/cmd/api/api.go +++ b/pkg/cmd/api/api.go @@ -299,6 +299,15 @@ func apiRun(opts *ApiOptions) error { template := export.NewTemplate(opts.IO, opts.Template) + // TODO: Detect if we should create a map or slice. + var v map[string]interface{} + var buf *bytes.Buffer + out := opts.IO.Out + if opts.Paginate { + buf = &bytes.Buffer{} + out = buf + } + hasNextPage := true for hasNextPage { resp, err := httpRequest(httpClient, host, method, requestPath, requestBody, requestHeaders) @@ -306,7 +315,7 @@ func apiRun(opts *ApiOptions) error { return err } - endCursor, err := processResponse(resp, opts, headersOutputStream, &template) + endCursor, err := processResponse(resp, opts, headersOutputStream, out, &template) if err != nil { return err } @@ -316,6 +325,12 @@ func apiRun(opts *ApiOptions) error { } if isGraphQL { + err = mergeJSON(&v, buf) + if err != nil { + return err + } + buf.Reset() + hasNextPage = endCursor != "" if hasNextPage { params["endCursor"] = endCursor @@ -330,10 +345,31 @@ func apiRun(opts *ApiOptions) error { } } + // TODO: Properly refactor for interstitial or terminal writes + if isGraphQL && opts.Paginate { + buf, err := json.Marshal(v) + if err != nil { + return err + } + + r := bytes.NewBuffer(buf) + if opts.IO.ColorEnabled() { + err = jsoncolor.Write(opts.IO.Out, r, " ") + if err != nil { + return err + } + } else { + _, err = io.Copy(opts.IO.Out, r) + if err != nil { + return err + } + } + } + return template.End() } -func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream io.Writer, template *export.Template) (endCursor string, err error) { +func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream io.Writer, out io.Writer, template *export.Template) (endCursor string, err error) { if opts.ShowResponseHeaders { fmt.Fprintln(headersOutputStream, resp.Proto, resp.Status) printHeaders(headersOutputStream, resp.Header, opts.IO.ColorEnabled()) @@ -365,7 +401,7 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream if opts.FilterOutput != "" && serverError == "" { // TODO: reuse parsed query across pagination invocations - err = export.FilterJSON(opts.IO.Out, responseBody, opts.FilterOutput) + err = export.FilterJSON(out, responseBody, opts.FilterOutput) if err != nil { return } @@ -375,10 +411,11 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream if err != nil { return } - } else if isJSON && opts.IO.ColorEnabled() { - err = jsoncolor.Write(opts.IO.Out, responseBody, " ") + } else if isJSON && !isGraphQLPaginate && opts.IO.ColorEnabled() { + // TODO: Refactor to remove condition on isGraphQLPaginate + err = jsoncolor.Write(out, responseBody, " ") } else { - _, err = io.Copy(opts.IO.Out, responseBody) + _, err = io.Copy(out, responseBody) } if err != nil { return diff --git a/pkg/cmd/api/api_test.go b/pkg/cmd/api/api_test.go index 8e1eace152a..da460621cee 100644 --- a/pkg/cmd/api/api_test.go +++ b/pkg/cmd/api/api_test.go @@ -1299,7 +1299,7 @@ func Test_processResponse_template(t *testing.T) { Template: `{{range .}}{{.title}} ({{.labels | pluck "name" | join ", " }}){{"\n"}}{{end}}`, } template := export.NewTemplate(ios, opts.Template) - _, err := processResponse(&resp, &opts, io.Discard, &template) + _, err := processResponse(&resp, &opts, io.Discard, stdout, &template) require.NoError(t, err) err = template.End() diff --git a/pkg/cmd/api/pagination.go b/pkg/cmd/api/pagination.go index 65d816480e4..a977b051324 100644 --- a/pkg/cmd/api/pagination.go +++ b/pkg/cmd/api/pagination.go @@ -8,6 +8,8 @@ import ( "net/url" "regexp" "strings" + + "github.com/imdario/mergo" ) var linkRE = regexp.MustCompile(`<([^>]+)>;\s*rel="([^"]+)"`) @@ -106,3 +108,22 @@ func addPerPage(p string, perPage int, params map[string]interface{}) string { return fmt.Sprintf("%s%sper_page=%d", p, sep, perPage) } + +type arrayOrObject interface { + []interface{} | map[string]interface{} +} + +func mergeJSON[T arrayOrObject](v *T, r io.Reader) error { + buf, err := io.ReadAll(r) + if err != nil { + return err + } + + var m T + err = json.Unmarshal(buf, &m) + if err != nil { + return err + } + + return mergo.Merge(v, m, mergo.WithAppendSlice, mergo.WithOverride) +} diff --git a/pkg/cmd/api/pagination_test.go b/pkg/cmd/api/pagination_test.go index 3bb1a8ec5c3..6879cac8c00 100644 --- a/pkg/cmd/api/pagination_test.go +++ b/pkg/cmd/api/pagination_test.go @@ -2,9 +2,12 @@ package api import ( "bytes" + "encoding/json" "io" "net/http" "testing" + + "github.com/stretchr/testify/assert" ) func Test_findNextPage(t *testing.T) { @@ -167,3 +170,155 @@ func Test_addPerPage(t *testing.T) { }) } } + +func Test_mergeJSON_object(t *testing.T) { + page1 := `{ + "data": { + "repository": { + "labels": { + "nodes": [ + { + "name": "bug", + "description": "Something isn't working" + }, + { + "name": "tracking issue", + "description": "" + } + ], + "pageInfo": { + "hasNextPage": true, + "endCursor": "Y3Vyc29yOnYyOpK5MjAxOS0xMC0xMFQxMTozODowMy0wNjowMM5f3HZq" + } + } + } + } + }` + + page2 := `{ + "data": { + "repository": { + "labels": { + "nodes": [ + { + "name": "blocked", + "description": "" + }, + { + "name": "needs-design", + "description": "An engineering task needs design to proceed" + } + ], + "pageInfo": { + "hasNextPage": false, + "endCursor": "Y3Vyc29yOnYyOpK5MjAxOS0xMS0xOFQxMDowMzoxNi0wNzowMM5kXbLp" + } + } + } + } + }` + + var data map[string]interface{} + for _, page := range []string{page1, page2} { + err := mergeJSON(&data, bytes.NewBufferString(page)) + if !assert.NoError(t, err) { + return + } + } + + actual, err := json.Marshal(data) + if !assert.NoError(t, err) { + return + } + + expected := `{ + "data": { + "repository": { + "labels": { + "nodes": [ + { + "name": "bug", + "description": "Something isn't working" + }, + { + "name": "tracking issue", + "description": "" + }, + { + "name": "blocked", + "description": "" + }, + { + "name": "needs-design", + "description": "An engineering task needs design to proceed" + } + ], + "pageInfo": { + "hasNextPage": false, + "endCursor": "Y3Vyc29yOnYyOpK5MjAxOS0xMS0xOFQxMDowMzoxNi0wNzowMM5kXbLp" + } + } + } + } + }` + + assert.JSONEq(t, expected, string(actual)) +} + +func Test_mergeJSON_array(t *testing.T) { + page1 := `[ + { + "name": "bug", + "description": "Something isn't working" + }, + { + "name": "tracking issue", + "description": "" + } + ]` + + page2 := `[ + { + "name": "blocked", + "description": "" + }, + { + "name": "needs-design", + "description": "An engineering task needs design to proceed" + } + ]` + + var data []interface{} + for _, page := range []string{page1, page2} { + err := mergeJSON(&data, bytes.NewBufferString(page)) + if !assert.NoError(t, err) { + return + } + } + + actual, err := json.Marshal(data) + if !assert.NoError(t, err) { + return + } + + expected := `[ + { + "name": "bug", + "description": "Something isn't working" + }, + { + "name": "tracking issue", + "description": "" + }, + { + "name": "blocked", + "description": "" + }, + { + "name": "needs-design", + "description": "An engineering task needs design to proceed" + } + ]` + + assert.JSONEq(t, expected, string(actual)) +}