From 3c209937608940afdf4249d5f11add7fc2ca9ce8 Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Tue, 17 May 2022 00:07:00 -0700 Subject: [PATCH] Merge JSON responses from `gh api` Fixes #1268 --- go.mod | 6 +- go.sum | 4 + pkg/cmd/api/api.go | 80 +++++++++++++---- pkg/cmd/api/api_test.go | 14 +-- pkg/cmd/api/pagination.go | 30 +++++++ pkg/cmd/api/pagination_test.go | 155 +++++++++++++++++++++++++++++++++ 6 files changed, 264 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 70e6f039a92..e411a6d04f6 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-version v1.3.0 github.com/henvic/httpretty v0.0.6 + github.com/imdario/mergo v0.3.12 github.com/itchyny/gojq v0.12.7 github.com/joho/godotenv v1.4.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 @@ -70,4 +71,7 @@ require ( gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) -replace golang.org/x/crypto => github.com/cli/crypto v0.0.0-20210929142629-6be313f59b03 +replace ( + github.com/imdario/mergo => github.com/heaths/mergo v0.3.13-0.20220516084937-200f90d97053 + golang.org/x/crypto => github.com/cli/crypto v0.0.0-20210929142629-6be313f59b03 +) diff --git a/go.sum b/go.sum index f4957528d25..1943bf2df84 100644 --- a/go.sum +++ b/go.sum @@ -152,6 +152,8 @@ github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/heaths/mergo v0.3.13-0.20220516084937-200f90d97053 h1:vEGlfkWJUiK1W+n3sOMkThHLY1zr+a5bVQ4L1OEZd9k= +github.com/heaths/mergo v0.3.13-0.20220516084937-200f90d97053/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/henvic/httpretty v0.0.6 h1:JdzGzKZBajBfnvlMALXXMVQWxWMF/ofTy8C3/OSUTxs= github.com/henvic/httpretty v0.0.6/go.mod h1:X38wLjWXHkXT7r2+uK8LjCMne9rsuNaBLJ+5cU2/Pmo= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= @@ -498,6 +500,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= diff --git a/pkg/cmd/api/api.go b/pkg/cmd/api/api.go index 2d92173ab54..2fe1316e6bb 100644 --- a/pkg/cmd/api/api.go +++ b/pkg/cmd/api/api.go @@ -299,6 +299,15 @@ func apiRun(opts *ApiOptions) error { template := export.NewTemplate(opts.IO, opts.Template) + var pages interface{} + var outBuffer *bytes.Buffer + out := opts.IO.Out + if opts.Paginate { + outBuffer = &bytes.Buffer{} + out = outBuffer + } + + isJSON := false hasNextPage := true for hasNextPage { resp, err := httpRequest(httpClient, host, method, requestPath, requestBody, requestHeaders) @@ -306,7 +315,8 @@ func apiRun(opts *ApiOptions) error { return err } - endCursor, err := processResponse(resp, opts, headersOutputStream, &template) + var endCursor string + isJSON, endCursor, err = processResponse(resp, opts, headersOutputStream, out, &template) if err != nil { return err } @@ -315,6 +325,14 @@ func apiRun(opts *ApiOptions) error { break } + if opts.Paginate && isJSON { + err = mergeJSON(&pages, outBuffer) + if err != nil { + return err + } + outBuffer.Reset() + } + if isGraphQL { hasNextPage = endCursor != "" if hasNextPage { @@ -330,10 +348,24 @@ func apiRun(opts *ApiOptions) error { } } + if opts.Paginate && isJSON { + var buf []byte + buf, err = json.Marshal(pages) + if err != nil { + return err + } + + outBuffer := bytes.NewBuffer(buf) + err = printResponse(outBuffer, opts, isJSON, false, opts.IO.Out, &template) + if err != nil { + return err + } + } + return template.End() } -func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream io.Writer, template *export.Template) (endCursor string, err error) { +func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream io.Writer, out io.Writer, template *export.Template) (isJSON bool, endCursor string, err error) { if opts.ShowResponseHeaders { fmt.Fprintln(headersOutputStream, resp.Proto, resp.Status) printHeaders(headersOutputStream, resp.Header, opts.IO.ColorEnabled()) @@ -346,7 +378,7 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream var responseBody io.Reader = resp.Body defer resp.Body.Close() - isJSON, _ := regexp.MatchString(`[/+]json(;|$)`, resp.Header.Get("Content-Type")) + isJSON, _ = regexp.MatchString(`[/+]json(;|$)`, resp.Header.Get("Content-Type")) var serverError string if isJSON && (opts.RequestPath == "graphql" || resp.StatusCode >= 400) { @@ -363,25 +395,14 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream responseBody = io.TeeReader(responseBody, bodyCopy) } - if opts.FilterOutput != "" && serverError == "" { - // TODO: reuse parsed query across pagination invocations - err = export.FilterJSON(opts.IO.Out, responseBody, opts.FilterOutput) - if err != nil { - return - } - } else if opts.Template != "" && serverError == "" { - // TODO: reuse parsed template across pagination invocations - err = template.Execute(responseBody) + if !opts.Paginate { + // Print the body immediately if not paginating + err = printResponse(responseBody, opts, isJSON, serverError != "", out, template) + } else { + _, err = io.Copy(out, responseBody) if err != nil { return } - } else if isJSON && opts.IO.ColorEnabled() { - err = jsoncolor.Write(opts.IO.Out, responseBody, " ") - } else { - _, err = io.Copy(opts.IO.Out, responseBody) - } - if err != nil { - return } if serverError == "" && resp.StatusCode > 299 { @@ -406,6 +427,27 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream return } +func printResponse(responseBody io.Reader, opts *ApiOptions, isJSON, hasServerErr bool, out io.Writer, template *export.Template) (err error) { + if opts.FilterOutput != "" && !hasServerErr { + // TODO: reuse parsed query across pagination invocations + err = export.FilterJSON(out, responseBody, opts.FilterOutput) + if err != nil { + return + } + } else if opts.Template != "" && !hasServerErr { + // TODO: reuse parsed template across pagination invocations + err = template.Execute(responseBody) + if err != nil { + return + } + } else if isJSON && opts.IO.ColorEnabled() { + err = jsoncolor.Write(out, responseBody, " ") + } else { + _, err = io.Copy(out, responseBody) + } + return +} + var placeholderRE = regexp.MustCompile(`(\:(owner|repo|branch)\b|\{[a-z]+\})`) // fillPlaceholders replaces placeholders with values from the current repository diff --git a/pkg/cmd/api/api_test.go b/pkg/cmd/api/api_test.go index 8e1eace152a..3a665443c59 100644 --- a/pkg/cmd/api/api_test.go +++ b/pkg/cmd/api/api_test.go @@ -573,20 +573,24 @@ func Test_apiRun_paginationREST(t *testing.T) { StatusCode: 200, Body: io.NopCloser(bytes.NewBufferString(`{"page":1}`)), Header: http.Header{ - "Link": []string{`; rel="next", ; rel="last"`}, + "Content-Type": []string{"application/json"}, + "Link": []string{`; rel="next", ; rel="last"`}, }, }, { StatusCode: 200, Body: io.NopCloser(bytes.NewBufferString(`{"page":2}`)), Header: http.Header{ - "Link": []string{`; rel="next", ; rel="last"`}, + "Content-Type": []string{"application/json"}, + "Link": []string{`; rel="next", ; rel="last"`}, }, }, { StatusCode: 200, Body: io.NopCloser(bytes.NewBufferString(`{"page":3}`)), - Header: http.Header{}, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, }, } @@ -615,7 +619,7 @@ func Test_apiRun_paginationREST(t *testing.T) { err := apiRun(&options) assert.NoError(t, err) - assert.Equal(t, `{"page":1}{"page":2}{"page":3}`, stdout.String(), "stdout") + assert.Equal(t, `{"page":3}`, stdout.String(), "stdout") assert.Equal(t, "", stderr.String(), "stderr") assert.Equal(t, "https://api.github.com/issues?page=1&per_page=50", responses[0].Request.URL.String()) @@ -1299,7 +1303,7 @@ func Test_processResponse_template(t *testing.T) { Template: `{{range .}}{{.title}} ({{.labels | pluck "name" | join ", " }}){{"\n"}}{{end}}`, } template := export.NewTemplate(ios, opts.Template) - _, err := processResponse(&resp, &opts, io.Discard, &template) + _, _, err := processResponse(&resp, &opts, io.Discard, stdout, &template) require.NoError(t, err) err = template.End() diff --git a/pkg/cmd/api/pagination.go b/pkg/cmd/api/pagination.go index 65d816480e4..34af1716538 100644 --- a/pkg/cmd/api/pagination.go +++ b/pkg/cmd/api/pagination.go @@ -1,6 +1,7 @@ package api import ( + "bytes" "encoding/json" "fmt" "io" @@ -8,6 +9,8 @@ import ( "net/url" "regexp" "strings" + + "github.com/imdario/mergo" ) var linkRE = regexp.MustCompile(`<([^>]+)>;\s*rel="([^"]+)"`) @@ -106,3 +109,30 @@ func addPerPage(p string, perPage int, params map[string]interface{}) string { return fmt.Sprintf("%s%sper_page=%d", p, sep, perPage) } + +func mergeJSON(v *interface{}, r io.Reader) error { + buf, err := io.ReadAll(r) + if err != nil { + return err + } + + var m interface{} + if bytes.HasPrefix(buf, []byte("{")) { + if *v == nil { + *v = new(map[string]interface{}) + } + m = new(map[string]interface{}) + } else if bytes.HasPrefix(buf, []byte("[")) { + if *v == nil { + *v = new([]interface{}) + } + m = new([]interface{}) + } + + err = json.Unmarshal(buf, &m) + if err != nil { + return err + } + + return mergo.Merge(*v, m, mergo.WithAppendSlice, mergo.WithOverride) +} diff --git a/pkg/cmd/api/pagination_test.go b/pkg/cmd/api/pagination_test.go index 3bb1a8ec5c3..0f731104ba3 100644 --- a/pkg/cmd/api/pagination_test.go +++ b/pkg/cmd/api/pagination_test.go @@ -2,9 +2,12 @@ package api import ( "bytes" + "encoding/json" "io" "net/http" "testing" + + "github.com/stretchr/testify/assert" ) func Test_findNextPage(t *testing.T) { @@ -167,3 +170,155 @@ func Test_addPerPage(t *testing.T) { }) } } + +func Test_mergeJSON_object(t *testing.T) { + page1 := `{ + "data": { + "repository": { + "labels": { + "nodes": [ + { + "name": "bug", + "description": "Something isn't working" + }, + { + "name": "tracking issue", + "description": "" + } + ], + "pageInfo": { + "hasNextPage": true, + "endCursor": "Y3Vyc29yOnYyOpK5MjAxOS0xMC0xMFQxMTozODowMy0wNjowMM5f3HZq" + } + } + } + } + }` + + page2 := `{ + "data": { + "repository": { + "labels": { + "nodes": [ + { + "name": "blocked", + "description": "" + }, + { + "name": "needs-design", + "description": "An engineering task needs design to proceed" + } + ], + "pageInfo": { + "hasNextPage": false, + "endCursor": "Y3Vyc29yOnYyOpK5MjAxOS0xMS0xOFQxMDowMzoxNi0wNzowMM5kXbLp" + } + } + } + } + }` + + var data interface{} + for _, page := range []string{page1, page2} { + err := mergeJSON(&data, bytes.NewBufferString(page)) + if !assert.NoError(t, err) { + return + } + } + + actual, err := json.Marshal(data) + if !assert.NoError(t, err) { + return + } + + expected := `{ + "data": { + "repository": { + "labels": { + "nodes": [ + { + "name": "bug", + "description": "Something isn't working" + }, + { + "name": "tracking issue", + "description": "" + }, + { + "name": "blocked", + "description": "" + }, + { + "name": "needs-design", + "description": "An engineering task needs design to proceed" + } + ], + "pageInfo": { + "hasNextPage": false, + "endCursor": "Y3Vyc29yOnYyOpK5MjAxOS0xMS0xOFQxMDowMzoxNi0wNzowMM5kXbLp" + } + } + } + } + }` + + assert.JSONEq(t, expected, string(actual)) +} + +func Test_mergeJSON_array(t *testing.T) { + page1 := `[ + { + "name": "bug", + "description": "Something isn't working" + }, + { + "name": "tracking issue", + "description": "" + } + ]` + + page2 := `[ + { + "name": "blocked", + "description": "" + }, + { + "name": "needs-design", + "description": "An engineering task needs design to proceed" + } + ]` + + var data interface{} + for _, page := range []string{page1, page2} { + err := mergeJSON(&data, bytes.NewBufferString(page)) + if !assert.NoError(t, err) { + return + } + } + + actual, err := json.Marshal(data) + if !assert.NoError(t, err) { + return + } + + expected := `[ + { + "name": "bug", + "description": "Something isn't working" + }, + { + "name": "tracking issue", + "description": "" + }, + { + "name": "blocked", + "description": "" + }, + { + "name": "needs-design", + "description": "An engineering task needs design to proceed" + } + ]` + + assert.JSONEq(t, expected, string(actual)) +}