Merge JSON responses from gh api

Fixes #1268

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-version v1.3.0
 	github.com/henvic/httpretty v0.0.6
+	github.com/imdario/mergo v0.3.12
 	github.com/itchyny/gojq v0.12.7
 	github.com/joho/godotenv v1.4.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
@@ -70,4 +71,7 @@ require (
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 )
 
-replace golang.org/x/crypto => github.com/cli/crypto v0.0.0-20210929142629-6be313f59b03
+replace (
+	github.com/imdario/mergo => github.com/heaths/mergo v0.3.13-0.20220516084937-200f90d97053
+	golang.org/x/crypto => github.com/cli/crypto v0.0.0-20210929142629-6be313f59b03
+)

go.sum

@@ -152,6 +152,8 @@ github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04
 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/heaths/mergo v0.3.13-0.20220516084937-200f90d97053 h1:vEGlfkWJUiK1W+n3sOMkThHLY1zr+a5bVQ4L1OEZd9k=
+github.com/heaths/mergo v0.3.13-0.20220516084937-200f90d97053/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/henvic/httpretty v0.0.6 h1:JdzGzKZBajBfnvlMALXXMVQWxWMF/ofTy8C3/OSUTxs=
 github.com/henvic/httpretty v0.0.6/go.mod h1:X38wLjWXHkXT7r2+uK8LjCMne9rsuNaBLJ+5cU2/Pmo=
 github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
@@ -498,6 +500,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=

pkg/cmd/api/api.go

@@ -299,14 +299,24 @@ func apiRun(opts *ApiOptions) error {
 
 	template := export.NewTemplate(opts.IO, opts.Template)
 
+	var pages interface{}
+	var outBuffer *bytes.Buffer
+	out := opts.IO.Out
+	if opts.Paginate {
+		outBuffer = &bytes.Buffer{}
+		out = outBuffer
+	}
+
+	isJSON := false
 	hasNextPage := true
 	for hasNextPage {
 		resp, err := httpRequest(httpClient, host, method, requestPath, requestBody, requestHeaders)
 		if err != nil {
 			return err
 		}
 
-		endCursor, err := processResponse(resp, opts, headersOutputStream, &template)
+		var endCursor string
+		isJSON, endCursor, err = processResponse(resp, opts, headersOutputStream, out, &template)
 		if err != nil {
 			return err
 		}
@@ -315,6 +325,14 @@ func apiRun(opts *ApiOptions) error {
 			break
 		}
 
+		if isJSON {
+			err = mergeJSON(&pages, outBuffer)
+			if err != nil {
+				return err
+			}
+			outBuffer.Reset()
+		}
+
 		if isGraphQL {
 			hasNextPage = endCursor != ""
 			if hasNextPage {
@@ -330,10 +348,24 @@ func apiRun(opts *ApiOptions) error {
 		}
 	}
 
+	if opts.Paginate && isJSON {
+		var buf []byte
+		buf, err = json.Marshal(pages)
+		if err != nil {
+			return err
+		}
+
+		outBuffer := bytes.NewBuffer(buf)
+		err = printResponse(outBuffer, opts, isJSON, false, opts.IO.Out, &template)
+		if err != nil {
+			return err
+		}
+	}
+
 	return template.End()
 }
 
-func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream io.Writer, template *export.Template) (endCursor string, err error) {
+func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream io.Writer, out io.Writer, template *export.Template) (isJSON bool, endCursor string, err error) {
 	if opts.ShowResponseHeaders {
 		fmt.Fprintln(headersOutputStream, resp.Proto, resp.Status)
 		printHeaders(headersOutputStream, resp.Header, opts.IO.ColorEnabled())
@@ -346,7 +378,7 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream
 	var responseBody io.Reader = resp.Body
 	defer resp.Body.Close()
 
-	isJSON, _ := regexp.MatchString(`[/+]json(;|$)`, resp.Header.Get("Content-Type"))
+	isJSON, _ = regexp.MatchString(`[/+]json(;|$)`, resp.Header.Get("Content-Type"))
 
 	var serverError string
 	if isJSON && (opts.RequestPath == "graphql" || resp.StatusCode >= 400) {
@@ -363,25 +395,14 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream
 		responseBody = io.TeeReader(responseBody, bodyCopy)
 	}
 
-	if opts.FilterOutput != "" && serverError == "" {
-		// TODO: reuse parsed query across pagination invocations
-		err = export.FilterJSON(opts.IO.Out, responseBody, opts.FilterOutput)
-		if err != nil {
-			return
-		}
-	} else if opts.Template != "" && serverError == "" {
-		// TODO: reuse parsed template across pagination invocations
-		err = template.Execute(responseBody)
+	if !opts.Paginate {
+		// Print the body immediately if not paginating
+		err = printResponse(responseBody, opts, isJSON, serverError != "", out, template)
+	} else {
+		_, err = io.Copy(out, responseBody)
 		if err != nil {
 			return
 		}
-	} else if isJSON && opts.IO.ColorEnabled() {
-		err = jsoncolor.Write(opts.IO.Out, responseBody, "  ")
-	} else {
-		_, err = io.Copy(opts.IO.Out, responseBody)
-	}
-	if err != nil {
-		return
 	}
 
 	if serverError == "" && resp.StatusCode > 299 {
@@ -406,6 +427,27 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream
 	return
 }
 
+func printResponse(responseBody io.Reader, opts *ApiOptions, isJSON, hasServerErr bool, out io.Writer, template *export.Template) (err error) {
+	if opts.FilterOutput != "" && !hasServerErr {
+		// TODO: reuse parsed query across pagination invocations
+		err = export.FilterJSON(out, responseBody, opts.FilterOutput)
+		if err != nil {
+			return
+		}
+	} else if opts.Template != "" && !hasServerErr {
+		// TODO: reuse parsed template across pagination invocations
+		err = template.Execute(responseBody)
+		if err != nil {
+			return
+		}
+	} else if isJSON && opts.IO.ColorEnabled() {
+		err = jsoncolor.Write(out, responseBody, "  ")
+	} else {
+		_, err = io.Copy(out, responseBody)
+	}
+	return
+}
+
 var placeholderRE = regexp.MustCompile(`(\:(owner|repo|branch)\b|\{[a-z]+\})`)
 
 // fillPlaceholders replaces placeholders with values from the current repository

pkg/cmd/api/api_test.go

@@ -573,20 +573,24 @@ func Test_apiRun_paginationREST(t *testing.T) {
 			StatusCode: 200,
 			Body:       io.NopCloser(bytes.NewBufferString(`{"page":1}`)),
 			Header: http.Header{
-				"Link": []string{`<https://api.github.com/repositories/1227/issues?page=2>; rel="next", <https://api.github.com/repositories/1227/issues?page=3>; rel="last"`},
+				"Content-Type": []string{"application/json"},
+				"Link":         []string{`<https://api.github.com/repositories/1227/issues?page=2>; rel="next", <https://api.github.com/repositories/1227/issues?page=3>; rel="last"`},
 			},
 		},
 		{
 			StatusCode: 200,
 			Body:       io.NopCloser(bytes.NewBufferString(`{"page":2}`)),
 			Header: http.Header{
-				"Link": []string{`<https://api.github.com/repositories/1227/issues?page=3>; rel="next", <https://api.github.com/repositories/1227/issues?page=3>; rel="last"`},
+				"Content-Type": []string{"application/json"},
+				"Link":         []string{`<https://api.github.com/repositories/1227/issues?page=3>; rel="next", <https://api.github.com/repositories/1227/issues?page=3>; rel="last"`},
 			},
 		},
 		{
 			StatusCode: 200,
 			Body:       io.NopCloser(bytes.NewBufferString(`{"page":3}`)),
-			Header:     http.Header{},
+			Header: http.Header{
+				"Content-Type": []string{"application/json"},
+			},
 		},
 	}
 
@@ -615,7 +619,7 @@ func Test_apiRun_paginationREST(t *testing.T) {
 	err := apiRun(&options)
 	assert.NoError(t, err)
 
-	assert.Equal(t, `{"page":1}{"page":2}{"page":3}`, stdout.String(), "stdout")
+	assert.Equal(t, `{"page":3}`, stdout.String(), "stdout")
 	assert.Equal(t, "", stderr.String(), "stderr")
 
 	assert.Equal(t, "https://api.github.com/issues?page=1&per_page=50", responses[0].Request.URL.String())
@@ -1299,7 +1303,7 @@ func Test_processResponse_template(t *testing.T) {
 		Template: `{{range .}}{{.title}} ({{.labels | pluck "name" | join ", " }}){{"\n"}}{{end}}`,
 	}
 	template := export.NewTemplate(ios, opts.Template)
-	_, err := processResponse(&resp, &opts, io.Discard, &template)
+	_, _, err := processResponse(&resp, &opts, io.Discard, stdout, &template)
 	require.NoError(t, err)
 
 	err = template.End()

pkg/cmd/api/pagination.go

@@ -1,13 +1,16 @@
 package api
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
+
+	"github.com/imdario/mergo"
 )
 
 var linkRE = regexp.MustCompile(`<([^>]+)>;\s*rel="([^"]+)"`)
@@ -106,3 +109,30 @@ func addPerPage(p string, perPage int, params map[string]interface{}) string {
 
 	return fmt.Sprintf("%s%sper_page=%d", p, sep, perPage)
 }
+
+func mergeJSON(v *interface{}, r io.Reader) error {
+	buf, err := io.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	var m interface{}
+	if bytes.HasPrefix(buf, []byte("{")) {
+		if *v == nil {
+			*v = new(map[string]interface{})
+		}
+		m = new(map[string]interface{})
+	} else if bytes.HasPrefix(buf, []byte("[")) {
+		if *v == nil {
+			*v = new([]interface{})
+		}
+		m = new([]interface{})
+	}
+
+	err = json.Unmarshal(buf, &m)
+	if err != nil {
+		return err
+	}
+
+	return mergo.Merge(*v, m, mergo.WithAppendSlice, mergo.WithOverride)
+}