Skip to content

Commit

Permalink
Merge JSON responses from gh api
Browse files Browse the repository at this point in the history 
Fixes #1268
  • Loading branch information
@heaths
heaths committed May 16, 2022
1 parent d244346 commit c0ee199
Show file tree
Hide file tree
Showing 6 changed files with 227 additions and 8 deletions.
6 changes: 5 additions & 1 deletion go.mod
View file
Expand Up @@ -21,6 +21,7 @@ require (
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/go-version v1.3.0
github.com/henvic/httpretty v0.0.6
github.com/imdario/mergo v0.3.12
github.com/itchyny/gojq v0.12.7
github.com/joho/godotenv v1.4.0
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
Expand Down Expand Up @@ -70,4 +71,7 @@ require (
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
)

replace golang.org/x/crypto => github.com/cli/crypto v0.0.0-20210929142629-6be313f59b03
replace (
github.com/imdario/mergo => ../mergo
golang.org/x/crypto => github.com/cli/crypto v0.0.0-20210929142629-6be313f59b03
)
2 changes: 2 additions & 0 deletions go.sum
View file
Expand Up @@ -498,6 +498,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
Expand Down
49 changes: 43 additions & 6 deletions pkg/cmd/api/api.go
View file
Expand Up @@ -299,14 +299,23 @@ func apiRun(opts *ApiOptions) error {

template := export.NewTemplate(opts.IO, opts.Template)

// TODO: Detect if we should create a map or slice.
var v map[string]interface{}
var buf *bytes.Buffer
out := opts.IO.Out
if opts.Paginate {
buf = &bytes.Buffer{}
out = buf
}

hasNextPage := true
for hasNextPage {
resp, err := httpRequest(httpClient, host, method, requestPath, requestBody, requestHeaders)
if err != nil {
return err
}

endCursor, err := processResponse(resp, opts, headersOutputStream, &template)
endCursor, err := processResponse(resp, opts, headersOutputStream, out, &template)
if err != nil {
return err
}
Expand All @@ -316,6 +325,12 @@ func apiRun(opts *ApiOptions) error {
}

if isGraphQL {
err = mergeJSON(&v, buf)
if err != nil {
return err
}
buf.Reset()

hasNextPage = endCursor != ""
if hasNextPage {
params["endCursor"] = endCursor
Expand All @@ -330,10 +345,31 @@ func apiRun(opts *ApiOptions) error {
}
}

// TODO: Properly refactor for interstitial or terminal writes
if isGraphQL && opts.Paginate {
buf, err := json.Marshal(v)
if err != nil {
return err
}

r := bytes.NewBuffer(buf)
if opts.IO.ColorEnabled() {
err = jsoncolor.Write(opts.IO.Out, r, " ")
if err != nil {
return err
}
} else {
_, err = io.Copy(opts.IO.Out, r)
if err != nil {
return err
}
}
}

return template.End()
}

func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream io.Writer, template *export.Template) (endCursor string, err error) {
func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream io.Writer, out io.Writer, template *export.Template) (endCursor string, err error) {
if opts.ShowResponseHeaders {
fmt.Fprintln(headersOutputStream, resp.Proto, resp.Status)
printHeaders(headersOutputStream, resp.Header, opts.IO.ColorEnabled())
Expand Down Expand Up @@ -365,7 +401,7 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream

if opts.FilterOutput != "" && serverError == "" {
// TODO: reuse parsed query across pagination invocations
err = export.FilterJSON(opts.IO.Out, responseBody, opts.FilterOutput)
err = export.FilterJSON(out, responseBody, opts.FilterOutput)
if err != nil {
return
}
Expand All @@ -375,10 +411,11 @@ func processResponse(resp *http.Response, opts *ApiOptions, headersOutputStream
if err != nil {
return
}
} else if isJSON && opts.IO.ColorEnabled() {
err = jsoncolor.Write(opts.IO.Out, responseBody, " ")
} else if isJSON && !isGraphQLPaginate && opts.IO.ColorEnabled() {
// TODO: Refactor to remove condition on isGraphQLPaginate
err = jsoncolor.Write(out, responseBody, " ")
} else {
_, err = io.Copy(opts.IO.Out, responseBody)
_, err = io.Copy(out, responseBody)
}
if err != nil {
return
Expand Down
2 changes: 1 addition & 1 deletion pkg/cmd/api/api_test.go
View file
Expand Up @@ -1299,7 +1299,7 @@ func Test_processResponse_template(t *testing.T) {
Template: `{{range .}}{{.title}} ({{.labels | pluck "name" | join ", " }}){{"\n"}}{{end}}`,
}
template := export.NewTemplate(ios, opts.Template)
_, err := processResponse(&resp, &opts, io.Discard, &template)
_, err := processResponse(&resp, &opts, io.Discard, stdout, &template)
require.NoError(t, err)

err = template.End()
Expand Down
21 changes: 21 additions & 0 deletions pkg/cmd/api/pagination.go
View file
Expand Up @@ -8,6 +8,8 @@ import (
"net/url"
"regexp"
"strings"

"github.com/imdario/mergo"
)

var linkRE = regexp.MustCompile(`<([^>]+)>;\s*rel="([^"]+)"`)
Expand Down Expand Up @@ -106,3 +108,22 @@ func addPerPage(p string, perPage int, params map[string]interface{}) string {

return fmt.Sprintf("%s%sper_page=%d", p, sep, perPage)
}

type arrayOrObject interface {
[]interface{} | map[string]interface{}
}

func mergeJSON[T arrayOrObject](v *T, r io.Reader) error {
buf, err := io.ReadAll(r)
if err != nil {
return err
}

var m T
err = json.Unmarshal(buf, &m)
if err != nil {
return err
}

return mergo.Merge(v, m, mergo.WithAppendSlice, mergo.WithOverride)
}
155 changes: 155 additions & 0 deletions pkg/cmd/api/pagination_test.go
View file
Expand Up @@ -2,9 +2,12 @@ package api

import (
"bytes"
"encoding/json"
"io"
"net/http"
"testing"

"github.com/stretchr/testify/assert"
)

func Test_findNextPage(t *testing.T) {
Expand Down Expand Up @@ -167,3 +170,155 @@ func Test_addPerPage(t *testing.T) {
})
}
}

func Test_mergeJSON_object(t *testing.T) {
page1 := `{
"data": {
"repository": {
"labels": {
"nodes": [
{
"name": "bug",
"description": "Something isn't working"
},
{
"name": "tracking issue",
"description": ""
}
],
"pageInfo": {
"hasNextPage": true,
"endCursor": "Y3Vyc29yOnYyOpK5MjAxOS0xMC0xMFQxMTozODowMy0wNjowMM5f3HZq"
}
}
}
}
}`

page2 := `{
"data": {
"repository": {
"labels": {
"nodes": [
{
"name": "blocked",
"description": ""
},
{
"name": "needs-design",
"description": "An engineering task needs design to proceed"
}
],
"pageInfo": {
"hasNextPage": false,
"endCursor": "Y3Vyc29yOnYyOpK5MjAxOS0xMS0xOFQxMDowMzoxNi0wNzowMM5kXbLp"
}
}
}
}
}`

var data map[string]interface{}
for _, page := range []string{page1, page2} {
err := mergeJSON(&data, bytes.NewBufferString(page))
if !assert.NoError(t, err) {
return
}
}

actual, err := json.Marshal(data)
if !assert.NoError(t, err) {
return
}

expected := `{
"data": {
"repository": {
"labels": {
"nodes": [
{
"name": "bug",
"description": "Something isn't working"
},
{
"name": "tracking issue",
"description": ""
},
{
"name": "blocked",
"description": ""
},
{
"name": "needs-design",
"description": "An engineering task needs design to proceed"
}
],
"pageInfo": {
"hasNextPage": false,
"endCursor": "Y3Vyc29yOnYyOpK5MjAxOS0xMS0xOFQxMDowMzoxNi0wNzowMM5kXbLp"
}
}
}
}
}`

assert.JSONEq(t, expected, string(actual))
}

func Test_mergeJSON_array(t *testing.T) {
page1 := `[
{
"name": "bug",
"description": "Something isn't working"
},
{
"name": "tracking issue",
"description": ""
}
]`

page2 := `[
{
"name": "blocked",
"description": ""
},
{
"name": "needs-design",
"description": "An engineering task needs design to proceed"
}
]`

var data []interface{}
for _, page := range []string{page1, page2} {
err := mergeJSON(&data, bytes.NewBufferString(page))
if !assert.NoError(t, err) {
return
}
}

actual, err := json.Marshal(data)
if !assert.NoError(t, err) {
return
}

expected := `[
{
"name": "bug",
"description": "Something isn't working"
},
{
"name": "tracking issue",
"description": ""
},
{
"name": "blocked",
"description": ""
},
{
"name": "needs-design",
"description": "An engineering task needs design to proceed"
}
]`

assert.JSONEq(t, expected, string(actual))
}

0 comments on commit c0ee199

Please sign in to comment.