From 3df15accf5c5e0e8cb0a3c365d9b71b17ef363b2 Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Thu, 25 Jan 2024 02:24:20 -0800 Subject: [PATCH] Support merging JSON arrays, objects Partly resolves cli/cli#1268 and replaces cli/cli#5652 --- go.mod | 1 + go.sum | 2 + pkg/jsonmerge/array.go | 76 ++++++++++++++++++++++++++++++++++++ pkg/jsonmerge/array_test.go | 71 +++++++++++++++++++++++++++++++++ pkg/jsonmerge/merge.go | 12 ++++++ pkg/jsonmerge/object.go | 73 ++++++++++++++++++++++++++++++++++ pkg/jsonmerge/object_test.go | 74 +++++++++++++++++++++++++++++++++++ 7 files changed, 309 insertions(+) create mode 100644 pkg/jsonmerge/array.go create mode 100644 pkg/jsonmerge/array_test.go create mode 100644 pkg/jsonmerge/merge.go create mode 100644 pkg/jsonmerge/object.go create mode 100644 pkg/jsonmerge/object_test.go diff --git a/go.mod b/go.mod index 1286269..45dbaed 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/cli/go-gh/v2 go 1.21 require ( + dario.cat/mergo v1.0.0 github.com/AlecAivazis/survey/v2 v2.3.7 github.com/MakeNowJust/heredoc v1.0.0 github.com/charmbracelet/glamour v0.6.0 diff --git a/go.sum b/go.sum index 35c94da..4433fd6 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= diff --git a/pkg/jsonmerge/array.go b/pkg/jsonmerge/array.go new file mode 100644 index 0000000..3a8c2c7 --- /dev/null +++ b/pkg/jsonmerge/array.go @@ -0,0 +1,76 @@ +package jsonmerge + +import "io" + +type arrayMerger struct { + isFirstPage bool +} + +// NewArrayMerger creates a Merger for JSON arrays. +func NewArrayMerger() Merger { + return &arrayMerger{ + isFirstPage: true, + } +} + +func (merger *arrayMerger) NewPage(r io.Reader, isLastPage bool) io.ReadCloser { + return &arrayMergerPage{ + merger: merger, + Reader: r, + isLastPage: isLastPage, + } +} + +func (m *arrayMerger) Close() error { + // arrayMerger merges when reading, so any output was already written + // and there's nothing to do on Close. + return nil +} + +type arrayMergerPage struct { + merger *arrayMerger + + io.Reader + isLastPage bool + + isSubsequentRead bool + cachedByte byte +} + +func (page *arrayMergerPage) Read(p []byte) (int, error) { + var n int + var err error + + if page.cachedByte != 0 && len(p) > 0 { + p[0] = page.cachedByte + n, err = page.Reader.Read(p[1:]) + n += 1 + page.cachedByte = 0 + } else { + n, err = page.Reader.Read(p) + } + + if !page.isSubsequentRead && !page.merger.isFirstPage && n > 0 && p[0] == '[' { + if n > 1 && p[1] == ']' { + // Empty array case. + p[0] = ' ' + } else { + // Avoid starting a new array and continue with a comma instead. + p[0] = ',' + } + } + + if !page.isLastPage && n > 0 && p[n-1] == ']' { + // Avoid closing off an array in case we determine we are at EOF. + page.cachedByte = p[n-1] + n -= 1 + } + + page.isSubsequentRead = true + return n, err +} + +func (page *arrayMergerPage) Close() error { + page.merger.isFirstPage = false + return nil +} diff --git a/pkg/jsonmerge/array_test.go b/pkg/jsonmerge/array_test.go new file mode 100644 index 0000000..e350479 --- /dev/null +++ b/pkg/jsonmerge/array_test.go @@ -0,0 +1,71 @@ +package jsonmerge + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestArrayMerger_singleEmptyArray(t *testing.T) { + merger := NewArrayMerger() + w := &bytes.Buffer{} + + r1 := bytes.NewBufferString(`[]`) + p1 := merger.NewPage(r1, true) + n, err := io.Copy(w, p1) + require.NoError(t, err) + assert.Equal(t, int64(2), n) + assert.NoError(t, p1.Close()) + assert.Equal(t, `[]`, w.String()) + + require.NoError(t, merger.Close()) +} + +func TestArrayMerger_finalEmptyArray(t *testing.T) { + merger := NewArrayMerger() + w := &bytes.Buffer{} + + r1 := bytes.NewBufferString(`["a","b"]`) + p1 := merger.NewPage(r1, false) + n, err := io.Copy(w, p1) + require.NoError(t, err) + assert.Equal(t, int64(8), n) + assert.NoError(t, p1.Close()) + assert.Equal(t, `["a","b"`, w.String()) + + r2 := bytes.NewBufferString(`[]`) + p2 := merger.NewPage(r2, true) + n, err = io.Copy(w, p2) + require.NoError(t, err) + assert.Equal(t, int64(2), n) + assert.NoError(t, p2.Close()) + assert.Equal(t, `["a","b" ]`, w.String()) + + require.NoError(t, merger.Close()) +} + +func TestArrayMerger_multiplePages(t *testing.T) { + merger := NewArrayMerger() + w := &bytes.Buffer{} + + r1 := bytes.NewBufferString(`["a","b"]`) + p1 := merger.NewPage(r1, false) + n, err := io.Copy(w, p1) + require.NoError(t, err) + assert.Equal(t, int64(8), n) + assert.NoError(t, p1.Close()) + assert.Equal(t, `["a","b"`, w.String()) + + r2 := bytes.NewBufferString(`["c","d"]`) + p2 := merger.NewPage(r2, true) + n, err = io.Copy(w, p2) + require.NoError(t, err) + assert.Equal(t, int64(9), n) + assert.NoError(t, p2.Close()) + assert.Equal(t, `["a","b","c","d"]`, w.String()) + + require.NoError(t, merger.Close()) +} diff --git a/pkg/jsonmerge/merge.go b/pkg/jsonmerge/merge.go new file mode 100644 index 0000000..a40e4ee --- /dev/null +++ b/pkg/jsonmerge/merge.go @@ -0,0 +1,12 @@ +// jsonmerge implements readers to merge JSON arrays or objects. +package jsonmerge + +import ( + "io" +) + +// Merger is implemented to merge JSON arrays or objects. +type Merger interface { + NewPage(r io.Reader, isLastPage bool) io.ReadCloser + Close() error +} diff --git a/pkg/jsonmerge/object.go b/pkg/jsonmerge/object.go new file mode 100644 index 0000000..59ea8c9 --- /dev/null +++ b/pkg/jsonmerge/object.go @@ -0,0 +1,73 @@ +package jsonmerge + +import ( + "bytes" + "encoding/json" + "io" + + "dario.cat/mergo" +) + +type objectMerger struct { + io.Writer + dst map[string]interface{} +} + +// NewObjectMerger creates a Merger for JSON objects. +func NewObjectMerger(w io.Writer) Merger { + return &objectMerger{ + Writer: w, + dst: make(map[string]interface{}), + } +} + +func (merger *objectMerger) NewPage(r io.Reader, isLastPage bool) io.ReadCloser { + return &objectMergerPage{ + merger: merger, + Reader: r, + } +} + +func (merger *objectMerger) Close() error { + // Marshal to JSON and write to output. + buf, err := json.Marshal(merger.dst) + if err != nil { + return err + } + + _, err = merger.Writer.Write(buf) + return err +} + +type objectMergerPage struct { + merger *objectMerger + + io.Reader + buffer bytes.Buffer +} + +func (page *objectMergerPage) Read(p []byte) (int, error) { + // Read into a temporary buffer to be merged and written later. + p = make([]byte, len(p)) + n, err := page.Reader.Read(p) + if err != nil { + return 0, err + } + if n == 0 { + return 0, io.EOF + } + + _, err = page.buffer.Write(p[:n]) + return 0, err +} + +func (page *objectMergerPage) Close() error { + var src map[string]interface{} + + err := json.Unmarshal(page.buffer.Bytes(), &src) + if err != nil { + return err + } + + return mergo.Merge(&page.merger.dst, src, mergo.WithAppendSlice, mergo.WithOverride) +} diff --git a/pkg/jsonmerge/object_test.go b/pkg/jsonmerge/object_test.go new file mode 100644 index 0000000..a05cb65 --- /dev/null +++ b/pkg/jsonmerge/object_test.go @@ -0,0 +1,74 @@ +package jsonmerge + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestObjectMerger_singleEmptyObject(t *testing.T) { + w := &bytes.Buffer{} + merger := NewObjectMerger(w) + + r1 := bytes.NewBufferString(`{}`) + p1 := merger.NewPage(r1, true) + n, err := io.Copy(w, p1) + require.NoError(t, err) + assert.Equal(t, int64(0), n) + assert.NoError(t, p1.Close()) + assert.Equal(t, ``, w.String()) + + require.NoError(t, merger.Close()) + assert.JSONEq(t, `{}`, w.String()) +} + +func TestObjectMerger_finalEmptyObject(t *testing.T) { + w := &bytes.Buffer{} + merger := NewObjectMerger(w) + + r1 := bytes.NewBufferString(`{"a":1,"b":2}`) + p1 := merger.NewPage(r1, false) + n, err := io.Copy(w, p1) + require.NoError(t, err) + assert.Equal(t, int64(0), n) + assert.NoError(t, p1.Close()) + assert.Equal(t, ``, w.String()) + + r2 := bytes.NewBufferString(`{}`) + p2 := merger.NewPage(r2, true) + n, err = io.Copy(w, p2) + require.NoError(t, err) + assert.Equal(t, int64(0), n) + assert.NoError(t, p2.Close()) + assert.Equal(t, ``, w.String()) + + require.NoError(t, merger.Close()) + assert.JSONEq(t, `{"a":1,"b":2}`, w.String()) +} + +func TestObjectMerger_multiplePages(t *testing.T) { + w := &bytes.Buffer{} + merger := NewObjectMerger(w) + + r1 := bytes.NewBufferString(`{"a":1,"b":2,"arr":["a","b"]}`) + p1 := merger.NewPage(r1, false) + n, err := io.Copy(w, p1) + require.NoError(t, err) + assert.Equal(t, int64(0), n) + assert.NoError(t, p1.Close()) + assert.Equal(t, ``, w.String()) + + r2 := bytes.NewBufferString(`{"b":3,"c":{"d":4},"arr":["c","d"]}`) + p2 := merger.NewPage(r2, true) + n, err = io.Copy(w, p2) + require.NoError(t, err) + assert.Equal(t, int64(0), n) + assert.NoError(t, p2.Close()) + assert.Equal(t, ``, w.String()) + + require.NoError(t, merger.Close()) + assert.JSONEq(t, `{"a":1,"b":3,"c":{"d":4},"arr":["a","b","c","d"]}`, w.String()) +}