Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support merging JSON arrays, objects
Partly resolves cli/cli#1268 and replaces cli/cli#5652
- Loading branch information
Showing
7 changed files
with
309 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
package jsonmerge | ||
|
||
import "io" | ||
|
||
type arrayMerger struct { | ||
isFirstPage bool | ||
} | ||
|
||
// NewArrayMerger creates a Merger for JSON arrays. | ||
func NewArrayMerger() Merger { | ||
return &arrayMerger{ | ||
isFirstPage: true, | ||
} | ||
} | ||
|
||
func (merger *arrayMerger) NewPage(r io.Reader, isLastPage bool) io.ReadCloser { | ||
return &arrayMergerPage{ | ||
merger: merger, | ||
Reader: r, | ||
isLastPage: isLastPage, | ||
} | ||
} | ||
|
||
func (m *arrayMerger) Close() error { | ||
// arrayMerger merges when reading, so any output was already written | ||
// and there's nothing to do on Close. | ||
return nil | ||
} | ||
|
||
type arrayMergerPage struct { | ||
merger *arrayMerger | ||
|
||
io.Reader | ||
isLastPage bool | ||
|
||
isSubsequentRead bool | ||
cachedByte byte | ||
} | ||
|
||
func (page *arrayMergerPage) Read(p []byte) (int, error) { | ||
var n int | ||
var err error | ||
|
||
if page.cachedByte != 0 && len(p) > 0 { | ||
p[0] = page.cachedByte | ||
n, err = page.Reader.Read(p[1:]) | ||
n += 1 | ||
page.cachedByte = 0 | ||
} else { | ||
n, err = page.Reader.Read(p) | ||
} | ||
|
||
if !page.isSubsequentRead && !page.merger.isFirstPage && n > 0 && p[0] == '[' { | ||
if n > 1 && p[1] == ']' { | ||
// Empty array case. | ||
p[0] = ' ' | ||
} else { | ||
// Avoid starting a new array and continue with a comma instead. | ||
p[0] = ',' | ||
} | ||
} | ||
|
||
if !page.isLastPage && n > 0 && p[n-1] == ']' { | ||
// Avoid closing off an array in case we determine we are at EOF. | ||
page.cachedByte = p[n-1] | ||
n -= 1 | ||
} | ||
|
||
page.isSubsequentRead = true | ||
return n, err | ||
} | ||
|
||
func (page *arrayMergerPage) Close() error { | ||
page.merger.isFirstPage = false | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
package jsonmerge | ||
|
||
import ( | ||
"bytes" | ||
"io" | ||
"testing" | ||
|
||
"github.com/stretchr/testify/assert" | ||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
func TestArrayMerger_singleEmptyArray(t *testing.T) { | ||
merger := NewArrayMerger() | ||
w := &bytes.Buffer{} | ||
|
||
r1 := bytes.NewBufferString(`[]`) | ||
p1 := merger.NewPage(r1, true) | ||
n, err := io.Copy(w, p1) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(2), n) | ||
assert.NoError(t, p1.Close()) | ||
assert.Equal(t, `[]`, w.String()) | ||
|
||
require.NoError(t, merger.Close()) | ||
} | ||
|
||
func TestArrayMerger_finalEmptyArray(t *testing.T) { | ||
merger := NewArrayMerger() | ||
w := &bytes.Buffer{} | ||
|
||
r1 := bytes.NewBufferString(`["a","b"]`) | ||
p1 := merger.NewPage(r1, false) | ||
n, err := io.Copy(w, p1) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(8), n) | ||
assert.NoError(t, p1.Close()) | ||
assert.Equal(t, `["a","b"`, w.String()) | ||
|
||
r2 := bytes.NewBufferString(`[]`) | ||
p2 := merger.NewPage(r2, true) | ||
n, err = io.Copy(w, p2) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(2), n) | ||
assert.NoError(t, p2.Close()) | ||
assert.Equal(t, `["a","b" ]`, w.String()) | ||
|
||
require.NoError(t, merger.Close()) | ||
} | ||
|
||
func TestArrayMerger_multiplePages(t *testing.T) { | ||
merger := NewArrayMerger() | ||
w := &bytes.Buffer{} | ||
|
||
r1 := bytes.NewBufferString(`["a","b"]`) | ||
p1 := merger.NewPage(r1, false) | ||
n, err := io.Copy(w, p1) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(8), n) | ||
assert.NoError(t, p1.Close()) | ||
assert.Equal(t, `["a","b"`, w.String()) | ||
|
||
r2 := bytes.NewBufferString(`["c","d"]`) | ||
p2 := merger.NewPage(r2, true) | ||
n, err = io.Copy(w, p2) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(9), n) | ||
assert.NoError(t, p2.Close()) | ||
assert.Equal(t, `["a","b","c","d"]`, w.String()) | ||
|
||
require.NoError(t, merger.Close()) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
// jsonmerge implements readers to merge JSON arrays or objects. | ||
package jsonmerge | ||
|
||
import ( | ||
"io" | ||
) | ||
|
||
// Merger is implemented to merge JSON arrays or objects. | ||
type Merger interface { | ||
NewPage(r io.Reader, isLastPage bool) io.ReadCloser | ||
Close() error | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,73 @@ | ||
package jsonmerge | ||
|
||
import ( | ||
"bytes" | ||
"encoding/json" | ||
"io" | ||
|
||
"dario.cat/mergo" | ||
) | ||
|
||
type objectMerger struct { | ||
io.Writer | ||
dst map[string]interface{} | ||
} | ||
|
||
// NewObjectMerger creates a Merger for JSON objects. | ||
func NewObjectMerger(w io.Writer) Merger { | ||
return &objectMerger{ | ||
Writer: w, | ||
dst: make(map[string]interface{}), | ||
} | ||
} | ||
|
||
func (merger *objectMerger) NewPage(r io.Reader, isLastPage bool) io.ReadCloser { | ||
return &objectMergerPage{ | ||
merger: merger, | ||
Reader: r, | ||
} | ||
} | ||
|
||
func (merger *objectMerger) Close() error { | ||
// Marshal to JSON and write to output. | ||
buf, err := json.Marshal(merger.dst) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
_, err = merger.Writer.Write(buf) | ||
return err | ||
} | ||
|
||
type objectMergerPage struct { | ||
merger *objectMerger | ||
|
||
io.Reader | ||
buffer bytes.Buffer | ||
} | ||
|
||
func (page *objectMergerPage) Read(p []byte) (int, error) { | ||
// Read into a temporary buffer to be merged and written later. | ||
p = make([]byte, len(p)) | ||
n, err := page.Reader.Read(p) | ||
if err != nil { | ||
return 0, err | ||
} | ||
if n == 0 { | ||
return 0, io.EOF | ||
} | ||
|
||
_, err = page.buffer.Write(p[:n]) | ||
return 0, err | ||
} | ||
|
||
func (page *objectMergerPage) Close() error { | ||
var src map[string]interface{} | ||
|
||
err := json.Unmarshal(page.buffer.Bytes(), &src) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
return mergo.Merge(&page.merger.dst, src, mergo.WithAppendSlice, mergo.WithOverride) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
package jsonmerge | ||
|
||
import ( | ||
"bytes" | ||
"io" | ||
"testing" | ||
|
||
"github.com/stretchr/testify/assert" | ||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
func TestObjectMerger_singleEmptyObject(t *testing.T) { | ||
w := &bytes.Buffer{} | ||
merger := NewObjectMerger(w) | ||
|
||
r1 := bytes.NewBufferString(`{}`) | ||
p1 := merger.NewPage(r1, true) | ||
n, err := io.Copy(w, p1) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(0), n) | ||
assert.NoError(t, p1.Close()) | ||
assert.Equal(t, ``, w.String()) | ||
|
||
require.NoError(t, merger.Close()) | ||
assert.JSONEq(t, `{}`, w.String()) | ||
} | ||
|
||
func TestObjectMerger_finalEmptyObject(t *testing.T) { | ||
w := &bytes.Buffer{} | ||
merger := NewObjectMerger(w) | ||
|
||
r1 := bytes.NewBufferString(`{"a":1,"b":2}`) | ||
p1 := merger.NewPage(r1, false) | ||
n, err := io.Copy(w, p1) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(0), n) | ||
assert.NoError(t, p1.Close()) | ||
assert.Equal(t, ``, w.String()) | ||
|
||
r2 := bytes.NewBufferString(`{}`) | ||
p2 := merger.NewPage(r2, true) | ||
n, err = io.Copy(w, p2) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(0), n) | ||
assert.NoError(t, p2.Close()) | ||
assert.Equal(t, ``, w.String()) | ||
|
||
require.NoError(t, merger.Close()) | ||
assert.JSONEq(t, `{"a":1,"b":2}`, w.String()) | ||
} | ||
|
||
func TestObjectMerger_multiplePages(t *testing.T) { | ||
w := &bytes.Buffer{} | ||
merger := NewObjectMerger(w) | ||
|
||
r1 := bytes.NewBufferString(`{"a":1,"b":2,"arr":["a","b"]}`) | ||
p1 := merger.NewPage(r1, false) | ||
n, err := io.Copy(w, p1) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(0), n) | ||
assert.NoError(t, p1.Close()) | ||
assert.Equal(t, ``, w.String()) | ||
|
||
r2 := bytes.NewBufferString(`{"b":3,"c":{"d":4},"arr":["c","d"]}`) | ||
p2 := merger.NewPage(r2, true) | ||
n, err = io.Copy(w, p2) | ||
require.NoError(t, err) | ||
assert.Equal(t, int64(0), n) | ||
assert.NoError(t, p2.Close()) | ||
assert.Equal(t, ``, w.String()) | ||
|
||
require.NoError(t, merger.Close()) | ||
assert.JSONEq(t, `{"a":1,"b":3,"c":{"d":4},"arr":["a","b","c","d"]}`, w.String()) | ||
} |