Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

return num bytes written from OCFWriter.Append #185

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
2 changes: 1 addition & 1 deletion fuzz_test.go
Expand Up @@ -408,7 +408,7 @@ func TestCrashers_OCF_e2e(t *testing.T) {
if err != nil {
t.Fatalf("GOT: %v; WANT: %v", err, nil)
}
if err := ocfw.Append(datums); err != nil {
if _, err := ocfw.Append(datums); err != nil {
t.Fatalf("GOT: %v; WANT: %v", err, nil)
}
})
Expand Down
2 changes: 1 addition & 1 deletion ocf_test.go
Expand Up @@ -39,7 +39,7 @@ func testOCFRoundTripWithHeaders(t *testing.T, compressionName string, headers m

valuesToWrite := []int64{13, 42, -12, -1234}

if err = ocfw.Append(valuesToWrite); err != nil {
if _, err = ocfw.Append(valuesToWrite); err != nil {
t.Fatal(err)
}

Expand Down
22 changes: 11 additions & 11 deletions ocf_writer.go
Expand Up @@ -158,30 +158,30 @@ func (ocfw *OCFWriter) quickScanToTail(ior io.Reader) error {
// more data items in the slice than MaxBlockCount allows, the data slice will
// be chunked into multiple blocks, each not having more than MaxBlockCount
// items.
func (ocfw *OCFWriter) Append(data interface{}) error {
func (ocfw *OCFWriter) Append(data interface{}) (int, error) {
arrayValues, err := convertArray(data)
if err != nil {
return err
return 0, err
}

// Chunk data so no block has more than MaxBlockCount items.
for int64(len(arrayValues)) > MaxBlockCount {
if err := ocfw.appendDataIntoBlock(arrayValues[:MaxBlockCount]); err != nil {
return err
if _, err := ocfw.appendDataIntoBlock(arrayValues[:MaxBlockCount]); err != nil {
return 0, err
}
arrayValues = arrayValues[MaxBlockCount:]
}
return ocfw.appendDataIntoBlock(arrayValues)
}

func (ocfw *OCFWriter) appendDataIntoBlock(data []interface{}) error {
func (ocfw *OCFWriter) appendDataIntoBlock(data []interface{}) (int, error) {
var block []byte // working buffer for encoding data values
var err error

// Encode and concatenate each data item into the block
for _, datum := range data {
if block, err = ocfw.header.codec.BinaryFromNative(block, datum); err != nil {
return fmt.Errorf("cannot translate datum to binary: %v; %s", datum, err)
return 0, fmt.Errorf("cannot translate datum to binary: %v; %s", datum, err)
}
}

Expand All @@ -196,10 +196,10 @@ func (ocfw *OCFWriter) appendDataIntoBlock(data []interface{}) error {
cw, _ := flate.NewWriter(bb, flate.DefaultCompression)
// writing bytes to cw will compress bytes and send to bb.
if _, err := cw.Write(block); err != nil {
return err
return 0, err
}
if err := cw.Close(); err != nil {
return err
return 0, err
}
block = bb.Bytes()

Expand All @@ -213,7 +213,7 @@ func (ocfw *OCFWriter) appendDataIntoBlock(data []interface{}) error {
block = compressed

default:
return fmt.Errorf("should not get here: cannot compress block using unrecognized compression: %d", ocfw.header.compressionID)
return 0, fmt.Errorf("should not get here: cannot compress block using unrecognized compression: %d", ocfw.header.compressionID)

}

Expand All @@ -224,8 +224,8 @@ func (ocfw *OCFWriter) appendDataIntoBlock(data []interface{}) error {
buf = append(buf, block...) // serialized objects
buf = append(buf, ocfw.header.syncMarker[:]...) // sync marker

_, err = ocfw.iow.Write(buf)
return err
n, err := ocfw.iow.Write(buf)
return n, err
}

// Codec returns the codec used by OCFWriter. This function provided because
Expand Down
14 changes: 11 additions & 3 deletions ocf_writer_test.go
Expand Up @@ -164,7 +164,7 @@ func TestOCFWriterAppendWhenCannotWrite(t *testing.T) {
t.Fatal(err)
}

err = ocfw.Append([]interface{}{13, 42})
_, err = ocfw.Append([]interface{}{13, 42})
ensureError(t, err, testPathname)
}

Expand All @@ -186,7 +186,8 @@ func TestOCFWriterAppendSomeItemsToNothing(t *testing.T) {
t.Fatal(err)
}

if err = ocfw.Append([]interface{}{13, 42}); err != nil {
bytesWritten, err := ocfw.Append([]interface{}{13, 42})
if err != nil {
t.Fatal(err)
}

Expand Down Expand Up @@ -227,6 +228,9 @@ func TestOCFWriterAppendSomeItemsToNothing(t *testing.T) {
if actual, expected := values[1], int64(42); actual != expected {
t.Errorf("GOT: %v; WANT: %v", actual, expected)
}
if actual, expected := bytesWritten, 20; actual != expected {
t.Errorf("GOT: %v; WANT: %v", actual, expected)
}
}

func TestOCFWriterAppendSomeItemsToSomeItems(t *testing.T) {
Expand All @@ -247,7 +251,8 @@ func TestOCFWriterAppendSomeItemsToSomeItems(t *testing.T) {
t.Fatal(err)
}

if err = ocfw.Append([]interface{}{-10, -100}); err != nil {
bytesWritten, err := ocfw.Append([]interface{}{-10, -100})
if err != nil {
t.Fatal(err)
}

Expand Down Expand Up @@ -294,4 +299,7 @@ func TestOCFWriterAppendSomeItemsToSomeItems(t *testing.T) {
if actual, expected := values[3], int64(-100); actual != expected {
t.Errorf("GOT: %v; WANT: %v", actual, expected)
}
if actual, expected := bytesWritten, 21; actual != expected {
t.Errorf("GOT: %v; WANT: %v", actual, expected)
}
}