From 9654d96ff327cee0037895ab0b454217b673d31e Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 7 Jun 2022 20:11:44 +0200 Subject: [PATCH] Update object Put to avoid loosing last chunk The old code might miss the last object if `Reader` return both a value and `EOF` Extract from `Reader` doc : > When Read encounters an error or end-of-file condition after successfully reading n > 0 bytes, it returns the number of bytes read. It may return the (non-nil) error from the same call or return the error (and n == 0) from a subsequent call. An instance of this general case is that a Reader returning a non-zero number of bytes at the end of the input stream may return either err == EOF or err == nil. The next Read should return 0, EOF. > > Callers should always process the n > 0 bytes returned before considering the error err. Doing so correctly handles I/O errors that happen after reading some bytes and also both of the allowed EOF behaviors. --- object.go | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/object.go b/object.go index f2646d885..d2124bb47 100644 --- a/object.go +++ b/object.go @@ -372,37 +372,43 @@ func (obs *obs) Put(meta *ObjectMeta, r io.Reader, opts ...ObjectOpt) (*ObjectIn // Actual read. // TODO(dlc) - Deadline? - n, err := r.Read(chunk) + n, readErr := r.Read(chunk) + + // Handle all non EOF errors + if readErr != nil && readErr != io.EOF { + purgePartial() + return nil, readErr + } + + // Add chunk only if we received data + if n > 0 { + // Chunk processing. + m.Data = chunk[:n] + h.Write(m.Data) + + // Send msg itself. + if _, err := js.PublishMsgAsync(m); err != nil { + purgePartial() + return nil, err + } + if err := getErr(); err != nil { + purgePartial() + return nil, err + } + // Update totals. + sent++ + total += uint64(n) + } // EOF Processing. - if err == io.EOF { + if readErr == io.EOF { // Finalize sha. sha := h.Sum(nil) // Place meta info. info.Size, info.Chunks = uint64(total), uint32(sent) info.Digest = fmt.Sprintf(objDigestTmpl, base64.URLEncoding.EncodeToString(sha[:])) break - } else if err != nil { - purgePartial() - return nil, err - } - - // Chunk processing. - m.Data = chunk[:n] - h.Write(m.Data) - - // Send msg itself. - if _, err := js.PublishMsgAsync(m); err != nil { - purgePartial() - return nil, err - } - if err := getErr(); err != nil { - purgePartial() - return nil, err } - // Update totals. - sent++ - total += uint64(n) } // Publish the metadata.