Use provided sha256 hasher (#1749)

api.go

@@ -845,7 +845,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
 		// Additionally, we also look if the initialized client is secure,
 		// if yes then we don't need to perform streaming signature.
 		req = signer.StreamingSignV4(req, accessKeyID,
-			secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC())
+			secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
 	default:
 		// Set sha256 sum for signature calculation only with signature version '4'.
 		shaHeader := unsignedPayload

core_test.go

@@ -41,6 +41,9 @@ const (
 
 // Tests for Core GetObject() function.
 func TestGetObjectCore(t *testing.T) {
+	if os.Getenv(serverEndpoint) == "" {
+		t.Skip("SERVER_ENDPOINT not set")
+	}
 	if testing.Short() {
 		t.Skip("skipping functional tests for the short runs")
 	}
@@ -238,6 +241,9 @@ func TestGetObjectCore(t *testing.T) {
 // Tests GetObject to return Content-Encoding properly set
 // and overrides any auto decoding.
 func TestGetObjectContentEncoding(t *testing.T) {
+	if os.Getenv(serverEndpoint) == "" {
+		t.Skip("SERVER_ENDPOINT not set")
+	}
 	if testing.Short() {
 		t.Skip("skipping functional tests for the short runs")
 	}
@@ -311,6 +317,9 @@ func TestGetObjectContentEncoding(t *testing.T) {
 
 // Tests get bucket policy core API.
 func TestGetBucketPolicy(t *testing.T) {
+	if os.Getenv(serverEndpoint) == "" {
+		t.Skip("SERVER_ENDPOINT not set")
+	}
 	if testing.Short() {
 		t.Skip("skipping functional tests for short runs")
 	}
@@ -374,6 +383,9 @@ func TestGetBucketPolicy(t *testing.T) {
 
 // Tests Core CopyObject API implementation.
 func TestCoreCopyObject(t *testing.T) {
+	if os.Getenv(serverEndpoint) == "" {
+		t.Skip("SERVER_ENDPOINT not set")
+	}
 	if testing.Short() {
 		t.Skip("skipping functional tests for short runs")
 	}
@@ -497,6 +509,9 @@ func TestCoreCopyObject(t *testing.T) {
 
 // Test Core CopyObjectPart implementation
 func TestCoreCopyObjectPart(t *testing.T) {
+	if os.Getenv(serverEndpoint) == "" {
+		t.Skip("SERVER_ENDPOINT not set")
+	}
 	if testing.Short() {
 		t.Skip("skipping functional tests for short runs")
 	}
@@ -650,6 +665,9 @@ func TestCoreCopyObjectPart(t *testing.T) {
 
 // Test Core PutObject.
 func TestCorePutObject(t *testing.T) {
+	if os.Getenv(serverEndpoint) == "" {
+		t.Skip("SERVER_ENDPOINT not set")
+	}
 	if testing.Short() {
 		t.Skip("skipping functional tests for short runs")
 	}
@@ -744,6 +762,9 @@ func TestCorePutObject(t *testing.T) {
 }
 
 func TestCoreGetObjectMetadata(t *testing.T) {
+	if os.Getenv(serverEndpoint) == "" {
+		t.Skip("SERVER_ENDPOINT not set")
+	}
 	if testing.Short() {
 		t.Skip("skipping functional tests for the short runs")
 	}
@@ -801,6 +822,9 @@ func TestCoreGetObjectMetadata(t *testing.T) {
 }
 
 func TestCoreMultipartUpload(t *testing.T) {
+	if os.Getenv(serverEndpoint) == "" {
+		t.Skip("SERVER_ENDPOINT not set")
+	}
 	if testing.Short() {
 		t.Skip("skipping functional tests for the short runs")
 	}

pkg/credentials/file_test.go

@@ -20,10 +20,14 @@ package credentials
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 )
 
 func TestFileAWS(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("\"/bin/cat\": file does not exist")
+	}
 	os.Clearenv()
 
 	creds := NewFileAWSCredentials("credentials.sample", "")

pkg/signer/request-signature-streaming.go

@@ -26,6 +26,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	md5simd "github.com/minio/md5-simd"
 )
 
 // Reference for constants used below -
@@ -90,28 +92,28 @@ func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 {
 
 // buildChunkStringToSign - returns the string to sign given chunk data
 // and previous signature.
-func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string {
+func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
 	stringToSignParts := []string{
 		streamingPayloadHdr,
 		t.Format(iso8601DateFormat),
 		getScope(region, t, ServiceTypeS3),
 		previousSig,
 		emptySHA256,
-		hex.EncodeToString(sum256(chunkData)),
+		chunkChecksum,
 	}
 
 	return strings.Join(stringToSignParts, "\n")
 }
 
 // buildTrailerChunkStringToSign - returns the string to sign given chunk data
 // and previous signature.
-func buildTrailerChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string {
+func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
 	stringToSignParts := []string{
 		streamingTrailerHdr,
 		t.Format(iso8601DateFormat),
 		getScope(region, t, ServiceTypeS3),
 		previousSig,
-		hex.EncodeToString(sum256(chunkData)),
+		chunkChecksum,
 	}
 
 	return strings.Join(stringToSignParts, "\n")
@@ -148,21 +150,21 @@ func buildChunkHeader(chunkLen int64, signature string) []byte {
 }
 
 // buildChunkSignature - returns chunk signature for a given chunk and previous signature.
-func buildChunkSignature(chunkData []byte, reqTime time.Time, region,
+func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region,
 	previousSignature, secretAccessKey string,
 ) string {
 	chunkStringToSign := buildChunkStringToSign(reqTime, region,
-		previousSignature, chunkData)
+		previousSignature, chunkCheckSum)
 	signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
 	return getSignature(signingKey, chunkStringToSign)
 }
 
 // buildChunkSignature - returns chunk signature for a given chunk and previous signature.
-func buildTrailerChunkSignature(chunkData []byte, reqTime time.Time, region,
+func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region,
 	previousSignature, secretAccessKey string,
 ) string {
 	chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region,
-		previousSignature, chunkData)
+		previousSignature, chunkChecksum)
 	signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
 	return getSignature(signingKey, chunkStringToSign)
 }
@@ -202,12 +204,17 @@ type StreamingReader struct {
 	totalChunks     int
 	lastChunkSize   int
 	trailer         http.Header
+	sh256           md5simd.Hasher
 }
 
 // signChunk - signs a chunk read from s.baseReader of chunkLen size.
 func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) {
 	// Compute chunk signature for next header
-	signature := buildChunkSignature(s.chunkBuf[:chunkLen], s.reqTime,
+	s.sh256.Reset()
+	s.sh256.Write(s.chunkBuf[:chunkLen])
+	chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil))
+
+	signature := buildChunkSignature(chunckChecksum, s.reqTime,
 		s.region, s.prevSignature, s.secretAccessKey)
 
 	// For next chunk signature computation
@@ -239,8 +246,11 @@ func (s *StreamingReader) addSignedTrailer(h http.Header) {
 		s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...)
 	}
 
+	s.sh256.Reset()
+	s.sh256.Write(s.chunkBuf)
+	chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil))
 	// Compute chunk signature
-	signature := buildTrailerChunkSignature(s.chunkBuf, s.reqTime,
+	signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime,
 		s.region, s.prevSignature, s.secretAccessKey)
 
 	// For next chunk signature computation
@@ -273,7 +283,7 @@ func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) {
 // StreamingSignV4 - provides chunked upload signatureV4 support by
 // implementing io.Reader.
 func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken,
-	region string, dataLen int64, reqTime time.Time,
+	region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher,
 ) *http.Request {
 	// Set headers needed for streaming signature.
 	prepareStreamingRequest(req, sessionToken, dataLen, reqTime)
@@ -294,6 +304,7 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok
 		chunkNum:        1,
 		totalChunks:     int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
 		lastChunkSize:   int(dataLen % payloadChunkSize),
+		sh256:           sh256,
 	}
 	if len(req.Trailer) > 0 {
 		stReader.trailer = req.Trailer
@@ -384,5 +395,9 @@ func (s *StreamingReader) Read(buf []byte) (int, error) {
 
 // Close - this method makes underlying io.ReadCloser's Close method available.
 func (s *StreamingReader) Close() error {
+	if s.sh256 != nil {
+		s.sh256.Close()
+		s.sh256 = nil
+	}
 	return s.baseReadCloser.Close()
 }

pkg/signer/request-signature-streaming_test.go

@@ -19,13 +19,37 @@ package signer
 
 import (
 	"bytes"
+	fipssha256 "crypto/sha256"
 	"encoding/hex"
+	"hash"
 	"io"
 	"net/http"
 	"testing"
 	"time"
+
+	md5simd "github.com/minio/md5-simd"
+	"github.com/minio/sha256-simd"
 )
 
+// hashWrapper implements the md5simd.Hasher interface.
+type hashWrapper struct {
+	hash.Hash
+}
+
+func newSHA256Hasher() md5simd.Hasher {
+	return &hashWrapper{Hash: fipssha256.New()}
+}
+
+func (m *hashWrapper) Close() {
+	m.Hash = nil
+}
+
+func sum256hex(data []byte) string {
+	hash := sha256.New()
+	hash.Write(data)
+	return hex.EncodeToString(hash.Sum(nil))
+}
+
 func TestGetSeedSignature(t *testing.T) {
 	accessKeyID := "AKIAIOSFODNN7EXAMPLE"
 	secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
@@ -42,7 +66,7 @@ func TestGetSeedSignature(t *testing.T) {
 		t.Fatalf("Failed to parse time - %v", err)
 	}
 
-	req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime)
+	req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime, newSHA256Hasher())
 	actualSeedSignature := req.Body.(*StreamingReader).seedSignature
 
 	expectedSeedSignature := "38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079"
@@ -58,7 +82,8 @@ func TestChunkSignature(t *testing.T) {
 	location := "us-east-1"
 	secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
 	expectedSignature := "ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648"
-	actualSignature := buildChunkSignature(chunkData, reqTime, location, previousSignature, secretAccessKeyID)
+	chunkCheckSum := sum256hex(chunkData)
+	actualSignature := buildChunkSignature(chunkCheckSum, reqTime, location, previousSignature, secretAccessKeyID)
 	if actualSignature != expectedSignature {
 		t.Errorf("Expected %s but received %s", expectedSignature, actualSignature)
 	}
@@ -72,7 +97,8 @@ func TestTrailerChunkSignature(t *testing.T) {
 	location := "us-east-1"
 	secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
 	expectedSignature := "41e14ac611e27a8bb3d66c3bad6856f209297767d5dd4fc87d8fa9e422e03faf"
-	actualSignature := buildTrailerChunkSignature(chunkData, reqTime, location, previousSignature, secretAccessKeyID)
+	chunkCheckSum := sum256hex(chunkData)
+	actualSignature := buildTrailerChunkSignature(chunkCheckSum, reqTime, location, previousSignature, secretAccessKeyID)
 	if actualSignature != expectedSignature {
 		t.Errorf("Expected %s but received %s", expectedSignature, actualSignature)
 	}
@@ -90,7 +116,7 @@ func TestSetStreamingAuthorization(t *testing.T) {
 
 	dataLen := int64(65 * 1024)
 	reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z")
-	req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime)
+	req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher())
 
 	expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079"
 
@@ -117,7 +143,7 @@ func TestSetStreamingAuthorizationTrailer(t *testing.T) {
 
 	dataLen := int64(65 * 1024)
 	reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z")
-	req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime)
+	req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher())
 
 	// (order of signed headers is different)
 	expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class;x-amz-trailer,Signature=106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e"
@@ -145,7 +171,7 @@ func TestStreamingReader(t *testing.T) {
 
 	baseReader := io.NopCloser(bytes.NewReader(bytes.Repeat([]byte("a"), 65*1024)))
 	req.Body = baseReader
-	req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime)
+	req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher())
 
 	b, err := io.ReadAll(req.Body)
 	if err != nil {