CHANGELOG.md

@@ -105,7 +105,7 @@ Version 2.x is JDK8 LTS bytecode compatible, with Docker and JUnit / direct Java
     * This will update all Spring Boot dependencies as well, including Spring Framework 6.x
   * Bump java version from 8 to 17
 
-## 2.14.0 - PLANNED
+## 2.15.0 - PLANNED
 2.x is JDK8 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
 
 * Features and fixes
@@ -116,6 +116,20 @@ Version 2.x is JDK8 LTS bytecode compatible, with Docker and JUnit / direct Java
 * Version updates
   * TBD
 
+## 2.14.0
+2.x is JDK8 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
+
+* Features and fixes
+  * Storage and retrieval headers like Content-Disposition (fixes #1163)
+* Refactorings
+  * Test that persisted file length is equal to uploaded file length
+    * This does not work if checksum validation is enabled (see #1123)
+* Version updates
+  * Bump spring-boot.version from 2.7.12 to 2.7.13
+  * Bump aws-v2.version from 2.20.86 to 2.20.92
+  * Bump aws-java-sdk-s3 from 1.12.488 to 1.12.494
+  * Bump maven-clean-plugin from 3.2.0 to 3.3.1
+
 ## 2.13.1
 2.x is JDK8 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
 

build-config/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>2.13.1</version>
+    <version>2.14.0</version>
   </parent>
 
   <artifactId>s3mock-build-config</artifactId>

docker/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>2.13.1</version>
+    <version>2.14.0</version>
   </parent>
 
   <artifactId>s3mock-docker</artifactId>

integration-tests/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>2.13.1</version>
+    <version>2.14.0</version>
   </parent>
 
   <artifactId>s3mock-integration-tests</artifactId>

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CrtAsyncV2IT.kt

@@ -16,9 +16,9 @@
 
 package com.adobe.testing.s3mock.its
 
+import com.adobe.testing.s3mock.util.DigestUtil
 import org.apache.commons.codec.digest.DigestUtils
 import org.apache.commons.lang3.ArrayUtils
-import org.assertj.core.api.Assertions
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.TestInfo
@@ -33,9 +33,36 @@ import software.amazon.awssdk.services.s3.model.GetObjectRequest
 import software.amazon.awssdk.services.s3.model.PutObjectRequest
 import software.amazon.awssdk.services.s3.model.UploadPartRequest
 import java.io.File
+import java.io.FileInputStream
+import java.io.InputStream
 
 internal class CrtAsyncV2IT : S3TestBase() {
 
+  @Test
+  @S3VerifiedTodo
+  fun testPutObject_etagCreation(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val uploadFileIs: InputStream = FileInputStream(uploadFile)
+    val expectedEtag = "\"${DigestUtil.hexDigest(uploadFileIs)}\""
+
+    val bucketName = randomName
+    val bucketFuture =
+      autoS3CrtAsyncClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build())
+    bucketFuture.join()
+
+    val putObjectFuture = autoS3CrtAsyncClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .build(),
+      AsyncRequestBody.fromFile(uploadFile)
+    )
+    val putObjectResponse = putObjectFuture.join()
+    val eTag = putObjectResponse.eTag()
+    assertThat(eTag).isNotBlank
+    assertThat(eTag).isEqualTo(expectedEtag)
+  }
+
   @Test
   @S3VerifiedTodo
   fun testPutGetObject_successWithMatchingEtag(testInfo: TestInfo) {
@@ -62,7 +89,8 @@ internal class CrtAsyncV2IT : S3TestBase() {
       AsyncResponseTransformer.toBytes()
     )
     val getObjectResponse = getObjectResponseCompletableFuture.join()
-    Assertions.assertThat(getObjectResponse.response().eTag()).isEqualTo(eTag)
+    assertThat(getObjectResponse.response().eTag()).isEqualTo(eTag)
+    assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
   }
 
   @Test

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV2IT.kt

@@ -19,9 +19,9 @@ package com.adobe.testing.s3mock.its
 import com.adobe.testing.s3mock.util.DigestUtil
 import org.assertj.core.api.Assertions
 import org.assertj.core.api.Assertions.assertThat
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.TestInfo
+import org.springframework.http.ContentDisposition
 import software.amazon.awssdk.core.sync.RequestBody
 import software.amazon.awssdk.services.s3.model.GetObjectRequest
 import software.amazon.awssdk.services.s3.model.HeadObjectRequest
@@ -30,6 +30,8 @@ import software.amazon.awssdk.services.s3.model.S3Exception
 import java.io.File
 import java.io.FileInputStream
 import java.io.InputStream
+import java.time.Instant
+import java.time.temporal.ChronoUnit
 
 internal class GetPutDeleteObjectV2IT : S3TestBase() {
 
@@ -66,6 +68,61 @@ internal class GetPutDeleteObjectV2IT : S3TestBase() {
     assertThat(object2.response().eTag()).isEqualTo(object2Again.response().eTag())
   }
 
+  @Test
+  fun testPutGetHeadObject_storeHeaders(testInfo: TestInfo) {
+    val bucket = givenRandomBucketV2()
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val contentDisposition = ContentDisposition.formData()
+      .name("file")
+      .filename("sampleFile.txt")
+      .build()
+      .toString()
+    val expires = Instant.now()
+    val encoding = "SomeEncoding"
+    val contentLanguage = "SomeLanguage"
+    val cacheControl = "SomeCacheControl"
+
+    s3ClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucket)
+        .key(UPLOAD_FILE_NAME)
+        .contentDisposition(contentDisposition)
+        .contentEncoding(encoding)
+        .expires(expires)
+        .contentLanguage(contentLanguage)
+        .cacheControl(cacheControl)
+        .build(),
+      RequestBody.fromFile(uploadFile))
+
+    val getObjectResponseResponse = getObjectV2(bucket, UPLOAD_FILE_NAME)
+
+    assertThat(getObjectResponseResponse.response().contentDisposition())
+      .isEqualTo(contentDisposition)
+    assertThat(getObjectResponseResponse.response().contentEncoding())
+      .isEqualTo(encoding)
+    // time in second precision, see
+    // https://www.rfc-editor.org/rfc/rfc7234#section-5.3
+    // https://www.rfc-editor.org/rfc/rfc7231#section-7.1.1.1
+    assertThat(getObjectResponseResponse.response().expires())
+      .isEqualTo(expires.truncatedTo(ChronoUnit.SECONDS))
+    assertThat(getObjectResponseResponse.response().contentLanguage())
+      .isEqualTo(contentLanguage)
+    assertThat(getObjectResponseResponse.response().cacheControl())
+      .isEqualTo(cacheControl)
+
+    val headObjectResponse = s3ClientV2.headObject(
+      HeadObjectRequest.builder()
+        .bucket(bucket)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    assertThat(headObjectResponse.contentDisposition()).isEqualTo(contentDisposition)
+    assertThat(headObjectResponse.contentEncoding()).isEqualTo(encoding)
+    assertThat(headObjectResponse.expires()).isEqualTo(expires.truncatedTo(ChronoUnit.SECONDS))
+    assertThat(headObjectResponse.contentLanguage()).isEqualTo(contentLanguage)
+    assertThat(headObjectResponse.cacheControl()).isEqualTo(cacheControl)
+  }
+
   @Test
   @S3VerifiedSuccess(year = 2022)
   fun testGetObject_successWithMatchingEtag(testInfo: TestInfo) {
@@ -86,6 +143,24 @@ internal class GetPutDeleteObjectV2IT : S3TestBase() {
     assertThat(responseInputStream.response().eTag()).isEqualTo(eTag)
   }
 
+  @Test
+  @S3VerifiedTodo
+  fun testGetObject_successWithSameLength(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val uploadFileIs: InputStream = FileInputStream(uploadFile)
+    val matchingEtag = "\"${DigestUtil.hexDigest(uploadFileIs)}\""
+
+    val (bucketName, _) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME)
+    val responseInputStream = s3ClientV2.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .ifMatch(matchingEtag)
+        .build()
+    )
+    assertThat(responseInputStream.response().contentLength()).isEqualTo(uploadFile.length())
+  }
+
   @Test
   @S3VerifiedSuccess(year = 2022)
   fun testGetObject_successWithMatchingWildcardEtag(testInfo: TestInfo) {

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt

@@ -229,6 +229,8 @@ internal abstract class S3TestBase {
       .endpointOverride(URI.create("http://$host:$httpPort"))
       .targetThroughputInGbps(20.0)
       .minimumPartSizeInBytes((8 * MB).toLong())
+      //S3Mock currently does not support checksum validation. See #1123
+      .checksumValidationEnabled(false)
       .build() as S3CrtAsyncClient;
   }
 

pom.xml

@@ -21,7 +21,7 @@
 
   <groupId>com.adobe.testing</groupId>
   <artifactId>s3mock-parent</artifactId>
-  <version>2.13.1</version>
+  <version>2.14.0</version>
   <packaging>pom</packaging>
 
   <name>S3Mock - Parent</name>
@@ -73,7 +73,7 @@
   <scm>
     <connection>scm:git:https://github.com/adobe/S3Mock.git</connection>
     <developerConnection>scm:git:https://github.com/adobe/S3Mock.git</developerConnection>
-    <tag>2.13.1</tag>
+    <tag>2.14.0</tag>
     <url>https://github.com/adobe/S3Mock/tree/main</url>
   </scm>
 
@@ -89,9 +89,9 @@
   </distributionManagement>
 
   <properties>
-    <aws-v2.version>2.20.86</aws-v2.version>
+    <aws-v2.version>2.20.92</aws-v2.version>
 
-    <aws.version>1.12.488</aws.version>
+    <aws.version>1.12.494</aws.version>
     <checkstyle.version>10.12.0</checkstyle.version>
     <commons-codec.version>1.15</commons-codec.version>
     <commons-io.version>2.13.0</commons-io.version>
@@ -105,7 +105,7 @@
     <kotlin.version>1.8.22</kotlin.version>
     <license-maven-plugin-git.version>4.2</license-maven-plugin-git.version>
     <maven-checkstyle-plugin.version>3.3.0</maven-checkstyle-plugin.version>
-    <maven-clean-plugin.version>3.2.0</maven-clean-plugin.version>
+    <maven-clean-plugin.version>3.3.1</maven-clean-plugin.version>
     <maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
     <maven-deploy-plugin.version>3.1.1</maven-deploy-plugin.version>
     <maven-enforcer-plugin.version>3.3.0</maven-enforcer-plugin.version>
@@ -124,7 +124,7 @@
     <!-- Run Docker build by default -->
     <skipDocker>false</skipDocker>
     <sortpom-maven-plugin.version>3.2.1</sortpom-maven-plugin.version>
-    <spring-boot.version>2.7.12</spring-boot.version>
+    <spring-boot.version>2.7.13</spring-boot.version>
     <spring-security-oauth2.version>2.5.2.RELEASE</spring-security-oauth2.version>
     <testcontainers.version>1.18.3</testcontainers.version>
     <testng.version>7.8.0</testng.version>

server/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>2.13.1</version>
+    <version>2.14.0</version>
   </parent>
 
   <artifactId>s3mock</artifactId>

server/src/main/java/com/adobe/testing/s3mock/MultipartController.java

@@ -30,9 +30,9 @@
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.PART_NUMBER;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.UPLOADS;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.UPLOAD_ID;
-import static com.adobe.testing.s3mock.util.HeaderUtil.getUserMetadata;
 import static com.adobe.testing.s3mock.util.HeaderUtil.isV4ChunkedWithSigningEnabled;
-import static org.springframework.http.HttpHeaders.CONTENT_ENCODING;
+import static com.adobe.testing.s3mock.util.HeaderUtil.parseStoreHeaders;
+import static com.adobe.testing.s3mock.util.HeaderUtil.parseUserMetadata;
 import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
 import static org.springframework.http.MediaType.APPLICATION_XML_VALUE;
 
@@ -322,16 +322,16 @@ public ResponseEntity<InitiateMultipartUploadResult> createMultipartUpload(
           value = X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID,
           required = false) String kmsKeyId,
       @RequestHeader(value = CONTENT_TYPE, required = false) String contentType,
-      @RequestHeader(value = CONTENT_ENCODING, required = false) String contentEncoding,
       @RequestHeader HttpHeaders httpHeaders) {
     bucketService.verifyBucketExists(bucketName);
 
-    Map<String, String> userMetadata = getUserMetadata(httpHeaders);
+    Map<String, String> userMetadata = parseUserMetadata(httpHeaders);
+    Map<String, String> storeHeaders = parseStoreHeaders(httpHeaders);
 
     String uploadId = UUID.randomUUID().toString();
     InitiateMultipartUploadResult result =
         multipartService.prepareMultipartUpload(bucketName, key.getKey(),
-            contentType, contentEncoding, uploadId,
+            contentType, storeHeaders, uploadId,
             DEFAULT_OWNER, DEFAULT_OWNER, userMetadata);
 
     return ResponseEntity.ok(result);

server/src/main/java/com/adobe/testing/s3mock/ObjectController.java

@@ -44,10 +44,10 @@
 import static com.adobe.testing.s3mock.util.HeaderUtil.createEncryptionHeaders;
 import static com.adobe.testing.s3mock.util.HeaderUtil.createOverrideHeaders;
 import static com.adobe.testing.s3mock.util.HeaderUtil.createUserMetadataHeaders;
-import static com.adobe.testing.s3mock.util.HeaderUtil.getUserMetadata;
 import static com.adobe.testing.s3mock.util.HeaderUtil.isV4ChunkedWithSigningEnabled;
 import static com.adobe.testing.s3mock.util.HeaderUtil.parseMediaType;
-import static org.springframework.http.HttpHeaders.CONTENT_ENCODING;
+import static com.adobe.testing.s3mock.util.HeaderUtil.parseStoreHeaders;
+import static com.adobe.testing.s3mock.util.HeaderUtil.parseUserMetadata;
 import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
 import static org.springframework.http.HttpHeaders.IF_MATCH;
 import static org.springframework.http.HttpHeaders.IF_NONE_MATCH;
@@ -173,8 +173,8 @@ public ResponseEntity<Void> headObject(@PathVariable String bucketName,
       objectService.verifyObjectMatching(match, noneMatch, s3ObjectMetadata);
       return ResponseEntity.ok()
           .eTag(s3ObjectMetadata.getEtag())
-          .header(HttpHeaders.CONTENT_ENCODING, s3ObjectMetadata.getContentEncoding())
           .header(HttpHeaders.ACCEPT_RANGES, RANGES_BYTES)
+          .headers(headers -> headers.setAll(s3ObjectMetadata.getStoreHeaders()))
           .headers(headers -> headers.setAll(createUserMetadataHeaders(s3ObjectMetadata)))
           .headers(headers -> headers.setAll(createEncryptionHeaders(s3ObjectMetadata)))
           .lastModified(s3ObjectMetadata.getLastModified())
@@ -254,8 +254,8 @@ public ResponseEntity<StreamingResponseBody> getObject(@PathVariable String buck
     return ResponseEntity
         .ok()
         .eTag(s3ObjectMetadata.getEtag())
-        .header(HttpHeaders.CONTENT_ENCODING, s3ObjectMetadata.getContentEncoding())
         .header(HttpHeaders.ACCEPT_RANGES, RANGES_BYTES)
+        .headers(headers -> headers.setAll(s3ObjectMetadata.getStoreHeaders()))
         .headers(headers -> headers.setAll(createUserMetadataHeaders(s3ObjectMetadata)))
         .headers(headers -> headers.setAll(createEncryptionHeaders(s3ObjectMetadata)))
         .lastModified(s3ObjectMetadata.getLastModified())
@@ -531,7 +531,6 @@ public ResponseEntity<Void> putObject(@PathVariable String bucketName,
           value = X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID,
           required = false) String kmsKeyId,
       @RequestHeader(name = X_AMZ_TAGGING, required = false) List<Tag> tags,
-      @RequestHeader(value = CONTENT_ENCODING, required = false) String contentEncoding,
       @RequestHeader(value = CONTENT_TYPE, required = false) String contentType,
       @RequestHeader(value = CONTENT_MD5, required = false) String contentMd5,
       @RequestHeader(value = X_AMZ_CONTENT_SHA256, required = false) String sha256Header,
@@ -542,12 +541,13 @@ public ResponseEntity<Void> putObject(@PathVariable String bucketName,
     InputStream stream = objectService.verifyMd5(inputStream, contentMd5, sha256Header);
     //TODO: need to extract owner from headers
     Owner owner = Owner.DEFAULT_OWNER;
-    Map<String, String> userMetadata = getUserMetadata(headers);
+    Map<String, String> userMetadata = parseUserMetadata(headers);
+    Map<String, String> storeHeaders = parseStoreHeaders(headers);
     S3ObjectMetadata s3ObjectMetadata =
         objectService.putS3Object(bucketName,
             key.getKey(),
             parseMediaType(contentType).toString(),
-            contentEncoding,
+            storeHeaders,
             stream,
             isV4ChunkedWithSigningEnabled(sha256Header),
             userMetadata,
@@ -614,7 +614,7 @@ public ResponseEntity<CopyObjectResult> copyObject(@PathVariable String bucketNa
 
     Map<String, String> metadata = Collections.emptyMap();
     if (MetadataDirective.REPLACE == metadataDirective) {
-      metadata = getUserMetadata(httpHeaders);
+      metadata = parseUserMetadata(httpHeaders);
     }
 
     //TODO: this is potentially illegal on S3. S3 throws a 400: