From 5c52079fb5f52caf39a49ccb96df6251a9c728d3 Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Thu, 15 Dec 2022 16:19:24 -0500 Subject: [PATCH] feat: implement GrpcStorageImpl BucketAccessControl operations (#1816) Newly implemented methods (including any overloads): 1. GrpcStorageImpl#createAcl 2. GrpcStorageImpl#getAcl 3. GrpcStorageImpl#listAcl 4. GrpcStorageImpl#updateAcl 5. GrpcStorageImpl#deleteAcl 6. Bucket#createAcl 7. Bucket#getAcl 8. Bucket#listAcl 9. Bucket#updateAcl 10. Bucket#deleteAcl --- .../java/com/google/cloud/storage/Bucket.java | 10 +- .../google/cloud/storage/GrpcStorageImpl.java | 171 ++++++++-- .../com/google/cloud/storage/Storage.java | 20 +- .../cloud/storage/StorageV2ProtoUtils.java | 9 + .../com/google/cloud/storage/TestUtils.java | 32 ++ .../google/cloud/storage/it/ITAccessTest.java | 39 +-- .../cloud/storage/it/ITBucketAclTest.java | 294 ++++++++++++++++++ 7 files changed, 502 insertions(+), 73 deletions(-) create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketAclTest.java diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 251a74d76..6147552c0 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -1148,7 +1148,7 @@ public Blob create(String blob, InputStream content, BlobWriteOption... options) * * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) public Acl getAcl(Entity entity) { return storage.getAcl(getName(), entity); } @@ -1170,7 +1170,7 @@ public Acl getAcl(Entity entity) { * @return {@code true} if the ACL was deleted, {@code false} if it was not found * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) public boolean deleteAcl(Entity entity) { return storage.deleteAcl(getName(), entity); } @@ -1186,7 +1186,7 @@ public boolean deleteAcl(Entity entity) { * * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) public Acl createAcl(Acl acl) { return storage.createAcl(getName(), acl); } @@ -1202,7 +1202,7 @@ public Acl createAcl(Acl acl) { * * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) public Acl updateAcl(Acl acl) { return storage.updateAcl(getName(), acl); } @@ -1221,7 +1221,7 @@ public Acl updateAcl(Acl acl) { * * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) public List listAcls() { return storage.listAcls(getName()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java index 42f31ebcd..bcf4f584e 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java @@ -19,6 +19,7 @@ import static com.google.cloud.storage.ByteSizeConstants._16MiB; import static com.google.cloud.storage.ByteSizeConstants._256KiB; import static com.google.cloud.storage.GrpcToHttpStatusCodeTranslation.resultRetryAlgorithmToCodes; +import static com.google.cloud.storage.StorageV2ProtoUtils.bucketAclEntityOrAltEq; import static com.google.cloud.storage.StorageV2ProtoUtils.objectAclEntityOrAltEq; import static com.google.cloud.storage.Utils.bucketNameCodec; import static com.google.cloud.storage.Utils.ifNonNull; @@ -76,6 +77,7 @@ import com.google.iam.v1.TestIamPermissionsRequest; import com.google.protobuf.ByteString; import com.google.protobuf.FieldMask; +import com.google.storage.v2.BucketAccessControl; import com.google.storage.v2.ComposeObjectRequest; import com.google.storage.v2.ComposeObjectRequest.SourceObject; import com.google.storage.v2.CreateBucketRequest; @@ -152,6 +154,7 @@ final class GrpcStorageImpl extends BaseService implements Stora StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + private static final BucketSourceOption[] EMPTY_BUCKET_SOURCE_OPTIONS = new BucketSourceOption[0]; final StorageClient storageClient; final GrpcConversions codecs; @@ -853,63 +856,148 @@ public List delete(Iterable blobIds) { @Override public Acl getAcl(String bucket, Entity entity, BucketSourceOption... options) { - return throwNotYetImplemented( - fmtMethodName("getAcl", String.class, Entity.class, BucketSourceOption[].class)); + try { + Opts opts = Opts.unwrap(options); + com.google.storage.v2.Bucket resp = getBucketWithAcls(bucket, opts); + + Predicate entityPredicate = + bucketAclEntityOrAltEq(codecs.entity().encode(entity)); + + Optional first = + resp.getAclList().stream().filter(entityPredicate).findFirst(); + + // HttpStorageRpc defaults to null if Not Found + return first.map(codecs.bucketAcl()::decode).orElse(null); + } catch (NotFoundException e) { + return null; + } catch (StorageException se) { + if (se.getCode() == 404) { + return null; + } else { + throw se; + } + } } @Override public Acl getAcl(String bucket, Entity entity) { - return throwNotYetImplemented(fmtMethodName("getAcl", String.class, Entity.class)); + return getAcl(bucket, entity, EMPTY_BUCKET_SOURCE_OPTIONS); } @Override public boolean deleteAcl(String bucket, Entity entity, BucketSourceOption... options) { - return throwNotYetImplemented( - fmtMethodName("deleteAcl", String.class, Entity.class, BucketSourceOption[].class)); + try { + Opts opts = Opts.unwrap(options); + com.google.storage.v2.Bucket resp = getBucketWithAcls(bucket, opts); + String encode = codecs.entity().encode(entity); + + Predicate entityPredicate = bucketAclEntityOrAltEq(encode); + + List currentAcls = resp.getAclList(); + ImmutableList newAcls = + currentAcls.stream() + .filter(entityPredicate.negate()) + .collect(ImmutableList.toImmutableList()); + if (newAcls.equals(currentAcls)) { + // we didn't actually filter anything out, no need to send an RPC, simply return false + return false; + } + long metageneration = resp.getMetageneration(); + + UpdateBucketRequest req = createUpdateAclRequest(bucket, newAcls, metageneration); + + com.google.storage.v2.Bucket updateResult = updateBucket(req); + // read the response to ensure there is no longer an acl for the specified entity + Optional first = + updateResult.getAclList().stream().filter(entityPredicate).findFirst(); + return !first.isPresent(); + } catch (NotFoundException e) { + // HttpStorageRpc returns false if the bucket doesn't exist :( + return false; + } catch (StorageException se) { + if (se.getCode() == 404) { + return false; + } else { + throw se; + } + } } @Override public boolean deleteAcl(String bucket, Entity entity) { - return throwNotYetImplemented(fmtMethodName("deleteAcl", String.class, Entity.class)); + return deleteAcl(bucket, entity, EMPTY_BUCKET_SOURCE_OPTIONS); } @Override public Acl createAcl(String bucket, Acl acl, BucketSourceOption... options) { - return throwNotYetImplemented( - fmtMethodName("createAcl", String.class, Acl.class, BucketSourceOption[].class)); + return updateAcl(bucket, acl, options); } @Override public Acl createAcl(String bucket, Acl acl) { - return throwNotYetImplemented(fmtMethodName("createAcl", String.class, Acl.class)); + return createAcl(bucket, acl, EMPTY_BUCKET_SOURCE_OPTIONS); } @Override public Acl updateAcl(String bucket, Acl acl, BucketSourceOption... options) { - return throwNotYetImplemented( - fmtMethodName("updateAcl", String.class, Acl.class, BucketSourceOption[].class)); + try { + Opts opts = Opts.unwrap(options); + com.google.storage.v2.Bucket resp = getBucketWithAcls(bucket, opts); + BucketAccessControl encode = codecs.bucketAcl().encode(acl); + String entity = encode.getEntity(); + + Predicate entityPredicate = bucketAclEntityOrAltEq(entity); + + ImmutableList newDefaultAcls = + Streams.concat( + resp.getAclList().stream().filter(entityPredicate.negate()), Stream.of(encode)) + .collect(ImmutableList.toImmutableList()); + + UpdateBucketRequest req = + createUpdateAclRequest(bucket, newDefaultAcls, resp.getMetageneration()); + + com.google.storage.v2.Bucket updateResult = updateBucket(req); + + Optional first = + updateResult.getAclList().stream() + .filter(entityPredicate) + .findFirst() + .map(codecs.bucketAcl()::decode); + + return first.orElseThrow( + () -> new StorageException(0, "Acl update call success, but not in response")); + } catch (NotFoundException e) { + throw StorageException.coalesce(e); + } } @Override public Acl updateAcl(String bucket, Acl acl) { - return throwNotYetImplemented(fmtMethodName("updateAcl", String.class, Acl.class)); + return updateAcl(bucket, acl, EMPTY_BUCKET_SOURCE_OPTIONS); } @Override public List listAcls(String bucket, BucketSourceOption... options) { - return throwNotYetImplemented( - fmtMethodName("listAcls", String.class, BucketSourceOption[].class)); + try { + Opts opts = Opts.unwrap(options); + com.google.storage.v2.Bucket resp = getBucketWithAcls(bucket, opts); + return resp.getAclList().stream() + .map(codecs.bucketAcl()::decode) + .collect(ImmutableList.toImmutableList()); + } catch (NotFoundException e) { + throw StorageException.coalesce(e); + } } @Override public List listAcls(String bucket) { - return throwNotYetImplemented(fmtMethodName("listAcls", String.class)); + return listAcls(bucket, EMPTY_BUCKET_SOURCE_OPTIONS); } @Override public Acl getDefaultAcl(String bucket, Entity entity) { try { - com.google.storage.v2.Bucket resp = getBucketDefaultAcls(bucket); + com.google.storage.v2.Bucket resp = getBucketWithDefaultAcls(bucket); Predicate entityPredicate = objectAclEntityOrAltEq(codecs.entity().encode(entity)); @@ -933,7 +1021,7 @@ public Acl getDefaultAcl(String bucket, Entity entity) { @Override public boolean deleteDefaultAcl(String bucket, Entity entity) { try { - com.google.storage.v2.Bucket resp = getBucketDefaultAcls(bucket); + com.google.storage.v2.Bucket resp = getBucketWithDefaultAcls(bucket); String encode = codecs.entity().encode(entity); Predicate entityPredicate = objectAclEntityOrAltEq(encode); @@ -949,7 +1037,8 @@ public boolean deleteDefaultAcl(String bucket, Entity entity) { } long metageneration = resp.getMetageneration(); - UpdateBucketRequest req = createUpdateRequest(bucket, newDefaultAcls, metageneration); + UpdateBucketRequest req = + createUpdateDefaultAclRequest(bucket, newDefaultAcls, metageneration); com.google.storage.v2.Bucket updateResult = updateBucket(req); // read the response to ensure there is no longer an acl for the specified entity @@ -976,7 +1065,7 @@ public Acl createDefaultAcl(String bucket, Acl acl) { @Override public Acl updateDefaultAcl(String bucket, Acl acl) { try { - com.google.storage.v2.Bucket resp = getBucketDefaultAcls(bucket); + com.google.storage.v2.Bucket resp = getBucketWithDefaultAcls(bucket); ObjectAccessControl encode = codecs.objectAcl().encode(acl); String entity = encode.getEntity(); @@ -989,7 +1078,7 @@ public Acl updateDefaultAcl(String bucket, Acl acl) { .collect(ImmutableList.toImmutableList()); UpdateBucketRequest req = - createUpdateRequest(bucket, newDefaultAcls, resp.getMetageneration()); + createUpdateDefaultAclRequest(bucket, newDefaultAcls, resp.getMetageneration()); com.google.storage.v2.Bucket updateResult = updateBucket(req); @@ -1009,7 +1098,7 @@ public Acl updateDefaultAcl(String bucket, Acl acl) { @Override public List listDefaultAcls(String bucket) { try { - com.google.storage.v2.Bucket resp = getBucketDefaultAcls(bucket); + com.google.storage.v2.Bucket resp = getBucketWithDefaultAcls(bucket); return resp.getDefaultObjectAclList().stream() .map(codecs.objectAcl()::decode) .collect(ImmutableList.toImmutableList()); @@ -1481,7 +1570,7 @@ private SourceObject sourceObjectEncode(SourceBlob from) { return to.build(); } - private com.google.storage.v2.Bucket getBucketDefaultAcls(String bucketName) { + private com.google.storage.v2.Bucket getBucketWithDefaultAcls(String bucketName) { Fields fields = UnifiedOpts.fields( ImmutableSet.of( @@ -1503,6 +1592,25 @@ private com.google.storage.v2.Bucket getBucketDefaultAcls(String bucketName) { Decoder.identity()); } + private com.google.storage.v2.Bucket getBucketWithAcls( + String bucketName, Opts opts) { + Fields fields = + UnifiedOpts.fields(ImmutableSet.of(BucketField.ACL, BucketField.METAGENERATION)); + GrpcCallContext grpcCallContext = GrpcCallContext.createDefault(); + Mapper mapper = opts.getBucketsRequest().andThen(fields.getBucket()); + GetBucketRequest req = + mapper + .apply(GetBucketRequest.newBuilder()) + .setName(bucketNameCodec.encode(bucketName)) + .build(); + + return Retrying.run( + getOptions(), + retryAlgorithmManager.getFor(req), + () -> storageClient.getBucketCallable().call(req, grpcCallContext), + Decoder.identity()); + } + private com.google.storage.v2.Bucket updateBucket(UpdateBucketRequest req) { GrpcCallContext grpcCallContext = GrpcCallContext.createDefault(); return Retrying.run( @@ -1512,7 +1620,7 @@ private com.google.storage.v2.Bucket updateBucket(UpdateBucketRequest req) { Decoder.identity()); } - private static UpdateBucketRequest createUpdateRequest( + private static UpdateBucketRequest createUpdateDefaultAclRequest( String bucket, ImmutableList newDefaultAcls, long metageneration) { com.google.storage.v2.Bucket update = com.google.storage.v2.Bucket.newBuilder() @@ -1528,4 +1636,21 @@ private static UpdateBucketRequest createUpdateRequest( .setBucket(update) .build(); } + + private static UpdateBucketRequest createUpdateAclRequest( + String bucket, ImmutableList newDefaultAcls, long metageneration) { + com.google.storage.v2.Bucket update = + com.google.storage.v2.Bucket.newBuilder() + .setName(bucketNameCodec.encode(bucket)) + .addAllAcl(newDefaultAcls) + .build(); + Opts opts = + Opts.from( + UnifiedOpts.fields(ImmutableSet.of(BucketField.ACL)), + UnifiedOpts.metagenerationMatch(metageneration)); + return opts.updateBucketsRequest() + .apply(UpdateBucketRequest.newBuilder()) + .setBucket(update) + .build(); + } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 610a6504e..451c454fe 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -3332,11 +3332,11 @@ PostPolicyV4 generateSignedPostPolicyV4( * @param options extra parameters to apply to this operation * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) Acl getAcl(String bucket, Entity entity, BucketSourceOption... options); /** @see #getAcl(String, Entity, BucketSourceOption...) */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) Acl getAcl(String bucket, Entity entity); /** @@ -3369,11 +3369,11 @@ PostPolicyV4 generateSignedPostPolicyV4( * @return {@code true} if the ACL was deleted, {@code false} if it was not found * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) boolean deleteAcl(String bucket, Entity entity, BucketSourceOption... options); /** @see #deleteAcl(String, Entity, BucketSourceOption...) */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) boolean deleteAcl(String bucket, Entity entity); /** @@ -3399,11 +3399,11 @@ PostPolicyV4 generateSignedPostPolicyV4( * @param options extra parameters to apply to this operation * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) Acl createAcl(String bucket, Acl acl, BucketSourceOption... options); /** @see #createAcl(String, Acl, BucketSourceOption...) */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) Acl createAcl(String bucket, Acl acl); /** @@ -3429,11 +3429,11 @@ PostPolicyV4 generateSignedPostPolicyV4( * @param options extra parameters to apply to this operation * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) Acl updateAcl(String bucket, Acl acl, BucketSourceOption... options); /** @see #updateAcl(String, Acl, BucketSourceOption...) */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) Acl updateAcl(String bucket, Acl acl); /** @@ -3464,11 +3464,11 @@ PostPolicyV4 generateSignedPostPolicyV4( * @param options any number of BucketSourceOptions to apply to this operation * @throws StorageException upon failure */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) List listAcls(String bucket, BucketSourceOption... options); /** @see #listAcls(String, BucketSourceOption...) */ - @TransportCompatibility({Transport.HTTP}) + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) List listAcls(String bucket); /** diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageV2ProtoUtils.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageV2ProtoUtils.java index 53d355b4f..cbf49ef69 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageV2ProtoUtils.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageV2ProtoUtils.java @@ -22,6 +22,7 @@ import com.google.protobuf.MessageOrBuilder; import com.google.protobuf.util.JsonFormat; import com.google.protobuf.util.JsonFormat.Printer; +import com.google.storage.v2.BucketAccessControl; import com.google.storage.v2.ObjectAccessControl; import com.google.storage.v2.ReadObjectRequest; import java.util.function.Predicate; @@ -92,4 +93,12 @@ static String fmtProto(@NonNull final MessageOrBuilder msg) { static Predicate objectAclEntityOrAltEq(String s) { return oAcl -> oAcl.getEntity().equals(s) || oAcl.getEntityAlt().equals(s); } + + /** + * When evaluating an {@link BucketAccessControl} entity, look at both {@code entity} (generally + * project number format) and {@code entity_alt} (generally project id format). + */ + static Predicate bucketAclEntityOrAltEq(String s) { + return oAcl -> oAcl.getEntity().equals(s) || oAcl.getEntityAlt().equals(s); + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java index 117284068..1810060c7 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java @@ -20,11 +20,15 @@ import com.google.api.core.NanoClock; import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.retrying.BasicResultRetryAlgorithm; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.ApiExceptionFactory; import com.google.api.gax.rpc.ErrorDetails; import com.google.api.gax.rpc.StatusCode; +import com.google.cloud.RetryHelper; +import com.google.cloud.RetryHelper.RetryHelperException; +import com.google.cloud.http.BaseHttpServiceException; import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown; import com.google.cloud.storage.Retrying.RetryingDependencies; import com.google.common.collect.ImmutableList; @@ -40,6 +44,7 @@ import java.io.OutputStream; import java.nio.Buffer; import java.nio.ByteBuffer; +import java.util.concurrent.Callable; import java.util.function.Function; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -147,4 +152,31 @@ public static T findThrowable(Class c, Throwable t) { } return found; } + + public static T retry429s(Callable c, Storage storage) { + try { + return RetryHelper.runWithRetries( + c, + storage.getOptions().getRetrySettings(), + new BasicResultRetryAlgorithm() { + @Override + public boolean shouldRetry(Throwable previousThrowable, Object previousResponse) { + if (previousThrowable instanceof BaseHttpServiceException) { + BaseHttpServiceException httpException = + (BaseHttpServiceException) previousThrowable; + return httpException.getCode() == 429; + } + return false; + } + }, + storage.getOptions().getClock()); + } catch (RetryHelperException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } else { + throw e; + } + } + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAccessTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAccessTest.java index 2efe6a22b..b9dd61c74 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAccessTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAccessTest.java @@ -16,6 +16,7 @@ package com.google.cloud.storage.it; +import static com.google.cloud.storage.TestUtils.retry429s; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -25,13 +26,9 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import com.google.api.gax.retrying.BasicResultRetryAlgorithm; import com.google.cloud.Condition; import com.google.cloud.Identity; import com.google.cloud.Policy; -import com.google.cloud.RetryHelper; -import com.google.cloud.RetryHelper.RetryHelperException; -import com.google.cloud.http.BaseHttpServiceException; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Acl.Entity; import com.google.cloud.storage.Acl.Project.ProjectRole; @@ -72,7 +69,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.Callable; import java.util.function.Predicate; import java.util.stream.Collector; import java.util.stream.Collectors; @@ -1127,44 +1123,17 @@ public void testBlobAcl() { } } - static T retry429s(Callable c, Storage storage) { - try { - return RetryHelper.runWithRetries( - c, - storage.getOptions().getRetrySettings(), - new BasicResultRetryAlgorithm() { - @Override - public boolean shouldRetry(Throwable previousThrowable, Object previousResponse) { - if (previousThrowable instanceof BaseHttpServiceException) { - BaseHttpServiceException httpException = - (BaseHttpServiceException) previousThrowable; - return httpException.getCode() == 429; - } - return false; - } - }, - storage.getOptions().getClock()); - } catch (RetryHelperException e) { - Throwable cause = e.getCause(); - if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; - } else { - throw e; - } - } - } - - private static ImmutableList dropEtags(List defaultAcls) { + static ImmutableList dropEtags(List defaultAcls) { return defaultAcls.stream() .map(acl -> Acl.of(acl.getEntity(), acl.getRole())) .collect(ImmutableList.toImmutableList()); } - private static Predicate hasRole(Acl.Role expected) { + static Predicate hasRole(Acl.Role expected) { return acl -> acl.getRole().equals(expected); } - private static Predicate hasProjectRole(Acl.Project.ProjectRole expected) { + static Predicate hasProjectRole(Acl.Project.ProjectRole expected) { return acl -> { Entity entity = acl.getEntity(); if (entity.getType().equals(Entity.Type.PROJECT)) { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketAclTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketAclTest.java new file mode 100644 index 000000000..a1545202e --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketAclTest.java @@ -0,0 +1,294 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.cloud.storage.TestUtils.retry429s; +import static com.google.cloud.storage.it.ITAccessTest.dropEtags; +import static com.google.cloud.storage.it.ITAccessTest.hasProjectRole; +import static com.google.cloud.storage.it.ITAccessTest.hasRole; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.cloud.storage.Acl; +import com.google.cloud.storage.Acl.Entity; +import com.google.cloud.storage.Acl.Project.ProjectRole; +import com.google.cloud.storage.Acl.Role; +import com.google.cloud.storage.Acl.User; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BucketField; +import com.google.cloud.storage.Storage.BucketGetOption; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.annotations.ParallelFriendly; +import com.google.cloud.storage.it.runner.registry.Generator; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@CrossRun( + transports = {Transport.HTTP, Transport.GRPC}, + backends = {Backend.PROD}) +@ParallelFriendly +public final class ITBucketAclTest { + @Inject public Storage storage; + + @Inject public BucketInfo bucket; + + @Inject public Generator generator; + + @Test + public void bucket_acl_get() { + String bucketName = bucket.getName(); + // lookup an entity from the bucket which is known to exist + Bucket bucketWithAcls = storage.get(bucketName, BucketGetOption.fields(BucketField.ACL)); + + Acl actual = bucketWithAcls.getAcl().iterator().next(); + + Acl acl = retry429s(() -> storage.getAcl(bucketName, actual.getEntity()), storage); + + assertThat(acl).isEqualTo(actual); + } + + /** When a bucket does exist, but an acl for the specified entity is not defined return null */ + @Test + public void bucket_acl_get_notFoundReturnsNull() { + Acl acl = retry429s(() -> storage.getAcl(bucket.getName(), User.ofAllUsers()), storage); + + assertThat(acl).isNull(); + } + + /** When a bucket doesn't exist, return null for the acl value */ + @Test + public void bucket_acl_get_bucket404() { + Acl acl = retry429s(() -> storage.getAcl(bucket.getName() + "x", User.ofAllUsers()), storage); + + assertThat(acl).isNull(); + } + + @Test + public void bucket_acl_list() { + String bucketName = bucket.getName(); + // lookup an entity from the bucket which is known to exist + Bucket bucketWithAcls = storage.get(bucketName, BucketGetOption.fields(BucketField.ACL)); + + Acl actual = bucketWithAcls.getAcl().iterator().next(); + + List acls = retry429s(() -> storage.listAcls(bucketName), storage); + + assertThat(acls).contains(actual); + } + + @Test + public void bucket_acl_list_bucket404() { + StorageException storageException = + assertThrows( + StorageException.class, + () -> retry429s(() -> storage.listAcls(bucket.getName() + "x"), storage)); + + assertThat(storageException.getCode()).isEqualTo(404); + } + + @Test + public void bucket_acl_create() throws Exception { + BucketInfo bucketInfo = BucketInfo.newBuilder(generator.randomBucketName()).build(); + try (TemporaryBucket tempB = + TemporaryBucket.newBuilder().setBucketInfo(bucketInfo).setStorage(storage).build()) { + BucketInfo bucket = tempB.getBucket(); + + Acl readAll = Acl.of(User.ofAllAuthenticatedUsers(), Role.READER); + Acl actual = retry429s(() -> storage.createAcl(bucket.getName(), readAll), storage); + + assertThat(actual.getEntity()).isEqualTo(readAll.getEntity()); + assertThat(actual.getRole()).isEqualTo(readAll.getRole()); + assertThat(actual.getEtag()).isNotEmpty(); + + Bucket bucketUpdated = + storage.get(bucket.getName(), BucketGetOption.fields(BucketField.values())); + assertThat(bucketUpdated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); + + // etags change when updates happen, drop before our comparison + List expectedAcls = dropEtags(bucket.getAcl()); + List actualAcls = dropEtags(bucketUpdated.getAcl()); + assertThat(actualAcls).containsAtLeastElementsIn(expectedAcls); + assertThat(actualAcls).contains(readAll); + } + } + + @Test + public void bucket_acl_create_bucket404() { + Acl readAll = Acl.of(User.ofAllAuthenticatedUsers(), Role.READER); + StorageException storageException = + assertThrows( + StorageException.class, + () -> retry429s(() -> storage.createAcl(bucket.getName() + "x", readAll), storage)); + + assertThat(storageException.getCode()).isEqualTo(404); + } + + @Test + public void bucket_acl_update() throws Exception { + BucketInfo bucketInfo = BucketInfo.newBuilder(generator.randomBucketName()).build(); + try (TemporaryBucket tempB = + TemporaryBucket.newBuilder().setBucketInfo(bucketInfo).setStorage(storage).build()) { + BucketInfo bucket = tempB.getBucket(); + + List acls = bucket.getAcl(); + assertThat(acls).isNotEmpty(); + + Predicate isProjectEditor = hasProjectRole(ProjectRole.EDITORS); + + //noinspection OptionalGetWithoutIsPresent + Acl projectEditorAsOwner = + acls.stream().filter(hasRole(Role.OWNER).and(isProjectEditor)).findFirst().get(); + + // lower the privileges of project editors to writer from owner + Entity entity = projectEditorAsOwner.getEntity(); + Acl projectEditorAsReader = Acl.of(entity, Role.READER); + + Acl actual = + retry429s(() -> storage.updateAcl(bucket.getName(), projectEditorAsReader), storage); + + assertThat(actual.getEntity()).isEqualTo(projectEditorAsReader.getEntity()); + assertThat(actual.getRole()).isEqualTo(projectEditorAsReader.getRole()); + assertThat(actual.getEtag()).isNotEmpty(); + + Bucket bucketUpdated = + storage.get(bucket.getName(), BucketGetOption.fields(BucketField.values())); + assertThat(bucketUpdated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); + + // etags change when updates happen, drop before our comparison + List expectedAcls = + dropEtags( + bucket.getAcl().stream() + .filter(isProjectEditor.negate()) + .collect(Collectors.toList())); + List actualAcls = dropEtags(bucketUpdated.getAcl()); + assertThat(actualAcls).containsAtLeastElementsIn(expectedAcls); + assertThat(actualAcls).doesNotContain(projectEditorAsOwner); + assertThat(actualAcls).contains(projectEditorAsReader); + } + } + + @Test + public void bucket_acl_update_bucket404() { + Acl readAll = Acl.of(User.ofAllAuthenticatedUsers(), Role.READER); + StorageException storageException = + assertThrows( + StorageException.class, + () -> retry429s(() -> storage.updateAcl(bucket.getName() + "x", readAll), storage)); + + assertThat(storageException.getCode()).isEqualTo(404); + } + + /** Update of an acl that doesn't exist should create it */ + @Test + public void bucket_acl_404_acl_update() throws Exception { + BucketInfo bucketInfo = BucketInfo.newBuilder(generator.randomBucketName()).build(); + try (TemporaryBucket tempB = + TemporaryBucket.newBuilder().setBucketInfo(bucketInfo).setStorage(storage).build()) { + BucketInfo mgen1 = tempB.getBucket(); + Acl readAll = Acl.of(User.ofAllAuthenticatedUsers(), Role.READER); + Acl actual = + // todo: json non-idempotent? + retry429s(() -> storage.updateAcl(mgen1.getName(), readAll), storage); + + assertThat(actual.getEntity()).isEqualTo(readAll.getEntity()); + assertThat(actual.getRole()).isEqualTo(readAll.getRole()); + assertThat(actual.getEtag()).isNotEmpty(); + + Bucket updated = storage.get(mgen1.getName(), BucketGetOption.fields(BucketField.values())); + assertThat(updated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); + + // etags change when updates happen, drop before our comparison + List expectedAcls = dropEtags(mgen1.getAcl()); + List actualAcls = dropEtags(updated.getAcl()); + assertThat(actualAcls).containsAtLeastElementsIn(expectedAcls); + assertThat(actualAcls).contains(readAll); + } + } + + @Test + public void bucket_acl_delete() throws Exception { + BucketInfo bucketInfo = BucketInfo.newBuilder(generator.randomBucketName()).build(); + try (TemporaryBucket tempB = + TemporaryBucket.newBuilder().setBucketInfo(bucketInfo).setStorage(storage).build()) { + BucketInfo bucket = tempB.getBucket(); + + List acls = bucket.getAcl(); + assertThat(acls).isNotEmpty(); + + Predicate isProjectEditor = hasProjectRole(ProjectRole.VIEWERS); + + //noinspection OptionalGetWithoutIsPresent + Acl projectViewerAsReader = + acls.stream().filter(hasRole(Role.READER).and(isProjectEditor)).findFirst().get(); + + Entity entity = projectViewerAsReader.getEntity(); + + boolean actual = retry429s(() -> storage.deleteAcl(bucket.getName(), entity), storage); + + assertThat(actual).isTrue(); + + Bucket bucketUpdated = + storage.get(bucket.getName(), BucketGetOption.fields(BucketField.values())); + assertThat(bucketUpdated.getMetageneration()).isNotEqualTo(bucket.getMetageneration()); + + // etags change when deletes happen, drop before our comparison + List expectedAcls = + dropEtags( + bucket.getAcl().stream() + .filter(isProjectEditor.negate()) + .collect(Collectors.toList())); + List actualAcls = dropEtags(bucketUpdated.getAcl()); + assertThat(actualAcls).containsAtLeastElementsIn(expectedAcls); + Optional search = + actualAcls.stream().map(Acl::getEntity).filter(e -> e.equals(entity)).findAny(); + assertThat(search.isPresent()).isFalse(); + } + } + + @Test + public void bucket_acl_delete_bucket404() { + boolean actual = + retry429s(() -> storage.deleteAcl(bucket.getName() + "x", User.ofAllUsers()), storage); + + assertThat(actual).isEqualTo(false); + } + + @Test + public void bucket_acl_delete_noExistingAcl() throws Exception { + BucketInfo bucketInfo = BucketInfo.newBuilder(generator.randomBucketName()).build(); + try (TemporaryBucket tempB = + TemporaryBucket.newBuilder().setBucketInfo(bucketInfo).setStorage(storage).build()) { + BucketInfo bucket = tempB.getBucket(); + boolean actual = + retry429s(() -> storage.deleteAcl(bucket.getName(), User.ofAllUsers()), storage); + + assertThat(actual).isEqualTo(false); + } + } +}