diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index ba3233873..635e882a6 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -15,6 +15,12 @@ * writeAndClose(*) + + 7013 + com/google/cloud/storage/BucketInfo$Builder + com.google.cloud.storage.BucketInfo$Builder setHierarchicalNamespace(com.google.cloud.storage.BucketInfo$HierarchicalNamespace) + + 7013 com/google/cloud/storage/BlobInfo$Builder diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 3c1652bc9..6459e0a0a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -748,6 +748,12 @@ Builder setObjectRetention(ObjectRetention objectRetention) { return this; } + @Override + public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) { + infoBuilder.setHierarchicalNamespace(hierarchicalNamespace); + return this; + } + @Override public Bucket build() { return new Bucket(storage, infoBuilder); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 9fb46b5bd..af3e4436e 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -118,6 +118,7 @@ public class BucketInfo implements Serializable { private final Logging logging; private final CustomPlacementConfig customPlacementConfig; private final ObjectRetention objectRetention; + private final HierarchicalNamespace hierarchicalNamespace; private final transient ImmutableSet modifiedFields; @@ -713,6 +714,71 @@ public Logging build() { } } + /** The bucket's hierarchical namespace (Folders) configuration. Enable this to use HNS. */ + public static final class HierarchicalNamespace implements Serializable { + + private static final long serialVersionUID = 5932926691444613101L; + private Boolean enabled; + + public Boolean getEnabled() { + return enabled; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof HierarchicalNamespace)) { + return false; + } + HierarchicalNamespace that = (HierarchicalNamespace) o; + return Objects.equals(enabled, that.enabled); + } + + @Override + public int hashCode() { + return Objects.hash(enabled); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("enabled", enabled).toString(); + } + + private HierarchicalNamespace() {} + + private HierarchicalNamespace(Builder builder) { + this.enabled = builder.enabled; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public Builder toBuilder() { + return newBuilder().setEnabled(enabled); + } + + public static final class Builder { + private Boolean enabled; + + /** + * Sets whether Hierarchical Namespace (Folders) is enabled for this bucket. This can only be + * enabled at bucket create time. If this is enabled, Uniform Bucket-Level Access must also be + * enabled. + */ + public Builder setEnabled(Boolean enabled) { + this.enabled = enabled; + return this; + } + + public HierarchicalNamespace build() { + return new HierarchicalNamespace(this); + } + } + } + /** * Lifecycle rule for a bucket. Allows supported Actions, such as deleting and changing storage * class, to be executed when certain Conditions are met. @@ -1683,6 +1749,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) { public abstract Builder setCustomPlacementConfig(CustomPlacementConfig customPlacementConfig); + public abstract Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace); + abstract Builder setObjectRetention(ObjectRetention objectRetention); /** Creates a {@code BucketInfo} object. */ @@ -1783,6 +1851,7 @@ static final class BuilderImpl extends Builder { private Logging logging; private CustomPlacementConfig customPlacementConfig; private ObjectRetention objectRetention; + private HierarchicalNamespace hierarchicalNamespace; private final ImmutableSet.Builder modifiedFields = ImmutableSet.builder(); BuilderImpl(String name) { @@ -1822,6 +1891,7 @@ static final class BuilderImpl extends Builder { logging = bucketInfo.logging; customPlacementConfig = bucketInfo.customPlacementConfig; objectRetention = bucketInfo.objectRetention; + hierarchicalNamespace = bucketInfo.hierarchicalNamespace; } @Override @@ -2187,6 +2257,15 @@ Builder setObjectRetention(ObjectRetention objectRetention) { return this; } + @Override + public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) { + if (!Objects.equals(this.hierarchicalNamespace, hierarchicalNamespace)) { + modifiedFields.add(BucketField.HIERARCHICAL_NAMESPACE); + } + this.hierarchicalNamespace = hierarchicalNamespace; + return this; + } + @Override Builder setLocationType(String locationType) { if (!Objects.equals(this.locationType, locationType)) { @@ -2428,6 +2507,7 @@ private Builder clearDeleteLifecycleRules() { logging = builder.logging; customPlacementConfig = builder.customPlacementConfig; objectRetention = builder.objectRetention; + hierarchicalNamespace = builder.hierarchicalNamespace; modifiedFields = builder.modifiedFields.build(); } @@ -2768,6 +2848,11 @@ public ObjectRetention getObjectRetention() { return objectRetention; } + /** Returns the Hierarchical Namespace (Folders) Configuration */ + public HierarchicalNamespace getHierarchicalNamespace() { + return hierarchicalNamespace; + } + /** Returns a builder for the current bucket. */ public Builder toBuilder() { return new BuilderImpl(this); @@ -2805,6 +2890,7 @@ public int hashCode() { autoclass, locationType, objectRetention, + hierarchicalNamespace, logging); } @@ -2846,6 +2932,7 @@ public boolean equals(Object o) { && Objects.equals(autoclass, that.autoclass) && Objects.equals(locationType, that.locationType) && Objects.equals(objectRetention, that.objectRetention) + && Objects.equals(hierarchicalNamespace, that.hierarchicalNamespace) && Objects.equals(logging, that.logging); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java index 9ddf229c5..d1084bc41 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java @@ -110,6 +110,10 @@ final class GrpcConversions { private final Codec iamConditionCodec = Codec.of(this::conditionEncode, this::conditionDecode); + private final Codec + hierarchicalNamespaceCodec = + Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode); + @VisibleForTesting final Codec timestampCodec = Codec.of( @@ -297,6 +301,10 @@ private BucketInfo bucketInfoDecode(Bucket from) { .setDataLocations(customPlacementConfig.getDataLocationsList()) .build()); } + if (from.hasHierarchicalNamespace()) { + to.setHierarchicalNamespace( + hierarchicalNamespaceCodec.decode(from.getHierarchicalNamespace())); + } // TODO(frankyn): Add SelfLink when the field is available if (!from.getEtag().isEmpty()) { to.setEtag(from.getEtag()); @@ -382,6 +390,10 @@ private Bucket bucketInfoEncode(BucketInfo from) { .addAllDataLocations(customPlacementConfig.getDataLocations()) .build()); } + ifNonNull( + from.getHierarchicalNamespace(), + hierarchicalNamespaceCodec::encode, + to::setHierarchicalNamespace); // TODO(frankyn): Add SelfLink when the field is available ifNonNull(from.getEtag(), to::setEtag); return to.build(); @@ -589,6 +601,20 @@ private Bucket.Autoclass autoclassEncode(BucketInfo.Autoclass from) { return to.build(); } + private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode( + BucketInfo.HierarchicalNamespace from) { + Bucket.HierarchicalNamespace.Builder to = Bucket.HierarchicalNamespace.newBuilder(); + ifNonNull(from.getEnabled(), to::setEnabled); + return to.build(); + } + + private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode( + Bucket.HierarchicalNamespace from) { + BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder(); + to.setEnabled(from.getEnabled()); + return to.build(); + } + private Bucket.IamConfig iamConfigEncode(BucketInfo.IamConfiguration from) { Bucket.IamConfig.Builder to = Bucket.IamConfig.newBuilder(); to.setUniformBucketLevelAccess(ublaEncode(from)); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java index 4bbf08d4b..c5add241d 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java @@ -137,6 +137,10 @@ final class JsonConversions { private final Codec blobInfoCodec = Codec.of(this::blobInfoEncode, this::blobInfoDecode); + private final Codec + hierarchicalNamespaceCodec = + Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode); + private final Codec notificationInfoCodec = Codec.of(this::notificationEncode, this::notificationDecode); private final Codec @@ -437,6 +441,10 @@ private Bucket bucketInfoEncode(BucketInfo from) { this::customPlacementConfigEncode, to::setCustomPlacementConfig); ifNonNull(from.getObjectRetention(), this::objectRetentionEncode, to::setObjectRetention); + ifNonNull( + from.getHierarchicalNamespace(), + this::hierarchicalNamespaceEncode, + to::setHierarchicalNamespace); return to; } @@ -487,6 +495,10 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket from.getCustomPlacementConfig(), this::customPlacementConfigDecode, to::setCustomPlacementConfig); + ifNonNull( + from.getHierarchicalNamespace(), + this::hierarchicalNamespaceDecode, + to::setHierarchicalNamespace); ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention); return to.build(); } @@ -861,6 +873,20 @@ private com.google.api.services.storage.model.Notification notificationEncode( return to; } + private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode( + BucketInfo.HierarchicalNamespace from) { + Bucket.HierarchicalNamespace to = new Bucket.HierarchicalNamespace(); + ifNonNull(from.getEnabled(), to::setEnabled); + return to; + } + + private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode( + Bucket.HierarchicalNamespace from) { + BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder(); + to.setEnabled(from.getEnabled()); + return to.build(); + } + private NotificationInfo notificationDecode( com.google.api.services.storage.model.Notification from) { NotificationInfo.Builder builder = new NotificationInfo.BuilderImpl(from.getTopic()); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 5ca31c042..1d0ae8347 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -159,6 +159,9 @@ enum BucketField implements FieldSelector, NamedField { CUSTOM_PLACEMENT_CONFIG("customPlacementConfig", "custom_placement_config"), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) AUTOCLASS("autoclass"), + + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + HIERARCHICAL_NAMESPACE("hierarchicalNamespace", "hierarchical_namespace"), @TransportCompatibility({Transport.HTTP}) OBJECT_RETENTION("objectRetention"); @@ -1788,6 +1791,14 @@ public static BlobListOption matchGlob(@NonNull String glob) { return new BlobListOption(UnifiedOpts.matchGlob(glob)); } + /** + * Returns an option for whether to include all Folders (including empty Folders) in response. + */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public static BlobListOption includeFolders(boolean includeFolders) { + return new BlobListOption(UnifiedOpts.includeFoldersAsPrefixes(includeFolders)); + } + /** * Returns an option to define the billing user project. This option is required by buckets with * `requester_pays` flag enabled to assign operation costs. diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java index dbf104e90..3159cbebb 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java @@ -370,6 +370,10 @@ static Delimiter delimiter(@NonNull String delimiter) { return new Delimiter(delimiter); } + static IncludeFoldersAsPrefixes includeFoldersAsPrefixes(boolean includeFoldersAsPrefixes) { + return new IncludeFoldersAsPrefixes(includeFoldersAsPrefixes); + } + @Deprecated static DetectContentType detectContentType() { return DetectContentType.INSTANCE; @@ -636,6 +640,20 @@ public Mapper rewriteObject() { } } + static final class IncludeFoldersAsPrefixes extends RpcOptVal implements ObjectListOpt { + + private static final long serialVersionUID = 321916692864878282L; + + private IncludeFoldersAsPrefixes(boolean val) { + super(StorageRpc.Option.INCLUDE_FOLDERS_AS_PREFIXES, val); + } + + @Override + public Mapper listObjects() { + return b -> b.setIncludeFoldersAsPrefixes(val); + } + } + static final class Delimiter extends RpcOptVal implements ObjectListOpt { private static final long serialVersionUID = -3789556789947615714L; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 2f1c3e210..3ca2eabec 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -459,6 +459,7 @@ public Tuple> list(final String bucket, Map storageObjects = Iterables.concat( diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 7d671deed..3b40f6a23 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -73,7 +73,9 @@ enum Option { DETECT_CONTENT_TYPE("detectContentType"), ENABLE_OBJECT_RETENTION("enableObjectRetention"), RETURN_RAW_INPUT_STREAM("returnRawInputStream"), - OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention"); + OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention"), + INCLUDE_FOLDERS_AS_PREFIXES("includeFoldersAsPrefixes"); + ; private final String value; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java index 93099d41e..9c9776032 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java @@ -133,6 +133,7 @@ public ImmutableList parameters() { new Args<>(BucketField.TIME_CREATED, LazyAssertion.equal()), new Args<>(BucketField.UPDATED, LazyAssertion.equal()), new Args<>(BucketField.VERSIONING, LazyAssertion.equal()), + new Args<>(BucketField.HIERARCHICAL_NAMESPACE, LazyAssertion.equal()), new Args<>(BucketField.WEBSITE, LazyAssertion.equal())); List argsDefined = diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java index e32be5c32..5b759be6b 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java @@ -26,6 +26,7 @@ import static org.junit.Assert.assertTrue; import com.google.api.gax.paging.Page; +import com.google.api.services.storage.model.Folder; import com.google.cloud.Policy; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; @@ -44,6 +45,7 @@ import com.google.cloud.storage.Storage.BucketListOption; import com.google.cloud.storage.Storage.BucketTargetOption; import com.google.cloud.storage.StorageClass; +import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.TestUtils; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.StorageITRunner; @@ -53,6 +55,7 @@ import com.google.cloud.storage.it.runner.annotations.CrossRun; import com.google.cloud.storage.it.runner.annotations.Inject; import com.google.cloud.storage.it.runner.registry.Generator; +import com.google.cloud.storage.spi.v1.HttpStorageRpc; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.time.Duration; @@ -551,6 +554,79 @@ public void testUpdateBucket_noModification() throws Exception { } } + @Test + public void createBucketWithHierarchicalNamespace() { + String bucketName = generator.randomBucketName(); + storage.create( + BucketInfo.newBuilder(bucketName) + .setHierarchicalNamespace( + BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + BucketInfo.IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()) + .build()); + try { + Bucket remoteBucket = storage.get(bucketName); + assertNotNull(remoteBucket.getHierarchicalNamespace()); + assertTrue(remoteBucket.getHierarchicalNamespace().getEnabled()); + } finally { + BucketCleaner.doCleanup(bucketName, storage); + } + } + + @Test + public void testListObjectsWithFolders() throws Exception { + String bucketName = generator.randomBucketName(); + storage.create( + BucketInfo.newBuilder(bucketName) + .setHierarchicalNamespace( + BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + BucketInfo.IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()) + .build()); + try { + com.google.api.services.storage.Storage apiaryStorage = + new HttpStorageRpc(StorageOptions.getDefaultInstance()).getStorage(); + apiaryStorage + .folders() + .insert(bucketName, new Folder().setName("F").setBucket(bucketName)) + .execute(); + + Page blobs = + storage.list( + bucketName, + Storage.BlobListOption.delimiter("/"), + Storage.BlobListOption.includeFolders(false)); + + boolean found = false; + for (Blob blob : blobs.iterateAll()) { + if (blob.getName().equals("F/")) { + found = true; + } + } + assert (!found); + + blobs = + storage.list( + bucketName, + Storage.BlobListOption.delimiter("/"), + Storage.BlobListOption.includeFolders(true)); + + for (Blob blob : blobs.iterateAll()) { + if (blob.getName().equals("F/")) { + found = true; + } + } + assert (found); + + } finally { + BucketCleaner.doCleanup(bucketName, storage); + } + } + private void unsetRequesterPays() { Bucket remoteBucket = storage.get( diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java index 4dbe55adc..7627bf3af 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java @@ -326,6 +326,7 @@ public void storage_BucketGetOption_fields_BucketField() { "timeCreated", "updated", "versioning", + "hierarchicalNamespace", "website"); s.get( b.getName(), @@ -816,6 +817,7 @@ public void storage_BucketListOption_fields_BucketField() { "items/timeCreated", "items/updated", "items/versioning", + "items/hierarchicalNamespace", "items/website"); s.list(BucketListOption.fields(TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); requestAuditing.assertQueryParam("fields", expected, splitOnCommaToSet());