diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/HivePartitioningOptions.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/HivePartitioningOptions.java index db6e434ce..7df046ee1 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/HivePartitioningOptions.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/HivePartitioningOptions.java @@ -17,6 +17,7 @@ package com.google.cloud.bigquery; import com.google.common.base.MoreObjects; +import java.util.List; import java.util.Objects; /** HivePartitioningOptions currently supported types include: AVRO, CSV, JSON, ORC and Parquet. */ @@ -25,12 +26,14 @@ public final class HivePartitioningOptions { private final String mode; private final Boolean requirePartitionFilter; private final String sourceUriPrefix; + private final List fields; public static final class Builder { private String mode; private Boolean requirePartitionFilter; private String sourceUriPrefix; + private List fields; private Builder() {} @@ -38,6 +41,7 @@ private Builder(HivePartitioningOptions options) { this.mode = options.mode; this.requirePartitionFilter = options.requirePartitionFilter; this.sourceUriPrefix = options.sourceUriPrefix; + this.fields = options.fields; } /** @@ -78,6 +82,19 @@ public Builder setSourceUriPrefix(String sourceUriPrefix) { return this; } + /** + * [Output-only] For permanent external tables, this field is populated with the hive partition + * keys in the order they were inferred. + * + *

The types of the partition keys can be deduced by checking the table schema (which will + * include the partition keys). Not every API will populate this field in the output. For + * example, Tables.Get will populate it, but Tables.List will not contain this field. + */ + public Builder setFields(List fields) { + this.fields = fields; + return this; + } + /** Creates a {@link HivePartitioningOptions} object. */ public HivePartitioningOptions build() { return new HivePartitioningOptions(this); @@ -88,6 +105,7 @@ private HivePartitioningOptions(Builder builder) { this.mode = builder.mode; this.requirePartitionFilter = builder.requirePartitionFilter; this.sourceUriPrefix = builder.sourceUriPrefix; + this.fields = builder.fields; } /* Returns the mode of hive partitioning */ @@ -108,6 +126,11 @@ public String getSourceUriPrefix() { return sourceUriPrefix; } + /* Returns the fields of hive partitioning */ + public List getFields() { + return fields; + } + /** Returns a builder for the {@link HivePartitioningOptions} object. */ public Builder toBuilder() { return new Builder(this); @@ -135,13 +158,13 @@ public boolean equals(Object obj) { && Objects.equals(mode, ((HivePartitioningOptions) obj).getMode()) && Objects.equals( requirePartitionFilter, ((HivePartitioningOptions) obj).getRequirePartitionFilter()) - && Objects.equals( - sourceUriPrefix, ((HivePartitioningOptions) obj).getSourceUriPrefix()); + && Objects.equals(sourceUriPrefix, ((HivePartitioningOptions) obj).getSourceUriPrefix()) + && Objects.equals(fields, ((HivePartitioningOptions) obj).getFields()); } @Override public int hashCode() { - return Objects.hash(mode, sourceUriPrefix); + return Objects.hash(mode, sourceUriPrefix, fields); } com.google.api.services.bigquery.model.HivePartitioningOptions toPb() { @@ -150,6 +173,7 @@ com.google.api.services.bigquery.model.HivePartitioningOptions toPb() { options.setMode(mode); options.setRequirePartitionFilter(requirePartitionFilter); options.setSourceUriPrefix(sourceUriPrefix); + options.setFields(fields); return options; } @@ -165,6 +189,9 @@ static HivePartitioningOptions fromPb( if (options.getSourceUriPrefix() != null) { builder.setSourceUriPrefix(options.getSourceUriPrefix()); } + if (options.getFields() != null) { + builder.setFields(options.getFields()); + } return builder.build(); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/HivePartitioningOptionsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/HivePartitioningOptionsTest.java index 05bf05b9a..51baf918b 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/HivePartitioningOptionsTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/HivePartitioningOptionsTest.java @@ -18,6 +18,8 @@ import static com.google.common.truth.Truth.assertThat; +import java.util.Arrays; +import java.util.List; import org.junit.Test; public class HivePartitioningOptionsTest { @@ -25,11 +27,13 @@ public class HivePartitioningOptionsTest { private static final String MODE = "STRING"; private static final String SOURCE_URI_PREFIX = "gs://bucket/path_to_table"; private static final Boolean REQUIRE_PARTITION_FILTER = true; + private static final List FIELDS = Arrays.asList("FIELD1", "FIELD2"); private static final HivePartitioningOptions HIVE_PARTITIONING_OPTIONS = HivePartitioningOptions.newBuilder() .setMode(MODE) .setRequirePartitionFilter(REQUIRE_PARTITION_FILTER) .setSourceUriPrefix(SOURCE_URI_PREFIX) + .setFields(FIELDS) .build(); @Test diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index bf59c2b67..fda089c0e 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -5233,4 +5233,48 @@ public void testCloneTableCopyJob() throws InterruptedException { assertTrue(remoteTable.delete()); assertTrue(cloneTable.delete()); } + + @Test + public void testHivePartitioningOptionsFieldsFieldExistence() throws InterruptedException { + String tableName = "hive_partitioned_external_table"; + + // Create data on GCS + String sourceDirectory = "bigquery/hive-partitioning-table/example"; + BlobInfo blobInfo = BlobInfo.newBuilder(BUCKET, sourceDirectory + "/key=foo/data.json").build(); + assertNotNull( + "Failed to upload JSON to GCS", + storage.create(blobInfo, "{\"name\":\"bar\"}".getBytes(StandardCharsets.UTF_8))); + String sourceUri = "gs://" + BUCKET + "/" + sourceDirectory + "/*"; + String sourceUriPrefix = "gs://" + BUCKET + "/" + sourceDirectory + "/"; + + // Create the external table + HivePartitioningOptions hivePartitioningOptions = + HivePartitioningOptions.newBuilder() + .setMode("AUTO") + .setRequirePartitionFilter(true) + .setSourceUriPrefix(sourceUriPrefix) + .build(); + + TableId tableId = TableId.of(DATASET, tableName); + ExternalTableDefinition customTable = + ExternalTableDefinition.newBuilder(sourceUri, FormatOptions.json()) + .setAutodetect(true) + .setHivePartitioningOptions(hivePartitioningOptions) + .build(); + bigquery.create(TableInfo.of(tableId, customTable)); + + // Validate the existence of the field HivePartitioningOptions.fields + Table table = bigquery.getTable(tableId); + assertThat(table).isNotNull(); + HivePartitioningOptions options = + ((ExternalTableDefinition) table.getDefinition()).getHivePartitioningOptions(); + List fields = options.getFields(); + assertThat(fields).isNotNull(); + assertThat(fields).hasSize(1); + assertThat(fields).contains("key"); + + // Clean up + assertTrue(table.delete()); + assertTrue(storage.delete(blobInfo.getBlobId())); + } } diff --git a/pom.xml b/pom.xml index d458d7aac..22615bb3c 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ UTF-8 github google-cloud-bigquery-parent - v2-rev20230408-2.0.0 + v2-rev20230422-2.0.0 3.8.0 11.0.0