From eb4597314fad72bbdb666a832f0f15f732f40817 Mon Sep 17 00:00:00 2001 From: Jonathan Beluch Date: Tue, 7 Mar 2023 12:28:58 -0700 Subject: [PATCH] fix: external table definition parquet format options (#2535) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: set parquet format options for external tables * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: set parquet format options for external tables * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: set parquet format options for external tables --------- Co-authored-by: Owl Bot Co-authored-by: Neenu Shaji --- README.md | 4 ++-- .../cloud/bigquery/ExternalTableDefinition.java | 3 +++ .../com/google/cloud/bigquery/FormatOptions.java | 2 +- .../bigquery/ExternalTableDefinitionTest.java | 15 +++++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b01a38c0e..907a391fb 100644 --- a/README.md +++ b/README.md @@ -59,13 +59,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.23.0' +implementation 'com.google.cloud:google-cloud-bigquery:2.23.1' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.23.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.23.1" ``` ## Authentication diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index cb327a3a9..18c78e750 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -311,6 +311,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getDecimalTargetTypes() != null) { externalConfigurationPb.setDecimalTargetTypes(getDecimalTargetTypes()); } + if (getFormatOptions() != null && FormatOptions.PARQUET.equals(getFormatOptions().getType())) { + externalConfigurationPb.setParquetOptions(((ParquetOptions) getFormatOptions()).toPb()); + } if (getFormatOptions() != null && FormatOptions.AVRO.equals(getFormatOptions().getType())) { externalConfigurationPb.setAvroOptions(((AvroOptions) getFormatOptions()).toPb()); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java index c4f411175..9ea5600dd 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java @@ -107,7 +107,7 @@ public static FormatOptions googleSheets() { /** Default options for PARQUET format. */ public static FormatOptions parquet() { - return new FormatOptions(PARQUET); + return ParquetOptions.newBuilder().build(); } /** Default options for the ORC format. */ diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java index b7b9531e5..4a7409162 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java @@ -52,6 +52,7 @@ public class ExternalTableDefinitionTest { private static final Boolean AUTODETECT = true; private static final AvroOptions AVRO_OPTIONS = AvroOptions.newBuilder().build(); private static final CsvOptions CSV_OPTIONS = CsvOptions.newBuilder().build(); + private static final ParquetOptions PARQUET_OPTIONS = ParquetOptions.newBuilder().build(); private static final HivePartitioningOptions HIVE_PARTITIONING_OPTIONS = HivePartitioningOptions.newBuilder() .setMode("AUTO") @@ -71,6 +72,9 @@ public class ExternalTableDefinitionTest { private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION_AVRO = ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, AVRO_OPTIONS).build(); + private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION_PARQUET = + ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, PARQUET_OPTIONS).build(); + @Test public void testToBuilder() { compareExternalTableDefinition( @@ -136,6 +140,17 @@ public void testToAndFromPb() { externalTableDefinition, ExternalTableDefinition.fromPb(externalTableDefinition.toPb())); } + @Test + public void testToAndFromPbParquet() { + compareExternalTableDefinition( + EXTERNAL_TABLE_DEFINITION_PARQUET, + ExternalTableDefinition.fromPb(EXTERNAL_TABLE_DEFINITION_PARQUET.toPb())); + ExternalTableDefinition externalTableDefinition = + ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, PARQUET_OPTIONS).build(); + compareExternalTableDefinition( + externalTableDefinition, ExternalTableDefinition.fromPb(externalTableDefinition.toPb())); + } + private void compareExternalTableDefinition( ExternalTableDefinition expected, ExternalTableDefinition value) { assertEquals(expected, value);