From 4c01698e571d7adbaf914984cdf65f6c35e4edb8 Mon Sep 17 00:00:00 2001 From: emkornfield Date: Fri, 5 May 2023 11:48:12 -0400 Subject: [PATCH] feat: Allow passing autodetect_schema on table update (#2661) Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - Didn't open an issue because this seems trivial, happy to adapt if necessary. - [x ] Ensure the tests and linter pass (mvn test verify - [x ] Code coverage does not decrease (if any source code was changed) - [x ] Appropriate docs were updated (if necessary) --- .../com/google/cloud/bigquery/BigQuery.java | 8 ++++ .../cloud/bigquery/spi/v2/BigQueryRpc.java | 1 + .../bigquery/spi/v2/HttpBigQueryRpc.java | 1 + .../cloud/bigquery/BigQueryImplTest.java | 18 +++++++++ .../cloud/bigquery/it/ITBigQueryTest.java | 39 +++++++++++++++++++ 5 files changed, 67 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index bfcdc542a..8f11d03fa 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -376,6 +376,14 @@ public static TableOption fields(TableField... fields) { return new TableOption( BigQueryRpc.Option.FIELDS, Helper.selector(TableField.REQUIRED_FIELDS, fields)); } + + /** + * Returns an option to specify the schema of the table (only applicable for external tables) + * should be autodetected when updating the table from the underlying source. + */ + public static TableOption autodetectSchema(boolean autodetect) { + return new TableOption(BigQueryRpc.Option.AUTODETECT_SCHEMA, autodetect); + } } /* Class for specifying IAM options. */ diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index eecf5d36f..33480a91b 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -46,6 +46,7 @@ enum Option { DELETE_CONTENTS("deleteContents"), ALL_DATASETS("all"), ALL_USERS("allUsers"), + AUTODETECT_SCHEMA("autodetectSchema"), LABEL_FILTER("filter"), MIN_CREATION_TIME("minCreationTime"), MAX_CREATION_TIME("maxCreationTime"), diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 6bad06199..a85e4b6bf 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -279,6 +279,7 @@ public Table patch(Table table, Map options) { .patch(reference.getProjectId(), reference.getDatasetId(), reference.getTableId(), table) .setPrettyPrint(false) .setFields(Option.FIELDS.getString(options)) + .setAutodetectSchema(BigQueryRpc.Option.AUTODETECT_SCHEMA.getBoolean(options)) .execute(); } catch (IOException ex) { throw translate(ex); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java index 80ba366f8..3bbdb98fd 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java @@ -1260,6 +1260,24 @@ public void testUpdateTableWithSelectedFields() { .patch(eq(updatedTableInfoWithProject.toPb()), capturedOptions.capture()); } + @Test + public void testUpdateTableWithAutoDetectSchema() { + TableInfo updatedTableInfo = TABLE_INFO.toBuilder().setDescription("newDescription").build(); + TableInfo updatedTableInfoWithProject = + TABLE_INFO_WITH_PROJECT.toBuilder().setDescription("newDescription").build(); + when(bigqueryRpcMock.patch(eq(updatedTableInfoWithProject.toPb()), capturedOptions.capture())) + .thenReturn(updatedTableInfoWithProject.toPb()); + bigquery = options.getService(); + Table table = bigquery.update(updatedTableInfo, BigQuery.TableOption.autodetectSchema(true)); + Boolean selector = + (Boolean) capturedOptions.getValue().get(BigQueryRpc.Option.AUTODETECT_SCHEMA); + assertTrue(selector); + assertEquals( + new Table(bigquery, new TableInfo.BuilderImpl(updatedTableInfoWithProject)), table); + verify(bigqueryRpcMock) + .patch(eq(updatedTableInfoWithProject.toPb()), capturedOptions.capture()); + } + @Test public void testInsertAllWithRowIdShouldRetry() { Map row1 = ImmutableMap.of("field", "value1"); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index fda089c0e..6d61fd211 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1566,6 +1566,45 @@ public void testSetPermExternalTableSchema() { assertTrue(remoteTable.delete()); } + @Test + public void testUpdatePermExternableTableWithAutodetectSchemaUpdatesSchema() { + String tableName = "test_create_external_table_perm_with_auto_detect"; + TableId tableId = TableId.of(DATASET, tableName); + Schema setSchema = Schema.of(TIMESTAMP_FIELD_SCHEMA, STRING_FIELD_SCHEMA); + + ExternalTableDefinition externalTableDefinition = + ExternalTableDefinition.newBuilder( + "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()) + .setSchema(setSchema) + .build(); + TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition); + Table createdTable = bigquery.create(tableInfo); + + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + Table remoteTable = bigquery.getTable(DATASET, tableName); + assertNotNull(remoteTable); + assertEquals(setSchema, remoteTable.getDefinition().getSchema()); + + Table updatedTable = + bigquery.update( + createdTable + .toBuilder() + .setDefinition( + ((ExternalTableDefinition) createdTable.getDefinition()) + .toBuilder() + .setSchema(null) + .setAutodetect(true) + .build()) + .build(), + BigQuery.TableOption.autodetectSchema(true)); + // Schema should change. + assertTrue(!updatedTable.getDefinition().getSchema().equals(setSchema)); + + assertTrue(remoteTable.delete()); + } + @Test public void testCreateViewTable() throws InterruptedException { String tableName = "test_create_view_table";