From 88c88d5176e121d594a536759581514c04e86610 Mon Sep 17 00:00:00 2001 From: Farhan Ahmed Date: Wed, 14 Jun 2023 11:47:40 -0400 Subject: [PATCH 1/2] feat: Partial Projection of Table Metadata --- .../com/google/cloud/bigquery/BigQuery.java | 19 +++ .../java/com/google/cloud/bigquery/Table.java | 4 +- .../cloud/bigquery/spi/v2/BigQueryRpc.java | 5 +- .../bigquery/spi/v2/HttpBigQueryRpc.java | 8 ++ .../cloud/bigquery/it/ITBigQueryTest.java | 128 ++++++++++++++++++ 5 files changed, 160 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 8f11d03fa..271278357 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -121,6 +121,20 @@ public String getSelector() { } } + /** + * Metadata of a BigQuery Table. + * + * @see Table + * Resource + */ + enum TableMetadataView { + BASIC, + FULL, + STORAGE_STATS, + TABLE_METADATA_VIEW_UNSPECIFIED; + } + /** * Fields of a BigQuery Model resource. * @@ -384,6 +398,11 @@ public static TableOption fields(TableField... fields) { public static TableOption autodetectSchema(boolean autodetect) { return new TableOption(BigQueryRpc.Option.AUTODETECT_SCHEMA, autodetect); } + + /** Returns an option to specify the metadata of the table. */ + public static TableOption tableMetadataView(TableMetadataView tableMetadataView) { + return new TableOption(BigQueryRpc.Option.TABLE_METADATA_VIEW, tableMetadataView); + } } /* Class for specifying IAM options. */ diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java index 3e67e8d75..1235c45ec 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Table.java @@ -49,10 +49,10 @@ public static class Builder extends TableInfo.Builder { private final BigQuery bigquery; private final TableInfo.BuilderImpl infoBuilder; - Builder(BigQuery bigquery, TableId tableId, TableDefinition defintion) { + Builder(BigQuery bigquery, TableId tableId, TableDefinition definition) { this.bigquery = bigquery; this.infoBuilder = new TableInfo.BuilderImpl(); - this.infoBuilder.setTableId(tableId).setDefinition(defintion); + this.infoBuilder.setTableId(tableId).setDefinition(definition); } Builder(Table table) { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index 33480a91b..341640919 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -40,7 +40,7 @@ @InternalExtensionOnly public interface BigQueryRpc extends ServiceRpc { - // These options are part of the Google Cloud BigQuery query parameters + // These options are part of the Google Cloud BigQuery query parameters. enum Option { FIELDS("fields"), DELETE_CONTENTS("deleteContents"), @@ -56,7 +56,8 @@ enum Option { START_INDEX("startIndex"), STATE_FILTER("stateFilter"), TIMEOUT("timeoutMs"), - REQUESTED_POLICY_VERSION("requestedPolicyVersion"); + REQUESTED_POLICY_VERSION("requestedPolicyVersion"), + TABLE_METADATA_VIEW("view"); private final String value; diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index a63e3cc2b..96a10deb3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -294,6 +294,7 @@ public Table getTable( .get(projectId, datasetId, tableId) .setPrettyPrint(false) .setFields(Option.FIELDS.getString(options)) + .setView(getTableMetadataOption(options)) .execute(); } catch (IOException ex) { BigQueryException serviceException = translate(ex); @@ -304,6 +305,13 @@ public Table getTable( } } + private String getTableMetadataOption(Map options) { + if (options.containsKey(Option.TABLE_METADATA_VIEW)) { + return options.get(Option.TABLE_METADATA_VIEW).toString(); + } + return "STORAGE_STATS"; + } + @Override public Tuple> listTables( String projectId, String datasetId, Map options) { diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index cdfcb8003..43d3a1df4 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -49,6 +49,7 @@ import com.google.cloud.bigquery.BigQuery.JobListOption; import com.google.cloud.bigquery.BigQuery.JobOption; import com.google.cloud.bigquery.BigQuery.TableField; +import com.google.cloud.bigquery.BigQuery.TableMetadataView; import com.google.cloud.bigquery.BigQuery.TableOption; import com.google.cloud.bigquery.BigQueryDryRunResult; import com.google.cloud.bigquery.BigQueryError; @@ -1532,6 +1533,133 @@ public void testCreateAndGetTable() { assertTrue(remoteTable.delete()); } + @Test + public void testCreateAndGetTableWithBasicTableMetadataView() { + String tableName = "test_create_and_get_table_with_basic_metadata_view"; + TableId tableId = TableId.of(DATASET, tableName); + TimePartitioning partitioning = TimePartitioning.of(Type.DAY); + Clustering clustering = + Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); + StandardTableDefinition tableDefinition = + StandardTableDefinition.newBuilder() + .setSchema(TABLE_SCHEMA) + .setTimePartitioning(partitioning) + .setClustering(clustering) + .build(); + Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + TableOption tableOption = BigQuery.TableOption.tableMetadataView(TableMetadataView.BASIC); + Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption); + assertNotNull(remoteTable); + assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); + assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); + // Next four values are considered transient fields that should not be calculated + assertNull(remoteTable.getLastModifiedTime()); + assertNull(remoteTable.getDefinition().getNumBytes()); + assertNull(remoteTable.getDefinition().getNumLongTermBytes()); + assertNull(remoteTable.getDefinition().getNumRows()); + assertTrue(remoteTable.delete()); + } + + @Test + public void testCreateAndGetTableWithFullTableMetadataView() { + String tableName = "test_create_and_get_table_with_full_metadata_view"; + TableId tableId = TableId.of(DATASET, tableName); + TimePartitioning partitioning = TimePartitioning.of(Type.DAY); + Clustering clustering = + Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); + StandardTableDefinition tableDefinition = + StandardTableDefinition.newBuilder() + .setSchema(TABLE_SCHEMA) + .setTimePartitioning(partitioning) + .setClustering(clustering) + .build(); + Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + TableOption tableOption = BigQuery.TableOption.tableMetadataView(TableMetadataView.FULL); + Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption); + assertNotNull(remoteTable); + assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); + assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); + assertNotNull(remoteTable.getLastModifiedTime()); + assertNotNull(remoteTable.getDefinition().getNumBytes()); + assertNotNull(remoteTable.getDefinition().getNumLongTermBytes()); + assertNotNull(remoteTable.getDefinition().getNumRows()); + assertTrue(remoteTable.delete()); + } + + @Test + public void testCreateAndGetTableWithStorageStatsTableMetadataView() { + String tableName = "test_create_and_get_table_with_storage_stats_metadata_view"; + TableId tableId = TableId.of(DATASET, tableName); + TimePartitioning partitioning = TimePartitioning.of(Type.DAY); + Clustering clustering = + Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); + StandardTableDefinition tableDefinition = + StandardTableDefinition.newBuilder() + .setSchema(TABLE_SCHEMA) + .setTimePartitioning(partitioning) + .setClustering(clustering) + .build(); + Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + TableOption tableOption = + BigQuery.TableOption.tableMetadataView(TableMetadataView.STORAGE_STATS); + Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption); + assertNotNull(remoteTable); + assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); + assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); + assertNotNull(remoteTable.getLastModifiedTime()); + assertNotNull(remoteTable.getDefinition().getNumBytes()); + assertNotNull(remoteTable.getDefinition().getNumLongTermBytes()); + assertNotNull(remoteTable.getDefinition().getNumRows()); + assertTrue(remoteTable.delete()); + } + + @Test + public void testCreateAndGetTableWithUnspecifiedTableMetadataView() { + String tableName = "test_create_and_get_table_with_unspecified_metadata_view"; + TableId tableId = TableId.of(DATASET, tableName); + TimePartitioning partitioning = TimePartitioning.of(Type.DAY); + Clustering clustering = + Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build(); + StandardTableDefinition tableDefinition = + StandardTableDefinition.newBuilder() + .setSchema(TABLE_SCHEMA) + .setTimePartitioning(partitioning) + .setClustering(clustering) + .build(); + Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition)); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + TableOption tableOption = + BigQuery.TableOption.tableMetadataView(TableMetadataView.TABLE_METADATA_VIEW_UNSPECIFIED); + Table remoteTable = bigquery.getTable(DATASET, tableName, tableOption); + assertNotNull(remoteTable); + assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType()); + assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); + assertNotNull(remoteTable.getLastModifiedTime()); + assertNotNull(remoteTable.getDefinition().getNumBytes()); + assertNotNull(remoteTable.getDefinition().getNumLongTermBytes()); + assertNotNull(remoteTable.getDefinition().getNumRows()); + assertTrue(remoteTable.delete()); + } + @Test public void testCreateAndGetTableWithSelectedField() { String tableName = "test_create_and_get_selected_fields_table"; From 5f02fa52abb600398e1e52962749c6f72a20c6b2 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 16 Jun 2023 18:15:24 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b7a5c463f..091fd0d7c 100644 --- a/README.md +++ b/README.md @@ -53,20 +53,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.16.0') +implementation platform('com.google.cloud:libraries-bom:26.17.0') implementation 'com.google.cloud:google-cloud-bigquery' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.27.0' +implementation 'com.google.cloud:google-cloud-bigquery:2.27.1' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.27.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.27.1" ``` @@ -350,7 +350,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquery/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquery.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.27.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.27.1 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles