From 6b988ad6acf4aef187fc8e64a4dec8b74520426a Mon Sep 17 00:00:00 2001 From: Brendan Greenley Date: Thu, 28 Sep 2023 14:14:51 -0400 Subject: [PATCH 1/3] Add support for storage billing models --- .../clirr-ignored-differences.xml | 5 ++++ .../google/cloud/bigquery/ConnectionImpl.java | 5 +++- .../com/google/cloud/bigquery/Dataset.java | 6 ++++ .../google/cloud/bigquery/DatasetInfo.java | 25 +++++++++++++++++ .../cloud/bigquery/ConnectionImplTest.java | 28 +++++++++++++++++++ .../cloud/bigquery/DatasetInfoTest.java | 6 ++++ .../google/cloud/bigquery/DatasetTest.java | 5 ++++ .../cloud/bigquery/it/ITBigQueryTest.java | 28 +++++++++++++++++++ 8 files changed, 107 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index eaeeded13..6600470f2 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -109,4 +109,9 @@ com/google/cloud/bigquery/DatasetInfo* *setExternalDatasetReference(*) + + 7013 + com/google/cloud/bigquery/DatasetInfo* + *setStorageBillingModel(*) + \ No newline at end of file diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java index 17a459312..2d0367790 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ConnectionImpl.java @@ -55,6 +55,7 @@ import java.io.IOException; import java.util.AbstractList; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -173,7 +174,9 @@ public BigQueryDryRunResult dryRun(String sql) throws BigQuerySQLException { List queryParametersPb = dryRunJob.getStatistics().getQuery().getUndeclaredQueryParameters(); List queryParameters = - Lists.transform(queryParametersPb, QUERY_PARAMETER_FROM_PB_FUNCTION); + queryParametersPb == null + ? Collections.emptyList() + : Lists.transform(queryParametersPb, QUERY_PARAMETER_FROM_PB_FUNCTION); QueryStatistics queryStatistics = JobStatistics.fromPb(dryRunJob); SessionInfo sessionInfo = queryStatistics.getSessionInfo() == null ? null : queryStatistics.getSessionInfo(); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java index 3ed4b8928..daadb9a85 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java @@ -158,6 +158,12 @@ public Builder setExternalDatasetReference(ExternalDatasetReference externalData return this; } + @Override + public Builder setStorageBillingModel(String storageBillingModel) { + infoBuilder.setStorageBillingModel(storageBillingModel); + return this; + } + @Override public Dataset build() { return new Dataset(bigquery, infoBuilder); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java index 44583e0c8..0dddc0e86 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java @@ -74,6 +74,7 @@ public Dataset apply(DatasetInfo datasetInfo) { private final Long defaultPartitionExpirationMs; private final String defaultCollation; private final ExternalDatasetReference externalDatasetReference; + private final String storageBillingModel; /** A builder for {@code DatasetInfo} objects. */ public abstract static class Builder { @@ -135,6 +136,12 @@ public abstract static class Builder { public abstract Builder setExternalDatasetReference( ExternalDatasetReference externalDatasetReference); + /** + * Optional. Storage billing model to be used for all tables in the dataset. Can be set to + * PHYSICAL. Default is LOGICAL. + */ + public abstract Builder setStorageBillingModel(String storageBillingModel); + /** * The default encryption key for all tables in the dataset. Once this property is set, all * newly-created partitioned tables in the dataset will have encryption key set to this value, @@ -192,6 +199,7 @@ static final class BuilderImpl extends Builder { private Long defaultPartitionExpirationMs; private String defaultCollation; private ExternalDatasetReference externalDatasetReference; + private String storageBillingModel; BuilderImpl() {} @@ -212,6 +220,7 @@ static final class BuilderImpl extends Builder { this.defaultPartitionExpirationMs = datasetInfo.defaultPartitionExpirationMs; this.defaultCollation = datasetInfo.defaultCollation; this.externalDatasetReference = datasetInfo.externalDatasetReference; + this.storageBillingModel = datasetInfo.storageBillingModel; } BuilderImpl(com.google.api.services.bigquery.model.Dataset datasetPb) { @@ -250,6 +259,7 @@ public Acl apply(Dataset.Access accessPb) { this.externalDatasetReference = ExternalDatasetReference.fromPb(datasetPb.getExternalDatasetReference()); } + this.storageBillingModel = datasetPb.getStorageBillingModel(); } @Override @@ -356,6 +366,12 @@ public Builder setExternalDatasetReference(ExternalDatasetReference externalData return this; } + @Override + public Builder setStorageBillingModel(String storageBillingModel) { + this.storageBillingModel = storageBillingModel; + return this; + } + @Override public DatasetInfo build() { return new DatasetInfo(this); @@ -379,6 +395,7 @@ public DatasetInfo build() { defaultPartitionExpirationMs = builder.defaultPartitionExpirationMs; defaultCollation = builder.defaultCollation; externalDatasetReference = builder.externalDatasetReference; + storageBillingModel = builder.storageBillingModel; } /** Returns the dataset identity. */ @@ -508,6 +525,10 @@ public String getDefaultCollation() { return defaultCollation; } + public String getStorageBillingModel() { + return storageBillingModel; + } + /** * Returns information about the external metadata storage where the dataset is defined. Filled * out when the dataset type is EXTERNAL. @@ -540,6 +561,7 @@ public String toString() { .add("defaultPartitionExpirationMs", defaultPartitionExpirationMs) .add("defaultCollation", defaultCollation) .add("externalDatasetReference", externalDatasetReference) + .add("storageBillingModel", storageBillingModel) .toString(); } @@ -621,6 +643,9 @@ public Dataset.Access apply(Acl acl) { if (externalDatasetReference != null) { datasetPb.setExternalDatasetReference(externalDatasetReference.toPb()); } + if (storageBillingModel != null) { + datasetPb.setStorageBillingModel(storageBillingModel); + } return datasetPb; } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java index d6348f053..dff73d6bd 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ConnectionImplTest.java @@ -237,6 +237,34 @@ public void testQueryDryRun() throws BigQuerySQLException { .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); } + @Test + public void testQueryDryRunNoQueryParameters() throws BigQuerySQLException { + com.google.api.services.bigquery.model.JobStatistics2 queryMock = + new com.google.api.services.bigquery.model.JobStatistics2() + .setSchema(FAST_QUERY_TABLESCHEMA); + com.google.api.services.bigquery.model.JobStatistics jobStatsMock = + new com.google.api.services.bigquery.model.JobStatistics() + .setCreationTime(1234L) + .setStartTime(5678L) + .setQuery(queryMock); + com.google.api.services.bigquery.model.JobConfigurationQuery jobConfigurationQuery = + new com.google.api.services.bigquery.model.JobConfigurationQuery(); + com.google.api.services.bigquery.model.JobConfiguration jobConfig = + new com.google.api.services.bigquery.model.JobConfiguration() + .setQuery(jobConfigurationQuery); + com.google.api.services.bigquery.model.Job mockDryRunJob = + new com.google.api.services.bigquery.model.Job() + .setStatistics(jobStatsMock) + .setConfiguration(jobConfig); + when(bigqueryRpcMock.createJobForQuery(any(com.google.api.services.bigquery.model.Job.class))) + .thenReturn(mockDryRunJob); + BigQueryDryRunResult dryRunResult = connection.dryRun(DRY_RUN_SQL); + assertEquals(0, dryRunResult.getQueryParameters().size()); + assertEquals(QUERY_SCHEMA, dryRunResult.getSchema()); + verify(bigqueryRpcMock, times(1)) + .createJobForQuery(any(com.google.api.services.bigquery.model.Job.class)); + } + @Test public void testParseDataTask() throws InterruptedException { BlockingQueue, Boolean>> pageCache = diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java index c91cbc2f3..df62b7dac 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java @@ -58,6 +58,7 @@ public class DatasetInfoTest { private static final DatasetId DATASET_ID_COMPLETE = DatasetId.of("project", "dataset"); private static final EncryptionConfiguration DATASET_ENCRYPTION_CONFIGURATION = EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build(); + private static final String STORAGE_BILLING_MODEL = "LOGICAL"; private static final ExternalDatasetReference EXTERNAL_DATASET_REFERENCE = ExternalDatasetReference.newBuilder() @@ -79,6 +80,7 @@ public class DatasetInfoTest { .setLabels(LABELS) .setDefaultEncryptionConfiguration(DATASET_ENCRYPTION_CONFIGURATION) .setDefaultPartitionExpirationMs(DEFAULT_PARTITION__EXPIRATION) + .setStorageBillingModel(STORAGE_BILLING_MODEL) .build(); private static final DatasetInfo DATASET_INFO_COMPLETE = DATASET_INFO @@ -170,6 +172,7 @@ public void testBuilder() { assertEquals( EXTERNAL_DATASET_REFERENCE, DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE.getExternalDatasetReference()); + assertEquals(STORAGE_BILLING_MODEL, DATASET_INFO_COMPLETE.getStorageBillingModel()); } @Test @@ -190,6 +193,7 @@ public void testOf() { assertNull(datasetInfo.getDefaultPartitionExpirationMs()); assertTrue(datasetInfo.getLabels().isEmpty()); assertNull(datasetInfo.getExternalDatasetReference()); + assertNull(datasetInfo.getStorageBillingModel()); datasetInfo = DatasetInfo.of(DATASET_ID); assertEquals(DATASET_ID, datasetInfo.getDatasetId()); @@ -207,6 +211,7 @@ public void testOf() { assertNull(datasetInfo.getDefaultPartitionExpirationMs()); assertTrue(datasetInfo.getLabels().isEmpty()); assertNull(datasetInfo.getExternalDatasetReference()); + assertNull(datasetInfo.getStorageBillingModel()); } @Test @@ -243,5 +248,6 @@ private void compareDatasets(DatasetInfo expected, DatasetInfo value) { assertEquals( expected.getDefaultPartitionExpirationMs(), value.getDefaultPartitionExpirationMs()); assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference()); + assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java index b244cf260..bc42976b2 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java @@ -66,6 +66,7 @@ public class DatasetTest { private static final String SELF_LINK = "http://bigquery/p/d"; private static final DatasetInfo DATASET_INFO = DatasetInfo.newBuilder(DATASET_ID).build(); private static final Field FIELD = Field.of("FieldName", LegacySQLTypeName.INTEGER); + private static final String STORAGE_BILLING_MODEL = "LOGICAL"; private static final StandardTableDefinition TABLE_DEFINITION = StandardTableDefinition.of(Schema.of(FIELD)); private static final ViewDefinition VIEW_DEFINITION = ViewDefinition.of("QUERY"); @@ -120,6 +121,7 @@ public void testBuilder() { .setLocation(LOCATION) .setSelfLink(SELF_LINK) .setLabels(LABELS) + .setStorageBillingModel(STORAGE_BILLING_MODEL) .build(); assertEquals(DATASET_ID, builtDataset.getDatasetId()); assertEquals(ACCESS_RULES, builtDataset.getAcl()); @@ -133,6 +135,7 @@ public void testBuilder() { assertEquals(LOCATION, builtDataset.getLocation()); assertEquals(SELF_LINK, builtDataset.getSelfLink()); assertEquals(LABELS, builtDataset.getLabels()); + assertEquals(STORAGE_BILLING_MODEL, builtDataset.getStorageBillingModel()); } @Test @@ -340,6 +343,7 @@ public void testExternalDatasetReference() { .setSelfLink(SELF_LINK) .setLabels(LABELS) .setExternalDatasetReference(EXTERNAL_DATASET_REFERENCE) + .setStorageBillingModel(STORAGE_BILLING_MODEL) .build(); assertEquals( EXTERNAL_DATASET_REFERENCE, @@ -369,5 +373,6 @@ private void compareDatasetInfo(DatasetInfo expected, DatasetInfo value) { assertEquals(expected.getDefaultTableLifetime(), value.getDefaultTableLifetime()); assertEquals(expected.getLastModified(), value.getLastModified()); assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference()); + assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index cf180f9a3..549066c5f 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2989,6 +2989,34 @@ public void testConnectionImplDryRun() throws SQLException { assertEquals(StatementType.SELECT, queryStatistics.getStatementType()); } + @Test + public void testConnectionImplDryRunNoQueryParameters() throws SQLException { + String query = + String.format( + "select StringField, BigNumericField, BooleanField, BytesField, IntegerField, " + + "TimestampField, FloatField, NumericField, TimeField, DateField, DateTimeField, " + + "GeographyField, RecordField.BytesField, RecordField.BooleanField, " + + "IntegerArrayField from %s order by TimestampField", + TABLE_ID_FASTQUERY_BQ_RESULTSET.getTable()); + ConnectionSettings connectionSettings = + ConnectionSettings.newBuilder() + .setDefaultDataset(DatasetId.of(DATASET)) + .setCreateSession(true) + .build(); + Connection connection = bigquery.createConnection(connectionSettings); + BigQueryDryRunResult bigQueryDryRunResultSet = connection.dryRun(query); + assertNotNull(bigQueryDryRunResultSet.getSchema()); + assertEquals( + BQ_RESULTSET_EXPECTED_SCHEMA, bigQueryDryRunResultSet.getSchema()); // match the schema + List queryParameters = bigQueryDryRunResultSet.getQueryParameters(); + assertEquals(0, queryParameters.size()); + QueryStatistics queryStatistics = bigQueryDryRunResultSet.getStatistics().getQueryStatistics(); + assertNotNull(queryStatistics); + SessionInfo sessionInfo = bigQueryDryRunResultSet.getStatistics().getSessionInfo(); + assertNotNull(sessionInfo.getSessionId()); + assertEquals(StatementType.SELECT, queryStatistics.getStatementType()); + } + @Test // This test case test the order of the records, making sure that the result is not jumbled up due // to the multithreaded BigQueryResult implementation From b8d6a03b3ff9f936161f1556a9195c6f3abda13d Mon Sep 17 00:00:00 2001 From: brendang Date: Thu, 19 Oct 2023 17:06:41 -0400 Subject: [PATCH 2/3] Update tests in ITBigQueryTest.java --- .../cloud/bigquery/it/ITBigQueryTest.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 1ff859745..909500be8 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -202,6 +202,7 @@ public class ITBigQueryTest { private static final String ROUTINE_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); private static final String RANDOM_ID = UUID.randomUUID().toString().substring(0, 8); + private static final String STORAGE_BILLING_MODEL = "LOGICAL"; private static final String CLOUD_SAMPLES_DATA = Optional.fromNullable(System.getenv("CLOUD_SAMPLES_DATA_BUCKET")).or("cloud-samples-data"); private static final Map LABELS = @@ -965,6 +966,7 @@ public void testGetDatasetWithSelectedFields() { assertNull(dataset.getLastModified()); assertNull(dataset.getLocation()); assertNull(dataset.getSelfLink()); + assertNull(dataset.getStorageBillingModel()); } @Test @@ -980,6 +982,7 @@ public void testUpdateDataset() { assertThat(dataset.getDatasetId().getDataset()).isEqualTo(OTHER_DATASET); assertThat(dataset.getDescription()).isEqualTo("Some Description"); assertThat(dataset.getLabels()).containsExactly("a", "b"); + assertThat(dataset.getStorageBillingModel()).isNull(); Map updateLabels = new HashMap<>(); updateLabels.put("x", "y"); @@ -990,9 +993,11 @@ public void testUpdateDataset() { .toBuilder() .setDescription("Updated Description") .setLabels(updateLabels) + .setStorageBillingModel("LOGICAL") .build()); assertThat(updatedDataset.getDescription()).isEqualTo("Updated Description"); assertThat(updatedDataset.getLabels()).containsExactly("x", "y"); + assertThat(updatedDataset.getStorageBillingModel()).isEqualTo("LOGICAL"); updatedDataset = bigquery.update(updatedDataset.toBuilder().setLabels(null).build()); assertThat(updatedDataset.getLabels()).isEmpty(); @@ -1022,6 +1027,7 @@ public void testUpdateDatasetWithSelectedFields() { assertNull(updatedDataset.getLastModified()); assertNull(updatedDataset.getLocation()); assertNull(updatedDataset.getSelfLink()); + assertNull(updatedDataset.getStorageBillingModel()); assertTrue(dataset.delete()); } @@ -1283,6 +1289,23 @@ public void testCreateTableWithConstraints() { bigquery.delete(tableId); } + @Test + public void testCreateDatasetWithSpecifiedStorageBillingModel() { + String billingModelDataset = RemoteBigQueryHelper.generateDatasetName(); + DatasetInfo info = + DatasetInfo.newBuilder(billingModelDataset) + .setDescription(DESCRIPTION) + .setStorageBillingModel(STORAGE_BILLING_MODEL) + .setLabels(LABELS) + .build(); + bigquery.create(info); + + Dataset dataset = bigquery.getDataset(DatasetId.of(billingModelDataset)); + assertEquals(STORAGE_BILLING_MODEL, dataset.getStorageBillingModel()); + + RemoteBigQueryHelper.forceDelete(bigquery, billingModelDataset); + } + @Test public void testCreateDatasetWithDefaultCollation() { String collationDataset = RemoteBigQueryHelper.generateDatasetName(); From 56545d1c6cee0ec170e220df73018e8103fc31d2 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 20 Oct 2023 12:25:17 +0000 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2a30c8073..879d879e2 100644 --- a/README.md +++ b/README.md @@ -53,20 +53,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.24.0') +implementation platform('com.google.cloud:libraries-bom:26.25.0') implementation 'com.google.cloud:google-cloud-bigquery' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.33.1' +implementation 'com.google.cloud:google-cloud-bigquery:2.33.2' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.33.1" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.33.2" ``` @@ -351,7 +351,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquery/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquery.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.33.1 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.33.2 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles