From e12122c4472ed4c3d00fc8c7515be210bbf68df3 Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Tue, 1 Feb 2022 12:03:32 -0500 Subject: [PATCH] docs(samples): fix CopyMultipleTables sample IT failure and improve a few other samples (#1817) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs(samples): fix CopyMultipleTables sample IT failure and improve a few other samples Fixes issue: https://github.com/googleapis/java-bigquery/issues/1805 * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .../example/bigquery/CopyMultipleTables.java | 17 +++++---- .../bigquery/QueryExternalBigtablePerm.java | 3 +- .../bigquery/QueryExternalBigtableTemp.java | 3 +- .../bigquery/CopyMultipleTablesIT.java | 36 +++++++++++++++---- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/bigquery/CopyMultipleTables.java b/samples/snippets/src/main/java/com/example/bigquery/CopyMultipleTables.java index 9c3c736fc..2a7e02344 100644 --- a/samples/snippets/src/main/java/com/example/bigquery/CopyMultipleTables.java +++ b/samples/snippets/src/main/java/com/example/bigquery/CopyMultipleTables.java @@ -32,25 +32,30 @@ public static void main(String[] args) { // TODO(developer): Replace these variables before running the sample. String destinationDatasetName = "MY_DATASET_NAME"; String destinationTableId = "MY_TABLE_NAME"; - copyMultipleTables(destinationDatasetName, destinationTableId); + String sourceTable1Id = "MY_SOURCE_TABLE_1"; + String sourceTable2Id = "MY_SOURCE_TABLE_2"; + copyMultipleTables(destinationDatasetName, destinationTableId, sourceTable1Id, sourceTable2Id); } - public static void copyMultipleTables(String destinationDatasetName, String destinationTableId) { + public static void copyMultipleTables( + String destinationDatasetName, + String destinationTableId, + String sourceTable1Id, + String sourceTable2Id) { try { // Initialize client that will be used to send requests. This client only needs to be created // once, and can be reused for multiple requests. BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); TableId destinationTable = TableId.of(destinationDatasetName, destinationTableId); + TableId sourceTable1 = TableId.of(destinationDatasetName, sourceTable1Id); + TableId sourceTable2 = TableId.of(destinationDatasetName, sourceTable2Id); // For more information on CopyJobConfiguration see: // https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/bigquery/JobConfiguration.html CopyJobConfiguration configuration = CopyJobConfiguration.newBuilder( - destinationTable, - Arrays.asList( - TableId.of(destinationDatasetName, "table1"), - TableId.of(destinationDatasetName, "table2"))) + destinationTable, Arrays.asList(sourceTable1, sourceTable2)) .build(); // For more information on Job see: diff --git a/samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtablePerm.java b/samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtablePerm.java index 916269707..fa5c30c16 100644 --- a/samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtablePerm.java +++ b/samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtablePerm.java @@ -17,6 +17,8 @@ package com.example.bigquery; // [START bigquery_query_external_bigtable_perm] + +import com.google.api.client.util.Base64; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; @@ -29,7 +31,6 @@ import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TableResult; import com.google.common.collect.ImmutableList; -import org.apache.commons.codec.binary.Base64; // Sample to queries an external bigtable data source using a permanent table public class QueryExternalBigtablePerm { diff --git a/samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtableTemp.java b/samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtableTemp.java index 51646fcb6..6a0463107 100644 --- a/samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtableTemp.java +++ b/samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtableTemp.java @@ -17,6 +17,8 @@ package com.example.bigquery; // [START bigquery_query_external_bigtable_temp] + +import com.google.api.client.util.Base64; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; @@ -27,7 +29,6 @@ import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.TableResult; import com.google.common.collect.ImmutableList; -import org.apache.commons.codec.binary.Base64; // Sample to queries an external bigtable data source using a temporary table public class QueryExternalBigtableTemp { diff --git a/samples/snippets/src/test/java/com/example/bigquery/CopyMultipleTablesIT.java b/samples/snippets/src/test/java/com/example/bigquery/CopyMultipleTablesIT.java index 685db52d1..e0d175c86 100644 --- a/samples/snippets/src/test/java/com/example/bigquery/CopyMultipleTablesIT.java +++ b/samples/snippets/src/test/java/com/example/bigquery/CopyMultipleTablesIT.java @@ -19,6 +19,9 @@ import static com.google.common.truth.Truth.assertThat; import static junit.framework.TestCase.assertNotNull; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.UUID; @@ -32,22 +35,27 @@ public class CopyMultipleTablesIT { private final Logger log = Logger.getLogger(this.getClass().getName()); + private String datasetName; private String tableName; + private String sourceTable1Name; + private String sourceTable2Name; private ByteArrayOutputStream bout; private PrintStream out; private PrintStream originalPrintStream; - private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME"); + private static final String PROJECT_ID = requireEnvVar("GOOGLE_CLOUD_PROJECT"); - private static void requireEnvVar(String varName) { + private static String requireEnvVar(String varName) { + String value = System.getenv(varName); assertNotNull( "Environment variable " + varName + " is required to perform these tests.", System.getenv(varName)); + return value; } @BeforeClass public static void checkRequirements() { - requireEnvVar("BIGQUERY_DATASET_NAME"); + requireEnvVar("GOOGLE_CLOUD_PROJECT"); } @Before @@ -56,15 +64,30 @@ public void setUp() throws Exception { out = new PrintStream(bout); originalPrintStream = System.out; System.setOut(out); + // Create a new destination table for each test since existing table cannot be overwritten + datasetName = "MY_DATASET_NAME_TEST_" + UUID.randomUUID().toString().substring(0, 8); tableName = "COPY_MULTIPLE_TABLE_TEST" + UUID.randomUUID().toString().substring(0, 8); - CreateTable.createTable(BIGQUERY_DATASET_NAME, tableName, null); + sourceTable1Name = + "COPY_MULTIPLE_TABLE_SOURCE1_TEST" + UUID.randomUUID().toString().substring(0, 8); + sourceTable2Name = + "COPY_MULTIPLE_TABLE_SOURCE2_TEST" + UUID.randomUUID().toString().substring(0, 8); + CreateDataset.createDataset(datasetName); + + Schema schema = + Schema.of( + Field.of("timestampField", StandardSQLTypeName.TIMESTAMP), + Field.of("stringField", StandardSQLTypeName.STRING), + Field.of("booleanField", StandardSQLTypeName.BOOL)); + CreateTable.createTable(datasetName, tableName, schema); + CreateTable.createTable(datasetName, sourceTable1Name, schema); + CreateTable.createTable(datasetName, sourceTable2Name, schema); } @After public void tearDown() { // Clean up - DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName); + DeleteDataset.deleteDataset(PROJECT_ID, datasetName); // restores print statements in the original method System.out.flush(); System.setOut(originalPrintStream); @@ -73,7 +96,8 @@ public void tearDown() { @Test public void testCopyMultipleTables() { - CopyMultipleTables.copyMultipleTables(BIGQUERY_DATASET_NAME, tableName); + CopyMultipleTables.copyMultipleTables( + datasetName, tableName, sourceTable1Name, sourceTable2Name); assertThat(bout.toString()).contains("Table copied successfully."); } }