Skip to content

Commit

Permalink
docs(samples): fix CopyMultipleTables sample IT failure and improve a…
Browse files Browse the repository at this point in the history
… few other samples (#1817)

* docs(samples): fix CopyMultipleTables sample IT failure and improve a few other samples

Fixes issue: #1805

* 🦉 Updates from OwlBot

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
  • Loading branch information
stephaniewang526 and gcf-owl-bot[bot] committed Feb 1, 2022
1 parent d48ae41 commit e12122c
Show file tree
Hide file tree
Showing 4 changed files with 45 additions and 14 deletions.
Expand Up @@ -32,25 +32,30 @@ public static void main(String[] args) {
// TODO(developer): Replace these variables before running the sample.
String destinationDatasetName = "MY_DATASET_NAME";
String destinationTableId = "MY_TABLE_NAME";
copyMultipleTables(destinationDatasetName, destinationTableId);
String sourceTable1Id = "MY_SOURCE_TABLE_1";
String sourceTable2Id = "MY_SOURCE_TABLE_2";
copyMultipleTables(destinationDatasetName, destinationTableId, sourceTable1Id, sourceTable2Id);
}

public static void copyMultipleTables(String destinationDatasetName, String destinationTableId) {
public static void copyMultipleTables(
String destinationDatasetName,
String destinationTableId,
String sourceTable1Id,
String sourceTable2Id) {
try {
// Initialize client that will be used to send requests. This client only needs to be created
// once, and can be reused for multiple requests.
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();

TableId destinationTable = TableId.of(destinationDatasetName, destinationTableId);
TableId sourceTable1 = TableId.of(destinationDatasetName, sourceTable1Id);
TableId sourceTable2 = TableId.of(destinationDatasetName, sourceTable2Id);

// For more information on CopyJobConfiguration see:
// https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/bigquery/JobConfiguration.html
CopyJobConfiguration configuration =
CopyJobConfiguration.newBuilder(
destinationTable,
Arrays.asList(
TableId.of(destinationDatasetName, "table1"),
TableId.of(destinationDatasetName, "table2")))
destinationTable, Arrays.asList(sourceTable1, sourceTable2))
.build();

// For more information on Job see:
Expand Down
Expand Up @@ -17,6 +17,8 @@
package com.example.bigquery;

// [START bigquery_query_external_bigtable_perm]

import com.google.api.client.util.Base64;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.BigQueryOptions;
Expand All @@ -29,7 +31,6 @@
import com.google.cloud.bigquery.TableInfo;
import com.google.cloud.bigquery.TableResult;
import com.google.common.collect.ImmutableList;
import org.apache.commons.codec.binary.Base64;

// Sample to queries an external bigtable data source using a permanent table
public class QueryExternalBigtablePerm {
Expand Down
Expand Up @@ -17,6 +17,8 @@
package com.example.bigquery;

// [START bigquery_query_external_bigtable_temp]

import com.google.api.client.util.Base64;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.BigQueryOptions;
Expand All @@ -27,7 +29,6 @@
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableResult;
import com.google.common.collect.ImmutableList;
import org.apache.commons.codec.binary.Base64;

// Sample to queries an external bigtable data source using a temporary table
public class QueryExternalBigtableTemp {
Expand Down
Expand Up @@ -19,6 +19,9 @@
import static com.google.common.truth.Truth.assertThat;
import static junit.framework.TestCase.assertNotNull;

import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.Schema;
import com.google.cloud.bigquery.StandardSQLTypeName;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.util.UUID;
Expand All @@ -32,22 +35,27 @@
public class CopyMultipleTablesIT {

private final Logger log = Logger.getLogger(this.getClass().getName());
private String datasetName;
private String tableName;
private String sourceTable1Name;
private String sourceTable2Name;
private ByteArrayOutputStream bout;
private PrintStream out;
private PrintStream originalPrintStream;

private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME");
private static final String PROJECT_ID = requireEnvVar("GOOGLE_CLOUD_PROJECT");

private static void requireEnvVar(String varName) {
private static String requireEnvVar(String varName) {
String value = System.getenv(varName);
assertNotNull(
"Environment variable " + varName + " is required to perform these tests.",
System.getenv(varName));
return value;
}

@BeforeClass
public static void checkRequirements() {
requireEnvVar("BIGQUERY_DATASET_NAME");
requireEnvVar("GOOGLE_CLOUD_PROJECT");
}

@Before
Expand All @@ -56,15 +64,30 @@ public void setUp() throws Exception {
out = new PrintStream(bout);
originalPrintStream = System.out;
System.setOut(out);

// Create a new destination table for each test since existing table cannot be overwritten
datasetName = "MY_DATASET_NAME_TEST_" + UUID.randomUUID().toString().substring(0, 8);
tableName = "COPY_MULTIPLE_TABLE_TEST" + UUID.randomUUID().toString().substring(0, 8);
CreateTable.createTable(BIGQUERY_DATASET_NAME, tableName, null);
sourceTable1Name =
"COPY_MULTIPLE_TABLE_SOURCE1_TEST" + UUID.randomUUID().toString().substring(0, 8);
sourceTable2Name =
"COPY_MULTIPLE_TABLE_SOURCE2_TEST" + UUID.randomUUID().toString().substring(0, 8);
CreateDataset.createDataset(datasetName);

Schema schema =
Schema.of(
Field.of("timestampField", StandardSQLTypeName.TIMESTAMP),
Field.of("stringField", StandardSQLTypeName.STRING),
Field.of("booleanField", StandardSQLTypeName.BOOL));
CreateTable.createTable(datasetName, tableName, schema);
CreateTable.createTable(datasetName, sourceTable1Name, schema);
CreateTable.createTable(datasetName, sourceTable2Name, schema);
}

@After
public void tearDown() {
// Clean up
DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName);
DeleteDataset.deleteDataset(PROJECT_ID, datasetName);
// restores print statements in the original method
System.out.flush();
System.setOut(originalPrintStream);
Expand All @@ -73,7 +96,8 @@ public void tearDown() {

@Test
public void testCopyMultipleTables() {
CopyMultipleTables.copyMultipleTables(BIGQUERY_DATASET_NAME, tableName);
CopyMultipleTables.copyMultipleTables(
datasetName, tableName, sourceTable1Name, sourceTable2Name);
assertThat(bout.toString()).contains("Table copied successfully.");
}
}

0 comments on commit e12122c

Please sign in to comment.