From b9f8aa9301180f1fe8aab3bd9e37d7d037c6035f Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Wed, 31 Aug 2022 10:47:12 +0300 Subject: [PATCH 01/11] Removed env variables and buckets from creating BQ. --- .../setup/EventsCreateBigQueryTable.java | 12 ++--- .../setup/ProductsCreateBigqueryTable.java | 10 ++-- .../src/main/java/setup/SetupCleanup.java | 53 +++++++++---------- 3 files changed, 34 insertions(+), 41 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java index 5b0cce2f..cd22c6b7 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java @@ -23,6 +23,8 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Schema; +import product.setup.ProductsCreateBigqueryTable; + import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; @@ -35,11 +37,9 @@ public static void main(String... args) throws IOException { String validEventsTable = "events"; String invalidEventsTable = "events_some_invalid"; String eventsSchemaFilePath = "src/main/resources/events_schema.json"; - String validEventsSourceFile = - String.format("gs://%s/user_events.json", EventsCreateGcsBucket.getBucketName()); + String validEventsSourceFile = ProductsCreateBigqueryTable.class.getResource("/user_events.json").getPath(); String invalidEventsSourceFile = - String.format( - "gs://%s/user_events_some_invalid.json", EventsCreateGcsBucket.getBucketName()); + ProductsCreateBigqueryTable.class.getResource("/user_events_some_invalid.json").getPath(); BufferedReader bufferedReader = new BufferedReader(new FileReader(eventsSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); @@ -49,8 +49,8 @@ public static void main(String... args) throws IOException { createBqDataset(dataset); createBqTable(dataset, validEventsTable, eventsSchema); - uploadDataToBqTable(dataset, validEventsTable, validEventsSourceFile, eventsSchema); + uploadDataToBqTable(dataset, validEventsTable, validEventsSourceFile); createBqTable(dataset, invalidEventsTable, eventsSchema); - uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile, eventsSchema); + uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile); } } diff --git a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java index 361144a8..d1214368 100644 --- a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java @@ -35,11 +35,9 @@ public static void main(String... args) throws IOException { String validProductsTable = "products"; String invalidProductsTable = "products_some_invalid"; String productSchemaFilePath = "src/main/resources/product_schema.json"; - String validProductsSourceFile = - String.format("gs://%s/products.json", ProductsCreateGcsBucket.getBucketName()); + String validProductsSourceFile = ProductsCreateBigqueryTable.class.getResource("/products.json").getPath(); String invalidProductsSourceFile = - String.format( - "gs://%s/products_some_invalid.json", ProductsCreateGcsBucket.getBucketName()); + ProductsCreateBigqueryTable.class.getResource("products_some_invalid.json").getPath(); BufferedReader bufferedReader = new BufferedReader(new FileReader(productSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); @@ -49,8 +47,8 @@ public static void main(String... args) throws IOException { createBqDataset(dataset); createBqTable(dataset, validProductsTable, productSchema); - uploadDataToBqTable(dataset, validProductsTable, validProductsSourceFile, productSchema); + uploadDataToBqTable(dataset, validProductsTable, validProductsSourceFile); createBqTable(dataset, invalidProductsTable, productSchema); - uploadDataToBqTable(dataset, invalidProductsTable, invalidProductsSourceFile, productSchema); + uploadDataToBqTable(dataset, invalidProductsTable, invalidProductsSourceFile); } } diff --git a/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java b/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java index 06f45b9d..f0d86258 100644 --- a/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java +++ b/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java @@ -22,25 +22,8 @@ import com.google.api.gax.paging.Page; import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.ServiceOptions; -import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.*; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.Dataset; -import com.google.cloud.bigquery.DatasetId; -import com.google.cloud.bigquery.DatasetInfo; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.FieldList; -import com.google.cloud.bigquery.FormatOptions; -import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.JobInfo; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.LoadJobConfiguration; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardTableDefinition; -import com.google.cloud.bigquery.TableDefinition; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; import com.google.cloud.retail.v2.CreateProductRequest; import com.google.cloud.retail.v2.DeleteProductRequest; import com.google.cloud.retail.v2.FulfillmentInfo; @@ -71,7 +54,10 @@ import com.google.protobuf.Int32Value; import com.google.protobuf.Timestamp; import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.util.Arrays; @@ -350,26 +336,35 @@ public static void createBqTable(String datasetName, String tableName, Schema sc } public static void uploadDataToBqTable( - String datasetName, String tableName, String sourceUri, Schema schema) { + String datasetName, String tableName, String sourceUri) { try { BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); TableId tableId = TableId.of(datasetName, tableName); - LoadJobConfiguration loadConfig = - LoadJobConfiguration.newBuilder(tableId, sourceUri) - .setFormatOptions(FormatOptions.json()) - .setSchema(schema) - .build(); - Job job = bigquery.create(JobInfo.of(loadConfig)); - job = job.waitFor(); + + WriteChannelConfiguration writeChannelConfiguration = + WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.json()).build(); + + String jobName = "jobId_" + UUID.randomUUID(); + JobId jobId = JobId.newBuilder().setLocation("us").setJob(jobName).build(); + + try (TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration); + OutputStream stream = Channels.newOutputStream(writer)) { + Files.copy(Paths.get(sourceUri), stream); + } + + Job job = bigquery.getJob(jobId); + Job completedJob = job.waitFor(); if (job.isDone()) { - System.out.printf("Json from GCS successfully loaded in a table '%s'.%n", tableName); + System.out.printf("Json successfully loaded in a table '%s'.%n", tableName); } else { System.out.println( - "BigQuery was unable to load into the table due to an error:" - + job.getStatus().getError()); + "BigQuery was unable to load into the table due to an error:" + + job.getStatus().getError()); } } catch (BigQueryException | InterruptedException e) { System.out.printf("Column not added during load append: %s%n", e.getMessage()); + } catch (IOException e) { + System.out.printf("Error copying file: %s%n", e.getMessage()); } } From 548b287198b49417078d8ccd1956f568942fb4d8 Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Wed, 31 Aug 2022 10:47:12 +0300 Subject: [PATCH 02/11] Fix: removed buckets from BQ import --- .../setup/EventsCreateBigQueryTable.java | 12 ++--- .../setup/ProductsCreateBigqueryTable.java | 10 ++-- .../src/main/java/setup/SetupCleanup.java | 53 +++++++++---------- 3 files changed, 34 insertions(+), 41 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java index 5b0cce2f..cd22c6b7 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java @@ -23,6 +23,8 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Schema; +import product.setup.ProductsCreateBigqueryTable; + import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; @@ -35,11 +37,9 @@ public static void main(String... args) throws IOException { String validEventsTable = "events"; String invalidEventsTable = "events_some_invalid"; String eventsSchemaFilePath = "src/main/resources/events_schema.json"; - String validEventsSourceFile = - String.format("gs://%s/user_events.json", EventsCreateGcsBucket.getBucketName()); + String validEventsSourceFile = ProductsCreateBigqueryTable.class.getResource("/user_events.json").getPath(); String invalidEventsSourceFile = - String.format( - "gs://%s/user_events_some_invalid.json", EventsCreateGcsBucket.getBucketName()); + ProductsCreateBigqueryTable.class.getResource("/user_events_some_invalid.json").getPath(); BufferedReader bufferedReader = new BufferedReader(new FileReader(eventsSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); @@ -49,8 +49,8 @@ public static void main(String... args) throws IOException { createBqDataset(dataset); createBqTable(dataset, validEventsTable, eventsSchema); - uploadDataToBqTable(dataset, validEventsTable, validEventsSourceFile, eventsSchema); + uploadDataToBqTable(dataset, validEventsTable, validEventsSourceFile); createBqTable(dataset, invalidEventsTable, eventsSchema); - uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile, eventsSchema); + uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile); } } diff --git a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java index 361144a8..d1214368 100644 --- a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java @@ -35,11 +35,9 @@ public static void main(String... args) throws IOException { String validProductsTable = "products"; String invalidProductsTable = "products_some_invalid"; String productSchemaFilePath = "src/main/resources/product_schema.json"; - String validProductsSourceFile = - String.format("gs://%s/products.json", ProductsCreateGcsBucket.getBucketName()); + String validProductsSourceFile = ProductsCreateBigqueryTable.class.getResource("/products.json").getPath(); String invalidProductsSourceFile = - String.format( - "gs://%s/products_some_invalid.json", ProductsCreateGcsBucket.getBucketName()); + ProductsCreateBigqueryTable.class.getResource("products_some_invalid.json").getPath(); BufferedReader bufferedReader = new BufferedReader(new FileReader(productSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); @@ -49,8 +47,8 @@ public static void main(String... args) throws IOException { createBqDataset(dataset); createBqTable(dataset, validProductsTable, productSchema); - uploadDataToBqTable(dataset, validProductsTable, validProductsSourceFile, productSchema); + uploadDataToBqTable(dataset, validProductsTable, validProductsSourceFile); createBqTable(dataset, invalidProductsTable, productSchema); - uploadDataToBqTable(dataset, invalidProductsTable, invalidProductsSourceFile, productSchema); + uploadDataToBqTable(dataset, invalidProductsTable, invalidProductsSourceFile); } } diff --git a/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java b/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java index 06f45b9d..f0d86258 100644 --- a/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java +++ b/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java @@ -22,25 +22,8 @@ import com.google.api.gax.paging.Page; import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.ServiceOptions; -import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.*; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.Dataset; -import com.google.cloud.bigquery.DatasetId; -import com.google.cloud.bigquery.DatasetInfo; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.FieldList; -import com.google.cloud.bigquery.FormatOptions; -import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.JobInfo; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.LoadJobConfiguration; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardTableDefinition; -import com.google.cloud.bigquery.TableDefinition; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; import com.google.cloud.retail.v2.CreateProductRequest; import com.google.cloud.retail.v2.DeleteProductRequest; import com.google.cloud.retail.v2.FulfillmentInfo; @@ -71,7 +54,10 @@ import com.google.protobuf.Int32Value; import com.google.protobuf.Timestamp; import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.util.Arrays; @@ -350,26 +336,35 @@ public static void createBqTable(String datasetName, String tableName, Schema sc } public static void uploadDataToBqTable( - String datasetName, String tableName, String sourceUri, Schema schema) { + String datasetName, String tableName, String sourceUri) { try { BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); TableId tableId = TableId.of(datasetName, tableName); - LoadJobConfiguration loadConfig = - LoadJobConfiguration.newBuilder(tableId, sourceUri) - .setFormatOptions(FormatOptions.json()) - .setSchema(schema) - .build(); - Job job = bigquery.create(JobInfo.of(loadConfig)); - job = job.waitFor(); + + WriteChannelConfiguration writeChannelConfiguration = + WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.json()).build(); + + String jobName = "jobId_" + UUID.randomUUID(); + JobId jobId = JobId.newBuilder().setLocation("us").setJob(jobName).build(); + + try (TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration); + OutputStream stream = Channels.newOutputStream(writer)) { + Files.copy(Paths.get(sourceUri), stream); + } + + Job job = bigquery.getJob(jobId); + Job completedJob = job.waitFor(); if (job.isDone()) { - System.out.printf("Json from GCS successfully loaded in a table '%s'.%n", tableName); + System.out.printf("Json successfully loaded in a table '%s'.%n", tableName); } else { System.out.println( - "BigQuery was unable to load into the table due to an error:" - + job.getStatus().getError()); + "BigQuery was unable to load into the table due to an error:" + + job.getStatus().getError()); } } catch (BigQueryException | InterruptedException e) { System.out.printf("Column not added during load append: %s%n", e.getMessage()); + } catch (IOException e) { + System.out.printf("Error copying file: %s%n", e.getMessage()); } } From 622d0c1a79db91bd0077bf891c3881088b02b03c Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Mon, 5 Sep 2022 09:20:06 +0300 Subject: [PATCH 03/11] pr fix: imports. --- .../src/main/java/setup/SetupCleanup.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java b/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java index f0d86258..0b932d47 100644 --- a/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java +++ b/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java @@ -22,7 +22,25 @@ import com.google.api.gax.paging.Page; import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.ServiceOptions; -import com.google.cloud.bigquery.*; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.WriteChannelConfiguration; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.FormatOptions; +import com.google.cloud.bigquery.TableDataWriteChannel; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; import com.google.cloud.retail.v2.CreateProductRequest; import com.google.cloud.retail.v2.DeleteProductRequest; From ad1e7dcf5ced3ea80b425e46ac594e8c547508cc Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 12 Sep 2022 17:40:26 +0000 Subject: [PATCH 04/11] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../setup/EventsCreateBigQueryTable.java | 8 ++-- .../setup/ProductsCreateBigqueryTable.java | 5 ++- .../src/main/java/setup/SetupCleanup.java | 40 +++++++++---------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java index cd22c6b7..9bf81492 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java @@ -23,12 +23,11 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Schema; -import product.setup.ProductsCreateBigqueryTable; - import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import java.util.stream.Collectors; +import product.setup.ProductsCreateBigqueryTable; public class EventsCreateBigQueryTable { @@ -37,9 +36,10 @@ public static void main(String... args) throws IOException { String validEventsTable = "events"; String invalidEventsTable = "events_some_invalid"; String eventsSchemaFilePath = "src/main/resources/events_schema.json"; - String validEventsSourceFile = ProductsCreateBigqueryTable.class.getResource("/user_events.json").getPath(); + String validEventsSourceFile = + ProductsCreateBigqueryTable.class.getResource("/user_events.json").getPath(); String invalidEventsSourceFile = - ProductsCreateBigqueryTable.class.getResource("/user_events_some_invalid.json").getPath(); + ProductsCreateBigqueryTable.class.getResource("/user_events_some_invalid.json").getPath(); BufferedReader bufferedReader = new BufferedReader(new FileReader(eventsSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); diff --git a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java index d1214368..03fba7b8 100644 --- a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java @@ -35,9 +35,10 @@ public static void main(String... args) throws IOException { String validProductsTable = "products"; String invalidProductsTable = "products_some_invalid"; String productSchemaFilePath = "src/main/resources/product_schema.json"; - String validProductsSourceFile = ProductsCreateBigqueryTable.class.getResource("/products.json").getPath(); + String validProductsSourceFile = + ProductsCreateBigqueryTable.class.getResource("/products.json").getPath(); String invalidProductsSourceFile = - ProductsCreateBigqueryTable.class.getResource("products_some_invalid.json").getPath(); + ProductsCreateBigqueryTable.class.getResource("products_some_invalid.json").getPath(); BufferedReader bufferedReader = new BufferedReader(new FileReader(productSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); diff --git a/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java b/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java index 0b932d47..9e6b3d5c 100644 --- a/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java +++ b/samples/interactive-tutorials/src/main/java/setup/SetupCleanup.java @@ -23,25 +23,25 @@ import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; +import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.Dataset; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.DatasetId; -import com.google.cloud.bigquery.TableDefinition; -import com.google.cloud.bigquery.StandardTableDefinition; -import com.google.cloud.bigquery.TableInfo; -import com.google.cloud.bigquery.WriteChannelConfiguration; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.FormatOptions; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobId; -import com.google.cloud.bigquery.FormatOptions; -import com.google.cloud.bigquery.TableDataWriteChannel; import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.FieldList; -import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableDataWriteChannel; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.WriteChannelConfiguration; import com.google.cloud.retail.v2.CreateProductRequest; import com.google.cloud.retail.v2.DeleteProductRequest; import com.google.cloud.retail.v2.FulfillmentInfo; @@ -75,7 +75,6 @@ import java.io.OutputStream; import java.nio.channels.Channels; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; import java.util.Arrays; @@ -353,20 +352,21 @@ public static void createBqTable(String datasetName, String tableName, Schema sc } } - public static void uploadDataToBqTable( - String datasetName, String tableName, String sourceUri) { + public static void uploadDataToBqTable(String datasetName, String tableName, String sourceUri) { try { BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); TableId tableId = TableId.of(datasetName, tableName); WriteChannelConfiguration writeChannelConfiguration = - WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.json()).build(); + WriteChannelConfiguration.newBuilder(tableId) + .setFormatOptions(FormatOptions.json()) + .build(); String jobName = "jobId_" + UUID.randomUUID(); JobId jobId = JobId.newBuilder().setLocation("us").setJob(jobName).build(); try (TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration); - OutputStream stream = Channels.newOutputStream(writer)) { + OutputStream stream = Channels.newOutputStream(writer)) { Files.copy(Paths.get(sourceUri), stream); } @@ -376,8 +376,8 @@ public static void uploadDataToBqTable( System.out.printf("Json successfully loaded in a table '%s'.%n", tableName); } else { System.out.println( - "BigQuery was unable to load into the table due to an error:" - + job.getStatus().getError()); + "BigQuery was unable to load into the table due to an error:" + + job.getStatus().getError()); } } catch (BigQueryException | InterruptedException e) { System.out.printf("Column not added during load append: %s%n", e.getMessage()); From e3bad7fd1beeb47dce143cc4bfe7d12757b476df Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Thu, 15 Sep 2022 09:47:38 +0300 Subject: [PATCH 05/11] pr fix: fixed test. --- .../src/test/java/product/AddFulfillmentPlacesTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/interactive-tutorials/src/test/java/product/AddFulfillmentPlacesTest.java b/samples/interactive-tutorials/src/test/java/product/AddFulfillmentPlacesTest.java index e561cb42..ea17c286 100644 --- a/samples/interactive-tutorials/src/test/java/product/AddFulfillmentPlacesTest.java +++ b/samples/interactive-tutorials/src/test/java/product/AddFulfillmentPlacesTest.java @@ -64,7 +64,7 @@ public void setUp() throws IOException, InterruptedException, ExecutionException public void testAddFulfillment() { String outputResult = bout.toString(); - assertThat(outputResult).contains("Add fulfilment places with current date"); + assertThat(outputResult).contains("Add fulfilment places"); assertThat(outputResult).contains("Add fulfillment places, wait 45 seconds"); } From ee334ac9cfcb7a455c361bf137e5a959d284eea3 Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Thu, 15 Sep 2022 09:52:25 +0300 Subject: [PATCH 06/11] pr fix: added comment. --- .../src/main/java/events/setup/EventsCreateBigQueryTable.java | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java index 10401efb..0c714d90 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java @@ -36,6 +36,7 @@ public static void main(String[] args) throws IOException { String validEventsTable = "events"; String invalidEventsTable = "events_some_invalid"; String eventsSchemaFilePath = "src/main/resources/events_schema.json"; + // user_events.json and user_events_some_invalid.json are located in the resources folder String validEventsSourceFile = ProductsCreateBigqueryTable.class.getResource("/user_events.json").getPath(); String invalidEventsSourceFile = From a8988b4a83a8101379201dea1567e8be5e6e9bc2 Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Tue, 20 Sep 2022 08:58:04 +0300 Subject: [PATCH 07/11] fix(samples): bug fix for removing buckets from bq. --- .../src/main/java/product/ImportProductsBigQueryTable.java | 4 +++- .../main/java/product/setup/ProductsCreateBigqueryTable.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java index 552751cf..4c061955 100644 --- a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java @@ -40,10 +40,12 @@ public static void main(String[] args) throws IOException, InterruptedException String branchName = String.format( "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); + // TO CHECK ERROR HANDLING PASTE THE INVALID BRANCH NAME HERE: + // branchName = "invalid_branch_name"; String datasetId = "products"; String tableId = "products"; // TO CHECK ERROR HANDLING USE THE TABLE WITH INVALID PRODUCTS: - // tableId = "products_some_invalid" + // tableId = "products_some_invalid"; importProductsFromBigQuery(projectId, branchName, datasetId, tableId); } diff --git a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java index a13c0b8d..2463622a 100644 --- a/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java @@ -38,7 +38,7 @@ public static void main(String[] args) throws IOException { String validProductsSourceFile = ProductsCreateBigqueryTable.class.getResource("/products.json").getPath(); String invalidProductsSourceFile = - ProductsCreateBigqueryTable.class.getResource("products_some_invalid.json").getPath(); + ProductsCreateBigqueryTable.class.getResource("/products_some_invalid.json").getPath(); BufferedReader bufferedReader = new BufferedReader(new FileReader(productSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); From 1963228612342303f93218072b847597e391a630 Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Tue, 20 Sep 2022 13:33:02 +0300 Subject: [PATCH 08/11] pr fix: moved commented lines --- .../main/java/product/ImportProductsBigQueryTable.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java index 4c061955..06fad15b 100644 --- a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java @@ -37,15 +37,15 @@ public class ImportProductsBigQueryTable { public static void main(String[] args) throws IOException, InterruptedException { String projectId = ServiceOptions.getDefaultProjectId(); + // To check for error handling, replace the below variable with the invalid branch name + // String branchName = "invalid_branch_name"; String branchName = String.format( "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); - // TO CHECK ERROR HANDLING PASTE THE INVALID BRANCH NAME HERE: - // branchName = "invalid_branch_name"; String datasetId = "products"; + // To check for error handling, replace the below variable with table id that contains invalid products + // String tableId = "products_some_invalid"; String tableId = "products"; - // TO CHECK ERROR HANDLING USE THE TABLE WITH INVALID PRODUCTS: - // tableId = "products_some_invalid"; importProductsFromBigQuery(projectId, branchName, datasetId, tableId); } From 79440435aeaae85b1f16a2845dd53689c57eacbe Mon Sep 17 00:00:00 2001 From: dfirova <93149631+dfirova@users.noreply.github.com> Date: Tue, 20 Sep 2022 15:22:32 +0300 Subject: [PATCH 09/11] pr fix: added dot to importproductsbigquerytable. Co-authored-by: Sita Lakshmi Sangameswaran --- .../src/main/java/product/ImportProductsBigQueryTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java index 06fad15b..ebd6a737 100644 --- a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java @@ -37,7 +37,7 @@ public class ImportProductsBigQueryTable { public static void main(String[] args) throws IOException, InterruptedException { String projectId = ServiceOptions.getDefaultProjectId(); - // To check for error handling, replace the below variable with the invalid branch name + // To check for error handling, replace the below variable with the invalid branch name. // String branchName = "invalid_branch_name"; String branchName = String.format( From 762c9d91d257433422ad0c21e10f3f0680f1de63 Mon Sep 17 00:00:00 2001 From: dfirova <93149631+dfirova@users.noreply.github.com> Date: Tue, 20 Sep 2022 15:23:06 +0300 Subject: [PATCH 10/11] pr fix: added dot in importproductsbigquerytable Co-authored-by: Sita Lakshmi Sangameswaran --- .../src/main/java/product/ImportProductsBigQueryTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java index ebd6a737..91eda154 100644 --- a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java @@ -43,7 +43,7 @@ public static void main(String[] args) throws IOException, InterruptedException String.format( "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); String datasetId = "products"; - // To check for error handling, replace the below variable with table id that contains invalid products + // To check for error handling, replace the below variable with table id that contains invalid products. // String tableId = "products_some_invalid"; String tableId = "products"; From 86d26420200dd758bf6244a15a1a12ad79653c9c Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 23 Sep 2022 19:50:07 +0000 Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../src/main/java/product/ImportProductsBigQueryTable.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java index 91eda154..83dce027 100644 --- a/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java @@ -43,7 +43,8 @@ public static void main(String[] args) throws IOException, InterruptedException String.format( "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); String datasetId = "products"; - // To check for error handling, replace the below variable with table id that contains invalid products. + // To check for error handling, replace the below variable with table id that contains invalid + // products. // String tableId = "products_some_invalid"; String tableId = "products";