From ec1812ca7decbfef12eeff54f195a7bac0dfdc17 Mon Sep 17 00:00:00 2001 From: Mridula Peddada Date: Tue, 25 Oct 2022 14:07:53 -0400 Subject: [PATCH 1/2] fix: restore native image configuration --- .github/.OwlBot.yaml | 1 + .kokoro/build.sh | 10 + .../reflect-config.json | 7 + .../it/ITBigQueryWriteManualClientTest.java | 304 +++++++++--------- 4 files changed, 170 insertions(+), 152 deletions(-) create mode 100644 google-cloud-bigquerystorage/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquerystorage/reflect-config.json diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml index dbe80cd9e4..2c9814a26a 100644 --- a/.github/.OwlBot.yaml +++ b/.github/.OwlBot.yaml @@ -83,6 +83,7 @@ deep-preserve-regex: - "/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/Exceptions.java" - "/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java" - "/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPool.java" +- "/google-cloud-bigquerystorage/src/main/resources/META-INF/native-image/" deep-copy-regex: - source: "/google/cloud/bigquery/storage/(v.*)/.*-java/proto-google-.*/src" diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 238abf9950..7a773764c8 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -68,6 +68,16 @@ integration) verify RETURN_CODE=$? ;; +graalvm) + # Run Unit and Integration Tests with Native Image + mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test + RETURN_CODE=$? + ;; +graalvm17) + # Run Unit and Integration Tests with Native Image + mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test + RETURN_CODE=$? + ;; samples) SAMPLES_DIR=samples # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise. diff --git a/google-cloud-bigquerystorage/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquerystorage/reflect-config.json b/google-cloud-bigquerystorage/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquerystorage/reflect-config.json new file mode 100644 index 0000000000..e10630f8cf --- /dev/null +++ b/google-cloud-bigquerystorage/src/main/resources/META-INF/native-image/com.google.cloud/google-cloud-bigquerystorage/reflect-config.json @@ -0,0 +1,7 @@ +[ + { + "name":"java.lang.Object", + "allDeclaredFields":true, + "queryAllDeclaredMethods":true, + "methods":[{"name":"","parameterTypes":[] }]} +] \ No newline at end of file diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java index 86de48a972..f215e4e226 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java @@ -572,158 +572,158 @@ public void testJsonStreamWriterWithDefaultSchemaNoTable() { public void testJsonStreamWriterWithDefaultStream() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException { - String tableName = "JsonTableDefaultStream"; - TableFieldSchema TEST_STRING = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.STRING) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_str") - .build(); - TableFieldSchema TEST_NUMERIC = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.NUMERIC) - .setMode(TableFieldSchema.Mode.REPEATED) - .setName("test_numerics") - .build(); - TableFieldSchema TEST_DATE = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.DATETIME) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_datetime") - .build(); - TableFieldSchema TEST_REPEATED_BYTESTRING = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.BYTES) - .setMode(TableFieldSchema.Mode.REPEATED) - .setName("test_bytestring_repeated") - .build(); - TableFieldSchema TEST_TIMESTAMP = - TableFieldSchema.newBuilder() - .setName("test_timeStamp") - .setType(TableFieldSchema.Type.TIMESTAMP) - .setMode(TableFieldSchema.Mode.NULLABLE) - .build(); - TableSchema tableSchema = - TableSchema.newBuilder() - .addFields(0, TEST_STRING) - .addFields(1, TEST_DATE) - .addFields(2, TEST_NUMERIC) - .addFields(3, TEST_REPEATED_BYTESTRING) - .addFields(4, TEST_TIMESTAMP) - .build(); - TableInfo tableInfo = - TableInfo.newBuilder( - TableId.of(DATASET, tableName), - StandardTableDefinition.of( - Schema.of( - com.google.cloud.bigquery.Field.newBuilder( - "test_str", StandardSQLTypeName.STRING) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_numerics", StandardSQLTypeName.NUMERIC) - .setMode(Field.Mode.REPEATED) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_datetime", StandardSQLTypeName.DATETIME) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_bytestring_repeated", StandardSQLTypeName.BYTES) - .setMode(Field.Mode.REPEATED) - .build(), - com.google.cloud.bigquery.Field.newBuilder( - "test_timestamp", StandardSQLTypeName.TIMESTAMP) - .build()))) - .build(); - - bigquery.create(tableInfo); - TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); - try (JsonStreamWriter jsonStreamWriter = - JsonStreamWriter.newBuilder(parent.toString(), tableSchema) - .setIgnoreUnknownFields(true) - .build()) { - LOG.info("Sending one message"); - JSONObject row1 = new JSONObject(); - row1.put("test_str", "aaa"); - row1.put( - "test_numerics", - new JSONArray( - new byte[][] { - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("123.4")) - .toByteArray(), - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-9000000")) - .toByteArray() - })); - row1.put("unknown_field", "a"); - row1.put( - "test_datetime", - CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2020, 10, 1, 12, 0))); - row1.put( - "test_bytestring_repeated", - new JSONArray( - new byte[][] { - ByteString.copyFromUtf8("a").toByteArray(), - ByteString.copyFromUtf8("b").toByteArray() - })); - row1.put("test_timestamp", "2022-02-06 07:24:47.84"); - JSONArray jsonArr1 = new JSONArray(new JSONObject[] {row1}); - - ApiFuture response1 = jsonStreamWriter.append(jsonArr1, -1); - - assertEquals(0, response1.get().getAppendResult().getOffset().getValue()); - - JSONObject row2 = new JSONObject(); - row1.put("test_str", "bbb"); - JSONObject row3 = new JSONObject(); - row2.put("test_str", "ccc"); - JSONArray jsonArr2 = new JSONArray(); - jsonArr2.put(row1); - jsonArr2.put(row2); - - JSONObject row4 = new JSONObject(); - row4.put("test_str", "ddd"); - JSONArray jsonArr3 = new JSONArray(); - jsonArr3.put(row4); - - JSONObject row5 = new JSONObject(); - // Add another ARRAY using a more idiomatic way - JSONArray testArr = new JSONArray(); // create empty JSONArray - testArr.put(0, ByteString.copyFromUtf8("a").toByteArray()); // insert 1st bytes array - testArr.put(1, ByteString.copyFromUtf8("b").toByteArray()); // insert 2nd bytes array - row5.put("test_bytestring_repeated", testArr); - JSONArray jsonArr4 = new JSONArray(); - jsonArr4.put(row5); - - LOG.info("Sending three more messages"); - ApiFuture response2 = jsonStreamWriter.append(jsonArr2, -1); - LOG.info("Sending two more messages"); - ApiFuture response3 = jsonStreamWriter.append(jsonArr3, -1); - LOG.info("Sending one more message"); - ApiFuture response4 = jsonStreamWriter.append(jsonArr4, -1); - Assert.assertFalse(response2.get().getAppendResult().hasOffset()); - Assert.assertFalse(response3.get().getAppendResult().hasOffset()); - Assert.assertFalse(response4.get().getAppendResult().hasOffset()); - - TableResult result = - bigquery.listTableData( - tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); - Iterator iter = result.getValues().iterator(); - FieldValueList currentRow = iter.next(); - assertEquals("aaa", currentRow.get(0).getStringValue()); - assertEquals("-9000000", currentRow.get(1).getRepeatedValue().get(1).getStringValue()); - assertEquals("2020-10-01T12:00:00", currentRow.get(2).getStringValue()); - assertEquals(2, currentRow.get(3).getRepeatedValue().size()); - assertEquals("Yg==", currentRow.get(3).getRepeatedValue().get(1).getStringValue()); - assertEquals( - Timestamp.valueOf("2022-02-06 07:24:47.84").getTime() * 1000, - currentRow.get(4).getTimestampValue()); // timestamp long of "2022-02-06 07:24:47.84" - assertEquals("bbb", iter.next().get(0).getStringValue()); - assertEquals("ccc", iter.next().get(0).getStringValue()); - assertEquals("ddd", iter.next().get(0).getStringValue()); - FieldValueList currentRow2 = iter.next(); - assertEquals("YQ==", currentRow2.get(3).getRepeatedValue().get(0).getStringValue()); - assertEquals("Yg==", currentRow2.get(3).getRepeatedValue().get(1).getStringValue()); - assertEquals(false, iter.hasNext()); - } +// String tableName = "JsonTableDefaultStream"; +// TableFieldSchema TEST_STRING = +// TableFieldSchema.newBuilder() +// .setType(TableFieldSchema.Type.STRING) +// .setMode(TableFieldSchema.Mode.NULLABLE) +// .setName("test_str") +// .build(); +// TableFieldSchema TEST_NUMERIC = +// TableFieldSchema.newBuilder() +// .setType(TableFieldSchema.Type.NUMERIC) +// .setMode(TableFieldSchema.Mode.REPEATED) +// .setName("test_numerics") +// .build(); +// TableFieldSchema TEST_DATE = +// TableFieldSchema.newBuilder() +// .setType(TableFieldSchema.Type.DATETIME) +// .setMode(TableFieldSchema.Mode.NULLABLE) +// .setName("test_datetime") +// .build(); +// TableFieldSchema TEST_REPEATED_BYTESTRING = +// TableFieldSchema.newBuilder() +// .setType(TableFieldSchema.Type.BYTES) +// .setMode(TableFieldSchema.Mode.REPEATED) +// .setName("test_bytestring_repeated") +// .build(); +// TableFieldSchema TEST_TIMESTAMP = +// TableFieldSchema.newBuilder() +// .setName("test_timeStamp") +// .setType(TableFieldSchema.Type.TIMESTAMP) +// .setMode(TableFieldSchema.Mode.NULLABLE) +// .build(); +// TableSchema tableSchema = +// TableSchema.newBuilder() +// .addFields(0, TEST_STRING) +// .addFields(1, TEST_DATE) +// .addFields(2, TEST_NUMERIC) +// .addFields(3, TEST_REPEATED_BYTESTRING) +// .addFields(4, TEST_TIMESTAMP) +// .build(); +// TableInfo tableInfo = +// TableInfo.newBuilder( +// TableId.of(DATASET, tableName), +// StandardTableDefinition.of( +// Schema.of( +// com.google.cloud.bigquery.Field.newBuilder( +// "test_str", StandardSQLTypeName.STRING) +// .build(), +// com.google.cloud.bigquery.Field.newBuilder( +// "test_numerics", StandardSQLTypeName.NUMERIC) +// .setMode(Field.Mode.REPEATED) +// .build(), +// com.google.cloud.bigquery.Field.newBuilder( +// "test_datetime", StandardSQLTypeName.DATETIME) +// .build(), +// com.google.cloud.bigquery.Field.newBuilder( +// "test_bytestring_repeated", StandardSQLTypeName.BYTES) +// .setMode(Field.Mode.REPEATED) +// .build(), +// com.google.cloud.bigquery.Field.newBuilder( +// "test_timestamp", StandardSQLTypeName.TIMESTAMP) +// .build()))) +// .build(); +// +// bigquery.create(tableInfo); +// TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); +// try (JsonStreamWriter jsonStreamWriter = +// JsonStreamWriter.newBuilder(parent.toString(), tableSchema) +// .setIgnoreUnknownFields(true) +// .build()) { +// LOG.info("Sending one message"); +// JSONObject row1 = new JSONObject(); +// row1.put("test_str", "aaa"); +// row1.put( +// "test_numerics", +// new JSONArray( +// new byte[][] { +// BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("123.4")) +// .toByteArray(), +// BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-9000000")) +// .toByteArray() +// })); +// row1.put("unknown_field", "a"); +// row1.put( +// "test_datetime", +// CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2020, 10, 1, 12, 0))); +// row1.put( +// "test_bytestring_repeated", +// new JSONArray( +// new byte[][] { +// ByteString.copyFromUtf8("a").toByteArray(), +// ByteString.copyFromUtf8("b").toByteArray() +// })); +// row1.put("test_timestamp", "2022-02-06 07:24:47.84"); +// JSONArray jsonArr1 = new JSONArray(new JSONObject[] {row1}); +// +// ApiFuture response1 = jsonStreamWriter.append(jsonArr1, -1); +// +// assertEquals(0, response1.get().getAppendResult().getOffset().getValue()); +// +// JSONObject row2 = new JSONObject(); +// row1.put("test_str", "bbb"); +// JSONObject row3 = new JSONObject(); +// row2.put("test_str", "ccc"); +// JSONArray jsonArr2 = new JSONArray(); +// jsonArr2.put(row1); +// jsonArr2.put(row2); +// +// JSONObject row4 = new JSONObject(); +// row4.put("test_str", "ddd"); +// JSONArray jsonArr3 = new JSONArray(); +// jsonArr3.put(row4); +// +// JSONObject row5 = new JSONObject(); +// // Add another ARRAY using a more idiomatic way +// JSONArray testArr = new JSONArray(); // create empty JSONArray +// testArr.put(0, ByteString.copyFromUtf8("a").toByteArray()); // insert 1st bytes array +// testArr.put(1, ByteString.copyFromUtf8("b").toByteArray()); // insert 2nd bytes array +// row5.put("test_bytestring_repeated", testArr); +// JSONArray jsonArr4 = new JSONArray(); +// jsonArr4.put(row5); +// +// LOG.info("Sending three more messages"); +// ApiFuture response2 = jsonStreamWriter.append(jsonArr2, -1); +// LOG.info("Sending two more messages"); +// ApiFuture response3 = jsonStreamWriter.append(jsonArr3, -1); +// LOG.info("Sending one more message"); +// ApiFuture response4 = jsonStreamWriter.append(jsonArr4, -1); +// Assert.assertFalse(response2.get().getAppendResult().hasOffset()); +// Assert.assertFalse(response3.get().getAppendResult().hasOffset()); +// Assert.assertFalse(response4.get().getAppendResult().hasOffset()); +// +// TableResult result = +// bigquery.listTableData( +// tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); +// Iterator iter = result.getValues().iterator(); +// FieldValueList currentRow = iter.next(); +// assertEquals("aaa", currentRow.get(0).getStringValue()); +// assertEquals("-9000000", currentRow.get(1).getRepeatedValue().get(1).getStringValue()); +// assertEquals("2020-10-01T12:00:00", currentRow.get(2).getStringValue()); +// assertEquals(2, currentRow.get(3).getRepeatedValue().size()); +// assertEquals("Yg==", currentRow.get(3).getRepeatedValue().get(1).getStringValue()); +// assertEquals( +// Timestamp.valueOf("2022-02-06 07:24:47.84").getTime() * 1000, +// currentRow.get(4).getTimestampValue()); // timestamp long of "2022-02-06 07:24:47.84" +// assertEquals("bbb", iter.next().get(0).getStringValue()); +// assertEquals("ccc", iter.next().get(0).getStringValue()); +// assertEquals("ddd", iter.next().get(0).getStringValue()); +// FieldValueList currentRow2 = iter.next(); +// assertEquals("YQ==", currentRow2.get(3).getRepeatedValue().get(0).getStringValue()); +// assertEquals("Yg==", currentRow2.get(3).getRepeatedValue().get(1).getStringValue()); +// assertEquals(false, iter.hasNext()); +// } } // This test runs about 1 min. From 8833f3d5e25c170995e7f7544b25fa032c650c3e Mon Sep 17 00:00:00 2001 From: Mridula Peddada Date: Tue, 25 Oct 2022 14:09:13 -0400 Subject: [PATCH 2/2] remove debugging changes --- .../it/ITBigQueryWriteManualClientTest.java | 304 +++++++++--------- 1 file changed, 152 insertions(+), 152 deletions(-) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java index f215e4e226..86de48a972 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteManualClientTest.java @@ -572,158 +572,158 @@ public void testJsonStreamWriterWithDefaultSchemaNoTable() { public void testJsonStreamWriterWithDefaultStream() throws IOException, InterruptedException, ExecutionException, Descriptors.DescriptorValidationException { -// String tableName = "JsonTableDefaultStream"; -// TableFieldSchema TEST_STRING = -// TableFieldSchema.newBuilder() -// .setType(TableFieldSchema.Type.STRING) -// .setMode(TableFieldSchema.Mode.NULLABLE) -// .setName("test_str") -// .build(); -// TableFieldSchema TEST_NUMERIC = -// TableFieldSchema.newBuilder() -// .setType(TableFieldSchema.Type.NUMERIC) -// .setMode(TableFieldSchema.Mode.REPEATED) -// .setName("test_numerics") -// .build(); -// TableFieldSchema TEST_DATE = -// TableFieldSchema.newBuilder() -// .setType(TableFieldSchema.Type.DATETIME) -// .setMode(TableFieldSchema.Mode.NULLABLE) -// .setName("test_datetime") -// .build(); -// TableFieldSchema TEST_REPEATED_BYTESTRING = -// TableFieldSchema.newBuilder() -// .setType(TableFieldSchema.Type.BYTES) -// .setMode(TableFieldSchema.Mode.REPEATED) -// .setName("test_bytestring_repeated") -// .build(); -// TableFieldSchema TEST_TIMESTAMP = -// TableFieldSchema.newBuilder() -// .setName("test_timeStamp") -// .setType(TableFieldSchema.Type.TIMESTAMP) -// .setMode(TableFieldSchema.Mode.NULLABLE) -// .build(); -// TableSchema tableSchema = -// TableSchema.newBuilder() -// .addFields(0, TEST_STRING) -// .addFields(1, TEST_DATE) -// .addFields(2, TEST_NUMERIC) -// .addFields(3, TEST_REPEATED_BYTESTRING) -// .addFields(4, TEST_TIMESTAMP) -// .build(); -// TableInfo tableInfo = -// TableInfo.newBuilder( -// TableId.of(DATASET, tableName), -// StandardTableDefinition.of( -// Schema.of( -// com.google.cloud.bigquery.Field.newBuilder( -// "test_str", StandardSQLTypeName.STRING) -// .build(), -// com.google.cloud.bigquery.Field.newBuilder( -// "test_numerics", StandardSQLTypeName.NUMERIC) -// .setMode(Field.Mode.REPEATED) -// .build(), -// com.google.cloud.bigquery.Field.newBuilder( -// "test_datetime", StandardSQLTypeName.DATETIME) -// .build(), -// com.google.cloud.bigquery.Field.newBuilder( -// "test_bytestring_repeated", StandardSQLTypeName.BYTES) -// .setMode(Field.Mode.REPEATED) -// .build(), -// com.google.cloud.bigquery.Field.newBuilder( -// "test_timestamp", StandardSQLTypeName.TIMESTAMP) -// .build()))) -// .build(); -// -// bigquery.create(tableInfo); -// TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); -// try (JsonStreamWriter jsonStreamWriter = -// JsonStreamWriter.newBuilder(parent.toString(), tableSchema) -// .setIgnoreUnknownFields(true) -// .build()) { -// LOG.info("Sending one message"); -// JSONObject row1 = new JSONObject(); -// row1.put("test_str", "aaa"); -// row1.put( -// "test_numerics", -// new JSONArray( -// new byte[][] { -// BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("123.4")) -// .toByteArray(), -// BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-9000000")) -// .toByteArray() -// })); -// row1.put("unknown_field", "a"); -// row1.put( -// "test_datetime", -// CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2020, 10, 1, 12, 0))); -// row1.put( -// "test_bytestring_repeated", -// new JSONArray( -// new byte[][] { -// ByteString.copyFromUtf8("a").toByteArray(), -// ByteString.copyFromUtf8("b").toByteArray() -// })); -// row1.put("test_timestamp", "2022-02-06 07:24:47.84"); -// JSONArray jsonArr1 = new JSONArray(new JSONObject[] {row1}); -// -// ApiFuture response1 = jsonStreamWriter.append(jsonArr1, -1); -// -// assertEquals(0, response1.get().getAppendResult().getOffset().getValue()); -// -// JSONObject row2 = new JSONObject(); -// row1.put("test_str", "bbb"); -// JSONObject row3 = new JSONObject(); -// row2.put("test_str", "ccc"); -// JSONArray jsonArr2 = new JSONArray(); -// jsonArr2.put(row1); -// jsonArr2.put(row2); -// -// JSONObject row4 = new JSONObject(); -// row4.put("test_str", "ddd"); -// JSONArray jsonArr3 = new JSONArray(); -// jsonArr3.put(row4); -// -// JSONObject row5 = new JSONObject(); -// // Add another ARRAY using a more idiomatic way -// JSONArray testArr = new JSONArray(); // create empty JSONArray -// testArr.put(0, ByteString.copyFromUtf8("a").toByteArray()); // insert 1st bytes array -// testArr.put(1, ByteString.copyFromUtf8("b").toByteArray()); // insert 2nd bytes array -// row5.put("test_bytestring_repeated", testArr); -// JSONArray jsonArr4 = new JSONArray(); -// jsonArr4.put(row5); -// -// LOG.info("Sending three more messages"); -// ApiFuture response2 = jsonStreamWriter.append(jsonArr2, -1); -// LOG.info("Sending two more messages"); -// ApiFuture response3 = jsonStreamWriter.append(jsonArr3, -1); -// LOG.info("Sending one more message"); -// ApiFuture response4 = jsonStreamWriter.append(jsonArr4, -1); -// Assert.assertFalse(response2.get().getAppendResult().hasOffset()); -// Assert.assertFalse(response3.get().getAppendResult().hasOffset()); -// Assert.assertFalse(response4.get().getAppendResult().hasOffset()); -// -// TableResult result = -// bigquery.listTableData( -// tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); -// Iterator iter = result.getValues().iterator(); -// FieldValueList currentRow = iter.next(); -// assertEquals("aaa", currentRow.get(0).getStringValue()); -// assertEquals("-9000000", currentRow.get(1).getRepeatedValue().get(1).getStringValue()); -// assertEquals("2020-10-01T12:00:00", currentRow.get(2).getStringValue()); -// assertEquals(2, currentRow.get(3).getRepeatedValue().size()); -// assertEquals("Yg==", currentRow.get(3).getRepeatedValue().get(1).getStringValue()); -// assertEquals( -// Timestamp.valueOf("2022-02-06 07:24:47.84").getTime() * 1000, -// currentRow.get(4).getTimestampValue()); // timestamp long of "2022-02-06 07:24:47.84" -// assertEquals("bbb", iter.next().get(0).getStringValue()); -// assertEquals("ccc", iter.next().get(0).getStringValue()); -// assertEquals("ddd", iter.next().get(0).getStringValue()); -// FieldValueList currentRow2 = iter.next(); -// assertEquals("YQ==", currentRow2.get(3).getRepeatedValue().get(0).getStringValue()); -// assertEquals("Yg==", currentRow2.get(3).getRepeatedValue().get(1).getStringValue()); -// assertEquals(false, iter.hasNext()); -// } + String tableName = "JsonTableDefaultStream"; + TableFieldSchema TEST_STRING = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.STRING) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_str") + .build(); + TableFieldSchema TEST_NUMERIC = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_numerics") + .build(); + TableFieldSchema TEST_DATE = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_datetime") + .build(); + TableFieldSchema TEST_REPEATED_BYTESTRING = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.BYTES) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_bytestring_repeated") + .build(); + TableFieldSchema TEST_TIMESTAMP = + TableFieldSchema.newBuilder() + .setName("test_timeStamp") + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build(); + TableSchema tableSchema = + TableSchema.newBuilder() + .addFields(0, TEST_STRING) + .addFields(1, TEST_DATE) + .addFields(2, TEST_NUMERIC) + .addFields(3, TEST_REPEATED_BYTESTRING) + .addFields(4, TEST_TIMESTAMP) + .build(); + TableInfo tableInfo = + TableInfo.newBuilder( + TableId.of(DATASET, tableName), + StandardTableDefinition.of( + Schema.of( + com.google.cloud.bigquery.Field.newBuilder( + "test_str", StandardSQLTypeName.STRING) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_numerics", StandardSQLTypeName.NUMERIC) + .setMode(Field.Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_datetime", StandardSQLTypeName.DATETIME) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_bytestring_repeated", StandardSQLTypeName.BYTES) + .setMode(Field.Mode.REPEATED) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "test_timestamp", StandardSQLTypeName.TIMESTAMP) + .build()))) + .build(); + + bigquery.create(tableInfo); + TableName parent = TableName.of(ServiceOptions.getDefaultProjectId(), DATASET, tableName); + try (JsonStreamWriter jsonStreamWriter = + JsonStreamWriter.newBuilder(parent.toString(), tableSchema) + .setIgnoreUnknownFields(true) + .build()) { + LOG.info("Sending one message"); + JSONObject row1 = new JSONObject(); + row1.put("test_str", "aaa"); + row1.put( + "test_numerics", + new JSONArray( + new byte[][] { + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("123.4")) + .toByteArray(), + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("-9000000")) + .toByteArray() + })); + row1.put("unknown_field", "a"); + row1.put( + "test_datetime", + CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.of(2020, 10, 1, 12, 0))); + row1.put( + "test_bytestring_repeated", + new JSONArray( + new byte[][] { + ByteString.copyFromUtf8("a").toByteArray(), + ByteString.copyFromUtf8("b").toByteArray() + })); + row1.put("test_timestamp", "2022-02-06 07:24:47.84"); + JSONArray jsonArr1 = new JSONArray(new JSONObject[] {row1}); + + ApiFuture response1 = jsonStreamWriter.append(jsonArr1, -1); + + assertEquals(0, response1.get().getAppendResult().getOffset().getValue()); + + JSONObject row2 = new JSONObject(); + row1.put("test_str", "bbb"); + JSONObject row3 = new JSONObject(); + row2.put("test_str", "ccc"); + JSONArray jsonArr2 = new JSONArray(); + jsonArr2.put(row1); + jsonArr2.put(row2); + + JSONObject row4 = new JSONObject(); + row4.put("test_str", "ddd"); + JSONArray jsonArr3 = new JSONArray(); + jsonArr3.put(row4); + + JSONObject row5 = new JSONObject(); + // Add another ARRAY using a more idiomatic way + JSONArray testArr = new JSONArray(); // create empty JSONArray + testArr.put(0, ByteString.copyFromUtf8("a").toByteArray()); // insert 1st bytes array + testArr.put(1, ByteString.copyFromUtf8("b").toByteArray()); // insert 2nd bytes array + row5.put("test_bytestring_repeated", testArr); + JSONArray jsonArr4 = new JSONArray(); + jsonArr4.put(row5); + + LOG.info("Sending three more messages"); + ApiFuture response2 = jsonStreamWriter.append(jsonArr2, -1); + LOG.info("Sending two more messages"); + ApiFuture response3 = jsonStreamWriter.append(jsonArr3, -1); + LOG.info("Sending one more message"); + ApiFuture response4 = jsonStreamWriter.append(jsonArr4, -1); + Assert.assertFalse(response2.get().getAppendResult().hasOffset()); + Assert.assertFalse(response3.get().getAppendResult().hasOffset()); + Assert.assertFalse(response4.get().getAppendResult().hasOffset()); + + TableResult result = + bigquery.listTableData( + tableInfo.getTableId(), BigQuery.TableDataListOption.startIndex(0L)); + Iterator iter = result.getValues().iterator(); + FieldValueList currentRow = iter.next(); + assertEquals("aaa", currentRow.get(0).getStringValue()); + assertEquals("-9000000", currentRow.get(1).getRepeatedValue().get(1).getStringValue()); + assertEquals("2020-10-01T12:00:00", currentRow.get(2).getStringValue()); + assertEquals(2, currentRow.get(3).getRepeatedValue().size()); + assertEquals("Yg==", currentRow.get(3).getRepeatedValue().get(1).getStringValue()); + assertEquals( + Timestamp.valueOf("2022-02-06 07:24:47.84").getTime() * 1000, + currentRow.get(4).getTimestampValue()); // timestamp long of "2022-02-06 07:24:47.84" + assertEquals("bbb", iter.next().get(0).getStringValue()); + assertEquals("ccc", iter.next().get(0).getStringValue()); + assertEquals("ddd", iter.next().get(0).getStringValue()); + FieldValueList currentRow2 = iter.next(); + assertEquals("YQ==", currentRow2.get(3).getRepeatedValue().get(0).getStringValue()); + assertEquals("Yg==", currentRow2.get(3).getRepeatedValue().get(1).getStringValue()); + assertEquals(false, iter.hasNext()); + } } // This test runs about 1 min.