fix: add precision overwritten to 9 digit if the passed in JSON type …

…is FLOAT or DOUBLE (#1932) * feat: Split writer into connection worker and wrapper, this is a prerequisite for multiplexing client * feat: add connection worker pool skeleton, used for multiplexing client * feat: add Load api for connection worker for multiplexing client * feat: add multiplexing support to connection worker. We will treat every new stream name as a switch of destinationt * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: port the multiplexing client core algorithm and basic tests also fixed a tiny bug inside fake bigquery write impl for getting thre response from offset * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: wire multiplexing connection pool to stream writer * feat: some fixes for multiplexing client * feat: fix some todos, and reject the mixed behavior of passed in client or not * feat: fix the bug that we may peek into the write_stream field but it's possible the proto schema does not contain this field * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: fix the bug that we may peek into the write_stream field but it's possible the proto schema does not contain this field * feat: add getInflightWaitSeconds implementation * feat: Add schema comparision in connection loop to ensure schema update for the same stream name can be notified * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: add schema update support to multiplexing * fix: fix windows build bug: windows Instant resolution is different with linux * fix: fix another failing tests for windows build * fix: fix another test failure for Windows build * feat: Change new thread for each retry to be a thread pool to avoid create/tear down too much threads if lots of retries happens * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: add back the background executor provider that's accidentally removed * feat: throw error when use connection pool for explicit stream * fix: Add precision truncation to the passed in value from JSON float and double type. * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * modify the bom version * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
googleapis · Jan 18, 2023 · 417bc6c · 417bc6c
1 parent 2d648cf
commit 417bc6c
Show file tree

Hide file tree

Showing 3 changed files with 70 additions and 5 deletions.
diff --git a/README.md b/README.md
@@ -49,7 +49,7 @@ If you are using Maven without BOM, add this to your dependencies:
 If you are using Gradle 5.x or later, add this to your dependencies:
 
 ```Groovy
-implementation platform('com.google.cloud:libraries-bom:26.3.0')
+implementation platform('com.google.cloud:libraries-bom:26.4.0')
 
 implementation 'com.google.cloud:google-cloud-bigquerystorage'
 ```

diff --git a/...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
@@ -28,6 +28,7 @@
 import com.google.protobuf.Message;
 import com.google.protobuf.UninitializedMessageException;
 import java.math.BigDecimal;
+import java.math.RoundingMode;
 import java.time.LocalDate;
 import java.util.List;
 import java.util.logging.Logger;
@@ -49,6 +50,7 @@
  */
 public class JsonToProtoMessage {
   private static final Logger LOG = Logger.getLogger(JsonToProtoMessage.class.getName());
+  private static int NUMERIC_SCALE = 9;
   private static ImmutableMap<FieldDescriptor.Type, String> FieldTypeToDebugMessage =
       new ImmutableMap.Builder<FieldDescriptor.Type, String>()
           .put(FieldDescriptor.Type.BOOL, "boolean")
@@ -315,10 +317,15 @@ private static void fillField(
                       new BigDecimal(((Number) val).longValue())));
               return;
             } else if (val instanceof Float || val instanceof Double) {
+              // In JSON, the precision passed in is machine dependent. We should round the number
+              // before passing to backend.
+              BigDecimal bigDecimal = new BigDecimal(String.valueOf(val));
+              if (bigDecimal.scale() > 9) {
+                bigDecimal = bigDecimal.setScale(NUMERIC_SCALE, RoundingMode.HALF_UP);
+              }
               protoMsg.setField(
                   fieldDescriptor,
-                  BigDecimalByteStringEncoder.encodeToNumericByteString(
-                      new BigDecimal(String.valueOf(val))));
+                  BigDecimalByteStringEncoder.encodeToNumericByteString(bigDecimal));
               return;
             } else if (val instanceof BigDecimal) {
               protoMsg.setField(
@@ -559,10 +566,13 @@ private static void fillRepeatedField(
                       new BigDecimal(((Number) val).longValue())));
               added = true;
             } else if (val instanceof Float || val instanceof Double) {
+              BigDecimal bigDecimal = new BigDecimal(String.valueOf(val));
+              if (bigDecimal.scale() > 9) {
+                bigDecimal = bigDecimal.setScale(NUMERIC_SCALE, RoundingMode.HALF_UP);
+              }
               protoMsg.addRepeatedField(
                   fieldDescriptor,
-                  BigDecimalByteStringEncoder.encodeToNumericByteString(
-                      new BigDecimal(String.valueOf(val))));
+                  BigDecimalByteStringEncoder.encodeToNumericByteString(bigDecimal));
               added = true;
             } else if (val instanceof BigDecimal) {
               protoMsg.addRepeatedField(

diff --git a/...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
@@ -20,6 +20,7 @@
 
 import com.google.cloud.bigquery.storage.test.JsonTest.*;
 import com.google.cloud.bigquery.storage.test.SchemaTest.*;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.Descriptors.Descriptor;
@@ -685,6 +686,60 @@ public void testDouble() throws Exception {
     assertEquals(expectedProto, protoMsg);
   }
 
+  @Test
+  public void testDoubleHighPrecision() throws Exception {
+    TableSchema tableSchema =
+        TableSchema.newBuilder()
+            .addFields(
+                TableFieldSchema.newBuilder()
+                    .setName("numeric")
+                    .setType(TableFieldSchema.Type.NUMERIC)
+                    .build())
+            .build();
+    TestNumeric expectedProto =
+        TestNumeric.newBuilder()
+            .setNumeric(
+                BigDecimalByteStringEncoder.encodeToNumericByteString(
+                    new BigDecimal("3.400500513")))
+            .build();
+    JSONObject json = new JSONObject();
+    json.put("numeric", 3.400500512978076);
+    DynamicMessage protoMsg =
+        JsonToProtoMessage.convertJsonToProtoMessage(
+            TestNumeric.getDescriptor(), tableSchema, json);
+    assertEquals(expectedProto, protoMsg);
+  }
+
+  @Test
+  public void testDoubleHighPrecision_RepeatedField() throws Exception {
+    TableSchema tableSchema =
+        TableSchema.newBuilder()
+            .addFields(
+                0,
+                TableFieldSchema.newBuilder()
+                    .setName("bignumeric")
+                    .setType(TableFieldSchema.Type.NUMERIC)
+                    .setMode(TableFieldSchema.Mode.REPEATED)
+                    .build())
+            .build();
+    TestBignumeric expectedProto =
+        TestBignumeric.newBuilder()
+            .addBignumeric(
+                BigDecimalByteStringEncoder.encodeToNumericByteString(
+                    new BigDecimal("3.400500513")))
+            .addBignumeric(
+                BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0.1")))
+            .addBignumeric(
+                BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0.12")))
+            .build();
+    JSONObject json = new JSONObject();
+    json.put("bignumeric", ImmutableList.of(3.400500512978076, 0.10000000000055, 0.12));
+    DynamicMessage protoMsg =
+        JsonToProtoMessage.convertJsonToProtoMessage(
+            TestBignumeric.getDescriptor(), tableSchema, json);
+    assertEquals(expectedProto, protoMsg);
+  }
+
   @Test
   public void testTimestamp() throws Exception {
     TableSchema tableSchema =