Skip to content

Commit

Permalink
fix: even better ISO compilance (#1607)
Browse files Browse the repository at this point in the history 
Using docs from google
https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json#data_types
as guide, for how types should be parsed

Signed-off-by: dark0dave <dark0dave@mykolab.com>
  • Loading branch information
@dark0dave
dark0dave committed Apr 8, 2022
1 parent 3148c3a commit 5701597
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 10 deletions.
44 changes: 37 additions & 7 deletions ...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.threeten.bp.ZoneOffset;
import org.threeten.bp.format.DateTimeFormatter;
import org.threeten.bp.format.DateTimeFormatterBuilder;
import org.threeten.bp.format.TextStyle;
import org.threeten.bp.temporal.ChronoField;
import org.threeten.bp.temporal.TemporalAccessor;

Expand All @@ -61,7 +62,7 @@ public class JsonToProtoMessage {
private static final DateTimeFormatter timestampFormatter =
new DateTimeFormatterBuilder()
.parseLenient()
.append(DateTimeFormatter.ISO_LOCAL_DATE)
.append(DateTimeFormatter.ofPattern("yyyy[/][-]MM[/][-]dd"))
.optionalStart()
.appendLiteral('T')
.optionalEnd()
Expand All @@ -74,10 +75,24 @@ public class JsonToProtoMessage {
.optionalStart()
.appendLiteral(':')
.appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.optionalEnd()
.optionalStart()
.appendValue(ChronoField.MILLI_OF_SECOND, 3)
.optionalEnd()
.optionalStart()
.appendFraction(ChronoField.MICRO_OF_SECOND, 3, 6, true)
.optionalEnd()
.optionalStart()
.appendFraction(ChronoField.NANO_OF_SECOND, 6, 9, true)
.optionalEnd()
.optionalStart()
.appendOffset("+HHMM", "+00:00")
.appendLiteral(' ')
.optionalEnd()
.optionalStart()
.appendOffset("+HH:MM", "+00:00")
.optionalEnd()
.optionalStart()
.appendZoneText(TextStyle.SHORT)
.optionalEnd()
.optionalStart()
.appendLiteral('Z')
Expand Down Expand Up @@ -336,6 +351,11 @@ private static void fillField(
}
} else if (fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
if (val instanceof String) {
Double parsed = Doubles.tryParse((String) val);
if (parsed != null) {
protoMsg.setField(fieldDescriptor, parsed.longValue() * 10000000);
return;
}
TemporalAccessor parsedTime = timestampFormatter.parse((String) val);
protoMsg.setField(
fieldDescriptor,
Expand All @@ -345,6 +365,9 @@ private static void fillField(
} else if (val instanceof Long) {
protoMsg.setField(fieldDescriptor, (Long) val);
return;
} else if (val instanceof Integer) {
protoMsg.setField(fieldDescriptor, new Long((Integer) val) * 10000000);
return;
}
}
}
Expand Down Expand Up @@ -549,13 +572,20 @@ private static void fillRepeatedField(
} else if (fieldSchema != null
&& fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
if (val instanceof String) {
TemporalAccessor parsedTime = timestampFormatter.parse((String) val);
protoMsg.addRepeatedField(
fieldDescriptor,
parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
+ parsedTime.getLong(ChronoField.MICRO_OF_SECOND));
Double parsed = Doubles.tryParse((String) val);
if (parsed != null) {
protoMsg.addRepeatedField(fieldDescriptor, parsed.longValue() * 10000000);
} else {
TemporalAccessor parsedTime = timestampFormatter.parse((String) val);
protoMsg.addRepeatedField(
fieldDescriptor,
parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
+ parsedTime.getLong(ChronoField.MICRO_OF_SECOND));
}
} else if (val instanceof Long) {
protoMsg.addRepeatedField(fieldDescriptor, (Long) val);
} else if (val instanceof Integer) {
protoMsg.addRepeatedField(fieldDescriptor, new Long((Integer) val) * 10000000);
} else {
fail = true;
}
Expand Down
21 changes: 18 additions & 3 deletions ...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -664,17 +664,32 @@ public void testTimestamp() throws Exception {
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_string_T_Z").build())
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_long").build())
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_int").build())
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_float").build())
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_offset").build())
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_timezone").build())
.addFields(TableFieldSchema.newBuilder(TEST_TIMESTAMP).setName("test_saformat").build())
.build();
TestTimestamp expectedProto =
TestTimestamp.newBuilder()
.setTestString(10L)
.setTestStringTZ(1648493279000000L)
.setTestStringTZ(1648493279010000L)
.setTestLong(0L)
.setTestInt(1534806950000000L)
.setTestFloat(1534680695000000000L)
.setTestOffset(1649135171000000L)
.setTestTimezone(1649174771000000L)
.setTestSaformat(1534680660000000L)
.build();
JSONObject json = new JSONObject();
json.put("test_string", "1970-01-01 00:00:00.000010");
json.put("test_string_T_Z", "2022-03-28T18:47:59.00Z");
json.put("test_string_T_Z", "2022-03-28T18:47:59.01Z");
json.put("test_long", 0L);
json.put("test_int", 153480695);
json.put("test_float", "1.534680695e11");
json.put("test_offset", "2022-04-05T09:06:11+04:00");
json.put("test_timezone", "2022-04-05 09:06:11 PST");
json.put("test_saformat", "2018/08/19 12:11");
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(
TestTimestamp.getDescriptor(), tableSchema, json);
Expand Down Expand Up @@ -860,7 +875,7 @@ public void testStructComplex() throws Exception {
.setTestNumeric(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456")))
.setTestGeo("POINT(1,1)")
.setTestTimestamp(12345678)
.setTestTimestamp(123456780000000L)
.setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1)))
.setTestTimeStr(89332507144L)
.addTestNumericRepeated(
Expand Down
5 changes: 5 additions & 0 deletions google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,11 @@ message TestTimestamp {
optional int64 test_string = 1;
optional int64 test_string_t_z = 2;
optional int64 test_long = 3;
optional int64 test_int = 4;
optional int64 test_float = 5;
optional int64 test_offset = 6;
optional int64 test_timezone = 7;
optional int64 test_saformat = 8;
}

message TestDate {
Expand Down

0 comments on commit 5701597

Please sign in to comment.