From a018d59c459e0b93966811c7693e851a1c373560 Mon Sep 17 00:00:00 2001 From: Lavkesh Lahngir Date: Thu, 12 Jan 2023 07:16:47 +0100 Subject: [PATCH] fix: add a floating point check (#76) --- build.gradle | 2 +- .../converter/MessageRecordConverter.java | 2 +- .../message/proto/ProtoOdpfParsedMessage.java | 13 +++++++ .../proto/ProtoOdpfParsedMessageTest.java | 36 +++++++++---------- .../depot/message/proto/TestProtoUtil.java | 2 ++ src/test/proto/TestMessage.proto | 8 +++++ 6 files changed, 42 insertions(+), 21 deletions(-) diff --git a/build.gradle b/build.gradle index 4add1ad4..7f3dcac4 100644 --- a/build.gradle +++ b/build.gradle @@ -22,7 +22,7 @@ plugins { } group 'io.odpf' -version '0.3.6' +version '0.3.7' repositories { mavenCentral() diff --git a/src/main/java/io/odpf/depot/bigquery/converter/MessageRecordConverter.java b/src/main/java/io/odpf/depot/bigquery/converter/MessageRecordConverter.java index 843a8eb7..020371fc 100644 --- a/src/main/java/io/odpf/depot/bigquery/converter/MessageRecordConverter.java +++ b/src/main/java/io/odpf/depot/bigquery/converter/MessageRecordConverter.java @@ -44,7 +44,7 @@ public Records convert(List messages) { } catch (EmptyMessageException | UnsupportedOperationException e) { ErrorInfo errorInfo = new ErrorInfo(e, ErrorType.INVALID_MESSAGE_ERROR); invalidRecords.add(new Record(message.getMetadata(), Collections.emptyMap(), index, errorInfo)); - } catch (DeserializerException e) { + } catch (DeserializerException | IllegalArgumentException e) { ErrorInfo errorInfo = new ErrorInfo(e, ErrorType.DESERIALIZATION_ERROR); invalidRecords.add(new Record(message.getMetadata(), Collections.emptyMap(), index, errorInfo)); } diff --git a/src/main/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessage.java b/src/main/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessage.java index 5f745dc7..7081699a 100644 --- a/src/main/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessage.java +++ b/src/main/java/io/odpf/depot/message/proto/ProtoOdpfParsedMessage.java @@ -1,6 +1,7 @@ package io.odpf.depot.message.proto; import com.google.api.client.util.DateTime; +import com.google.api.client.util.Preconditions; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import io.odpf.depot.common.Tuple; @@ -89,6 +90,7 @@ private Map getMappings(DynamicMessage message, Properties colum Tuple nestedColumns = getNestedColumnName(field, value); row.put(nestedColumns.getFirst(), nestedColumns.getSecond()); } else { + floatCheck(fieldValue); row.put(columnName, fieldValue); } } @@ -96,6 +98,16 @@ private Map getMappings(DynamicMessage message, Properties colum return row; } + private void floatCheck(Object fieldValue) { + if (fieldValue instanceof Float) { + float floatValue = ((Number) fieldValue).floatValue(); + Preconditions.checkArgument(!Float.isInfinite(floatValue) && !Float.isNaN(floatValue)); + } else if (fieldValue instanceof Double) { + double doubleValue = ((Number) fieldValue).doubleValue(); + Preconditions.checkArgument(!Double.isInfinite(doubleValue) && !Double.isNaN(doubleValue)); + } + } + private Tuple getNestedColumnName(Object field, Object value) { try { String columnName = getNestedColumnName((Properties) value); @@ -127,6 +139,7 @@ private void addRepeatedFields(Map row, Object value, List new ProtoOdpfParsedMessage(message).getMapping(odpfMessageSchema)); + } + @Test public void shouldParseDurationMessageSuccessfully() throws IOException { TestMessageBQ message = TestProtoUtil.generateTestMessage(now); @@ -425,7 +423,7 @@ public void shouldGetRepeatableStructField() throws IOException { JSONArray expectedArray = new JSONArray(); JSONArray actualArray = new JSONArray(); for (int ii = 0; ii < message.getAttributesCount(); ii++) { - expectedArray.put(printer.print(message.getAttributes(ii))); + expectedArray.put(PRINTER.print(message.getAttributes(ii))); actualArray.put(attributes.get(ii)); } Assert.assertEquals(expectedArray.toString(), actualArray.toString()); diff --git a/src/test/java/io/odpf/depot/message/proto/TestProtoUtil.java b/src/test/java/io/odpf/depot/message/proto/TestProtoUtil.java index b0b4db40..7f1d5779 100644 --- a/src/test/java/io/odpf/depot/message/proto/TestProtoUtil.java +++ b/src/test/java/io/odpf/depot/message/proto/TestProtoUtil.java @@ -15,6 +15,7 @@ public class TestProtoUtil { public static final int TRIP_DURATION_NANOS = 1000; private static final long TRIP_DURATION_SECONDS_1 = 12; private static final long TRIP_DURATION_SECONDS_2 = 15; + private static final float PRICE = 12.12f; private static int call = 0; public static TestMessageBQ generateTestMessage(Instant now) { @@ -25,6 +26,7 @@ public static TestMessageBQ generateTestMessage(Instant now) { .setOrderUrl("order-url-" + call) .setOrderDetails("order-details-" + call) .setCreatedAt(createdAt) + .setPrice(PRICE) .setStatus(StatusBQ.COMPLETED) .setTripDuration(Duration.newBuilder().setSeconds(1).setNanos(TRIP_DURATION_NANOS).build()) .addUpdatedAt(createdAt) diff --git a/src/test/proto/TestMessage.proto b/src/test/proto/TestMessage.proto index d6ac1a92..8e838ecc 100644 --- a/src/test/proto/TestMessage.proto +++ b/src/test/proto/TestMessage.proto @@ -16,6 +16,14 @@ message TestKey { string order_url = 2; } +message FloatTest { + FloatTestContainer container = 68; +} + +message FloatTestContainer { + float value = 4; +} + message TestMessage { string order_number = 1; string order_url = 2;