From 04722776ed0d1f71c2ef58fa6fb1707789e05958 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Wed, 11 Sep 2024 08:02:56 +0530 Subject: [PATCH 01/13] fix: cleaning up adapter module --- java/adapter/avro/pom.xml | 13 ------------- java/adapter/jdbc/pom.xml | 9 --------- java/adapter/orc/pom.xml | 9 --------- java/pom.xml | 1 + 4 files changed, 1 insertion(+), 31 deletions(-) diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml index 2c02e72e9c838..9acd9ba6ffd02 100644 --- a/java/adapter/avro/pom.xml +++ b/java/adapter/avro/pom.xml @@ -57,17 +57,4 @@ under the License. - - - - org.apache.maven.plugins - maven-compiler-plugin - - - -Werror - - - - - diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 5ebb4089cf72f..099798a95cd25 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -116,15 +116,6 @@ under the License. --add-reads=org.apache.arrow.adapter.jdbc=com.fasterxml.jackson.dataformat.yaml --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -Duser.timezone=UTC - - org.apache.maven.plugins - maven-compiler-plugin - - - -Werror - - - diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index cf35397c9917b..d9cd2bb21a526 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -160,15 +160,6 @@ under the License. - - org.apache.maven.plugins - maven-compiler-plugin - - - -Werror - - - diff --git a/java/pom.xml b/java/pom.xml index 808b0ad4d8cc7..4633380bfb044 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -934,6 +934,7 @@ under the License. -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + -Werror From 369ad2d5efe84df8a66b70080a4757990e3f21bd Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Wed, 11 Sep 2024 08:05:40 +0530 Subject: [PATCH 02/13] fix: removing plugin --- java/c/pom.xml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/java/c/pom.xml b/java/c/pom.xml index fe57bd2ea0ec5..52962354047b1 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -91,16 +91,5 @@ under the License. - - - org.apache.maven.plugins - maven-compiler-plugin - - - -Werror - - - - From 2f91c9b2a79f6d2950946b416f5ae544c5751ce2 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Wed, 11 Sep 2024 08:16:02 +0530 Subject: [PATCH 03/13] fix: removing configs from dataset module --- java/dataset/pom.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml index 92b67825517c6..a19e934f0de98 100644 --- a/java/dataset/pom.xml +++ b/java/dataset/pom.xml @@ -202,15 +202,6 @@ under the License. - - org.apache.maven.plugins - maven-compiler-plugin - - - -Werror - - - From 0e79ee9408bfc05abe5a55ac2030478cc2848172 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Wed, 11 Sep 2024 10:21:19 +0530 Subject: [PATCH 04/13] fix: cleaning vector module (unexpected) --- java/pom.xml | 2 +- java/vector/pom.xml | 14 +++----- .../src/main/codegen/templates/ArrowType.java | 3 ++ java/vector/src/main/java/module-info.java | 1 + .../apache/arrow/vector/AllocationHelper.java | 2 +- .../vector/BaseLargeVariableWidthVector.java | 3 ++ .../arrow/vector/BaseVariableWidthVector.java | 3 ++ .../vector/BaseVariableWidthViewVector.java | 33 ++++++++++++++++--- .../apache/arrow/vector/BitVectorHelper.java | 4 +-- .../arrow/vector/LargeVarBinaryVector.java | 1 + .../org/apache/arrow/vector/NullVector.java | 15 ++++++++- .../arrow/vector/TimeStampSecTZVector.java | 1 + .../apache/arrow/vector/VarBinaryVector.java | 1 + .../apache/arrow/vector/VarCharVector.java | 1 + .../apache/arrow/vector/VectorSchemaRoot.java | 20 +++++++++-- .../arrow/vector/ViewVarBinaryVector.java | 7 ++++ .../arrow/vector/ViewVarCharVector.java | 7 ++++ .../BaseLargeRepeatedValueViewVector.java | 4 +-- .../complex/BaseRepeatedValueViewVector.java | 12 +++---- .../vector/complex/LargeListViewVector.java | 4 +-- .../arrow/vector/complex/ListViewVector.java | 5 ++- .../vector/compression/CompressionCodec.java | 3 +- .../org/apache/arrow/vector/types/Types.java | 1 - .../arrow/vector/types/pojo/Schema.java | 4 +++ .../arrow/vector/util/IntObjectHashMap.java | 4 +-- .../org/apache/arrow/vector/util/Text.java | 8 +++-- 26 files changed, 122 insertions(+), 41 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 4633380bfb044..ba2df05cbd976 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -923,7 +923,7 @@ under the License. -XDcompilePolicy=simple - -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-source|format/src/main/java/org/apache/arrow/flatbuf)/.* + -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-source|target/generated-sources|format/src/main/java/org/apache/arrow/flatbuf)/.* -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED diff --git a/java/vector/pom.xml b/java/vector/pom.xml index eb0e39565332e..db4ebaffac111 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -86,6 +86,11 @@ under the License. hamcrest test + + com.google.errorprone + error_prone_annotations + compile + @@ -118,15 +123,6 @@ under the License. - - org.apache.maven.plugins - maven-compiler-plugin - - - -Werror - - - org.apache.drill.tools diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java index b08d4ad0afac0..dd2440b774a31 100644 --- a/java/vector/src/main/codegen/templates/ArrowType.java +++ b/java/vector/src/main/codegen/templates/ArrowType.java @@ -21,6 +21,7 @@ package org.apache.arrow.vector.types.pojo; +import com.google.errorprone.annotations.Immutable; import com.google.flatbuffers.FlatBufferBuilder; import java.util.Objects; @@ -49,6 +50,8 @@ @JsonSubTypes.Type(value = ArrowType.${type.name?remove_ending("_")}.class, name = "${type.name?remove_ending("_")?lower_case}"), }) + +@Immutable public abstract class ArrowType { public static abstract class PrimitiveType extends ArrowType { diff --git a/java/vector/src/main/java/module-info.java b/java/vector/src/main/java/module-info.java index 8ba1b3579e0e1..3e6c098447ddc 100644 --- a/java/vector/src/main/java/module-info.java +++ b/java/vector/src/main/java/module-info.java @@ -48,6 +48,7 @@ requires org.apache.arrow.memory.core; requires org.apache.commons.codec; requires org.slf4j; + requires com.google.errorprone.annotations; uses org.apache.arrow.vector.compression.CompressionCodec.Factory; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java index aae833c7addbc..801e7b4206513 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java @@ -50,7 +50,7 @@ public static void allocatePrecomputedChildCount( if (v instanceof FixedWidthVector) { ((FixedWidthVector) v).allocateNew(valueCount); } else if (v instanceof VariableWidthVector) { - ((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount); + ((VariableWidthVector) v).allocateNew(valueCount * ((long) bytesPerValue), valueCount); } else if (v instanceof RepeatedFixedWidthVectorLike) { ((RepeatedFixedWidthVectorLike) v).allocateNew(valueCount, childValCount); } else if (v instanceof RepeatedVariableWidthVectorLike) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java index 552a896ea8c36..0ef1688e1d867 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseLargeVariableWidthVector.java @@ -1006,6 +1006,7 @@ public void setIndexDefined(int index) { * @param index position of the element to set * @param length length of the element */ + @Override public void setValueLengthSafe(int index, int length) { assert index >= 0; handleSafe(index, length); @@ -1091,6 +1092,7 @@ public void set(int index, byte[] value, int start, int length) { * @param start start index in array of bytes * @param length length of data in array of bytes */ + @Override public void setSafe(int index, byte[] value, int start, int length) { assert index >= 0; handleSafe(index, length); @@ -1128,6 +1130,7 @@ public void set(int index, ByteBuffer value, int start, int length) { * @param start start index in ByteBuffer * @param length length of data in ByteBuffer */ + @Override public void setSafe(int index, ByteBuffer value, int start, int length) { assert index >= 0; handleSafe(index, length); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java index aaccec602f292..030a6515fe157 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java @@ -1129,6 +1129,7 @@ public void setSafe(int index, byte[] value) { * @param start start index in array of bytes * @param length length of data in array of bytes */ + @Override public void set(int index, byte[] value, int start, int length) { assert index >= 0; fillHoles(index); @@ -1146,6 +1147,7 @@ public void set(int index, byte[] value, int start, int length) { * @param start start index in array of bytes * @param length length of data in array of bytes */ + @Override public void setSafe(int index, byte[] value, int start, int length) { assert index >= 0; handleSafe(index, length); @@ -1183,6 +1185,7 @@ public void set(int index, ByteBuffer value, int start, int length) { * @param start start index in ByteBuffer * @param length length of data in ByteBuffer */ + @Override public void setSafe(int index, ByteBuffer value, int start, int length) { assert index >= 0; handleSafe(index, length); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index aee5233f9d466..498d559aedf9e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -401,7 +401,7 @@ private void setReaderAndWriterIndex() { viewBuffer.writerIndex(0); } else { validityBuffer.writerIndex(getValidityBufferSizeFromCount(valueCount)); - viewBuffer.writerIndex(valueCount * ELEMENT_SIZE); + viewBuffer.writerIndex((long) valueCount * ELEMENT_SIZE); } } @@ -619,7 +619,7 @@ public void reallocValidityBuffer() { } private long computeValidityBufferSize(int valueCount) { - return (valueCount + 7) / 8; + return (valueCount + 7L) / 8; } /** @@ -941,7 +941,7 @@ private void splitAndTransferViewBufferAndDataBuffer( } // allocate target view buffer - target.viewBuffer = target.allocator.buffer(length * ELEMENT_SIZE); + target.viewBuffer = target.allocator.buffer(length * ((long) ELEMENT_SIZE)); for (int i = startIndex; i < startIndex + length; i++) { final int stringLength = getValueLength(i); @@ -1153,6 +1153,7 @@ public int getValueLength(int index) { * @param index position of the element to set * @param value array of bytes to write */ + @Override public void set(int index, byte[] value) { assert index >= 0; BitVectorHelper.setBit(validityBuffer, index); @@ -1185,6 +1186,7 @@ public void setSafe(int index, byte[] value) { * @param start start index in an array of bytes * @param length length of data in an array of bytes */ + @Override public void set(int index, byte[] value, int start, int length) { assert index >= 0; BitVectorHelper.setBit(validityBuffer, index); @@ -1201,6 +1203,7 @@ public void set(int index, byte[] value, int start, int length) { * @param start start index in an array of bytes * @param length length of data in an array of bytes */ + @Override public void setSafe(int index, byte[] value, int start, int length) { assert index >= 0; handleSafe(index, length); @@ -1217,10 +1220,20 @@ public void setSafe(int index, byte[] value, int start, int length) { * @param start start index in ByteBuffer * @param length length of data in ByteBuffer */ + @Override public void set(int index, ByteBuffer value, int start, int length) { assert index >= 0; BitVectorHelper.setBit(validityBuffer, index); - setBytes(index, value.array(), start, length); + if (value.hasArray()) { + setBytes(index, value.array(), value.arrayOffset() + start, length); + } else { + byte[] bytes = new byte[length]; + int originalPosition = value.position(); + value.position(start); + value.get(bytes, 0, length); + value.position(originalPosition); // Restores the buffer position + setBytes(index, bytes, 0, length); + } lastSet = index; } @@ -1233,11 +1246,21 @@ public void set(int index, ByteBuffer value, int start, int length) { * @param start start index in ByteBuffer * @param length length of data in ByteBuffer */ + @Override public void setSafe(int index, ByteBuffer value, int start, int length) { assert index >= 0; handleSafe(index, length); BitVectorHelper.setBit(validityBuffer, index); - setBytes(index, value.array(), start, length); + if (value.hasArray()) { + setBytes(index, value.array(), value.arrayOffset() + start, length); + } else { + byte[] bytes = new byte[length]; + int originalPosition = value.position(); + value.position(start); + value.get(bytes, 0, length); + value.position(originalPosition); // Restores the buffer position + setBytes(index, bytes, 0, length); + } lastSet = index; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index 0ac56691a6f6c..ad9bb77f1520d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -296,7 +296,7 @@ public static byte getBitsFromCurrentByte( /** Returns the byte at index from left-shifted by (8 - offset). */ public static byte getBitsFromNextByte(ArrowBuf data, int index, int offset) { - return (byte) ((data.getByte(index) << (8 - offset))); + return (byte) (data.getByte(index) << (8 - offset)); } /** @@ -385,7 +385,7 @@ public static void concatBits( } // copy the first bit set - if (input1 != output) { + if (!input1.equals(output)) { MemoryUtil.copyMemory(input1.memoryAddress(), output.memoryAddress(), numBytes1); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java index f38627b933b6c..1ef859936017c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/LargeVarBinaryVector.java @@ -93,6 +93,7 @@ public MinorType getMinorType() { * @param index position of element to get * @return array of bytes for non-null element, null otherwise */ + @Override public byte[] get(int index) { assert index >= 0; if (isSet(index) == 0) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java index 227ca716f6391..36509d08c6644 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/NullVector.java @@ -18,6 +18,7 @@ import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; +import com.google.errorprone.annotations.InlineMe; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -94,6 +95,15 @@ public NullVector(Field field, int valueCount) { } @Deprecated + @InlineMe( + replacement = + "this(new Field(DATA_VECTOR_NAME, FieldType.nullable(new ArrowType.Null()), null))", + imports = { + "DATA_VECTOR_NAME", + "org.apache.arrow.vector.types.pojo.ArrowType", + "org.apache.arrow.vector.types.pojo.Field", + "org.apache.arrow.vector.types.pojo.FieldType" + }) public NullVector() { this(new Field(DATA_VECTOR_NAME, FieldType.nullable(new ArrowType.Null()), null)); } @@ -229,6 +239,7 @@ public List getFieldBuffers() { * vectors. */ @Deprecated + @SuppressWarnings("InlineMeSuggester") @Override public List getFieldInnerVectors() { return Collections.emptyList(); @@ -336,7 +347,9 @@ public TransferImpl(String ref) { @Deprecated public TransferImpl() { - to = new NullVector(); + to = + new NullVector( + new Field(DATA_VECTOR_NAME, FieldType.nullable(new ArrowType.Null()), null)); } public TransferImpl(NullVector to) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java index f1774f27035cb..125d28c05feb5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TimeStampSecTZVector.java @@ -126,6 +126,7 @@ public void get(int index, NullableTimeStampSecTZHolder holder) { * @param index position of element * @return element at given index */ + @Override public Long getObject(int index) { if (isSet(index) == 0) { return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java index 7196e9c910c13..15538d6cf87d7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarBinaryVector.java @@ -94,6 +94,7 @@ public MinorType getMinorType() { * @param index position of element to get * @return array of bytes for non-null element, null otherwise */ + @Override public byte[] get(int index) { assert index >= 0; if (NULL_CHECKING_ENABLED && isSet(index) == 0) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java index c81e34558c6f6..fa1eecec09b19 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VarCharVector.java @@ -96,6 +96,7 @@ public MinorType getMinorType() { * @param index position of element to get * @return array of bytes for non-null element, null otherwise */ + @Override public byte[] get(int index) { assert index >= 0; if (NULL_CHECKING_ENABLED && isSet(index) == 0) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java index a7cb9ced72141..36d8140126150 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java @@ -347,11 +347,15 @@ public VectorSchemaRoot slice(int index, int length) { return new VectorSchemaRoot(sliceVectors); } - /** Determine if two VectorSchemaRoots are exactly equal. */ - public boolean equals(VectorSchemaRoot other) { - if (other == null) { + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof VectorSchemaRoot)) { return false; } + VectorSchemaRoot other = (VectorSchemaRoot) obj; if (!this.schema.equals(other.schema)) { return false; @@ -372,6 +376,16 @@ public boolean equals(VectorSchemaRoot other) { return true; } + @Override + public int hashCode() { + int result = schema.hashCode(); + result = 31 * result + rowCount; + for (FieldVector vector : fieldVectors) { + result = 31 * result + (vector != null ? vector.hashCode() : 0); + } + return result; + } + /** * Determine if two VectorSchemaRoots are approximately equal using the given functions to * calculate difference between float/double values. Note that approx equals are in regards to diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java index 9481831fbd143..010772c075cbd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java @@ -18,6 +18,7 @@ import static org.apache.arrow.vector.NullCheckingForGet.NULL_CHECKING_ENABLED; +import com.google.errorprone.annotations.DoNotCall; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.ReusableBuffer; import org.apache.arrow.vector.complex.impl.ViewVarBinaryReaderImpl; @@ -94,6 +95,7 @@ public MinorType getMinorType() { * @param index position of an element to get * @return array of bytes for a non-null element, null otherwise */ + @Override public byte[] get(int index) { assert index >= 0; if (NULL_CHECKING_ENABLED && isSet(index) == 0) { @@ -131,6 +133,7 @@ public byte[] getObject(int index) { * @param index position of an element to get * @param holder data holder to be populated by this function */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void get(int index, NullableViewVarBinaryHolder holder) { // TODO: https://github.com/apache/arrow/issues/40936 throw new UnsupportedOperationException("Unsupported operation"); @@ -149,6 +152,7 @@ public void get(int index, NullableViewVarBinaryHolder holder) { * @param index position of the element to set * @param holder holder that carries data buffer. */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void set(int index, ViewVarBinaryHolder holder) { // TODO: https://github.com/apache/arrow/issues/40936 throw new UnsupportedOperationException("Unsupported operation"); @@ -161,6 +165,7 @@ public void set(int index, ViewVarBinaryHolder holder) { * @param index position of the element to set * @param holder holder that carries data buffer. */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void setSafe(int index, ViewVarBinaryHolder holder) { // TODO: https://github.com/apache/arrow/issues/40936 throw new UnsupportedOperationException("Unsupported operation"); @@ -173,6 +178,7 @@ public void setSafe(int index, ViewVarBinaryHolder holder) { * @param index position of the element to set * @param holder holder that carries data buffer. */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void set(int index, NullableViewVarBinaryHolder holder) { // TODO: https://github.com/apache/arrow/issues/40936 throw new UnsupportedOperationException("Unsupported operation"); @@ -185,6 +191,7 @@ public void set(int index, NullableViewVarBinaryHolder holder) { * @param index position of the element to set * @param holder holder that carries data buffer. */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void setSafe(int index, NullableViewVarBinaryHolder holder) { // TODO: https://github.com/apache/arrow/issues/40936 throw new UnsupportedOperationException("Unsupported operation"); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java index 900e0493fe804..309e45753beac 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java @@ -18,6 +18,7 @@ import static org.apache.arrow.vector.NullCheckingForGet.NULL_CHECKING_ENABLED; +import com.google.errorprone.annotations.DoNotCall; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.ReusableBuffer; import org.apache.arrow.vector.complex.impl.ViewVarCharReaderImpl; @@ -98,6 +99,7 @@ public MinorType getMinorType() { * @param index position of an element to get * @return array of bytes for a non-null element, null otherwise */ + @Override public byte[] get(int index) { assert index >= 0; if (NULL_CHECKING_ENABLED && isSet(index) == 0) { @@ -142,6 +144,7 @@ public void read(int index, ReusableBuffer buffer) { * @param index position of an element to get * @param holder data holder to be populated by this function */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void get(int index, NullableViewVarCharHolder holder) { // TODO: https://github.com/apache/arrow/issues/40937 throw new UnsupportedOperationException( @@ -161,6 +164,7 @@ public void get(int index, NullableViewVarCharHolder holder) { * @param index position of the element to set * @param holder holder that carries data buffer. */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void set(int index, ViewVarCharHolder holder) { // TODO: https://github.com/apache/arrow/issues/40937 throw new UnsupportedOperationException("ViewVarCharHolder set operation not supported"); @@ -173,6 +177,7 @@ public void set(int index, ViewVarCharHolder holder) { * @param index position of the element to set * @param holder holder that carries data buffer. */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void setSafe(int index, ViewVarCharHolder holder) { // TODO: https://github.com/apache/arrow/issues/40937 throw new UnsupportedOperationException("ViewVarCharHolder setSafe operation not supported"); @@ -185,6 +190,7 @@ public void setSafe(int index, ViewVarCharHolder holder) { * @param index position of the element to set * @param holder holder that carries data buffer. */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void set(int index, NullableViewVarCharHolder holder) { // TODO: https://github.com/apache/arrow/issues/40937 throw new UnsupportedOperationException( @@ -198,6 +204,7 @@ public void set(int index, NullableViewVarCharHolder holder) { * @param index position of the element to set * @param holder holder that carries data buffer. */ + @DoNotCall("Always throws java.lang.UnsupportedOperationException") public void setSafe(int index, NullableViewVarCharHolder holder) { // TODO: https://github.com/apache/arrow/issues/40937 throw new UnsupportedOperationException( diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java index 12edd6557bd9c..b9a40b0d4f0de 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseLargeRepeatedValueViewVector.java @@ -182,8 +182,8 @@ public FieldVector getDataVector() { @Override public void setInitialCapacity(int numRecords) { - offsetAllocationSizeInBytes = (long) (numRecords) * OFFSET_WIDTH; - sizeAllocationSizeInBytes = (long) (numRecords) * SIZE_WIDTH; + offsetAllocationSizeInBytes = (long) numRecords * OFFSET_WIDTH; + sizeAllocationSizeInBytes = (long) numRecords * SIZE_WIDTH; if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) { vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); } else { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java index e6213316b55a3..6ade943687448 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueViewVector.java @@ -181,8 +181,8 @@ public FieldVector getDataVector() { @Override public void setInitialCapacity(int numRecords) { - offsetAllocationSizeInBytes = (numRecords) * OFFSET_WIDTH; - sizeAllocationSizeInBytes = (numRecords) * SIZE_WIDTH; + offsetAllocationSizeInBytes = (long) numRecords * OFFSET_WIDTH; + sizeAllocationSizeInBytes = (long) numRecords * SIZE_WIDTH; if (vector instanceof BaseFixedWidthVector || vector instanceof BaseVariableWidthVector) { vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD); } else { @@ -196,8 +196,8 @@ public void setInitialCapacity(int numRecords, double density) { throw new OversizedAllocationException("Requested amount of memory is more than max allowed"); } - offsetAllocationSizeInBytes = numRecords * OFFSET_WIDTH; - sizeAllocationSizeInBytes = numRecords * SIZE_WIDTH; + offsetAllocationSizeInBytes = (long) numRecords * OFFSET_WIDTH; + sizeAllocationSizeInBytes = (long) numRecords * SIZE_WIDTH; int innerValueCapacity = Math.max((int) (numRecords * density), 1); @@ -222,8 +222,8 @@ public void setInitialCapacity(int numRecords, double density) { * all records. */ public void setInitialTotalCapacity(int numRecords, int totalNumberOfElements) { - offsetAllocationSizeInBytes = numRecords * OFFSET_WIDTH; - sizeAllocationSizeInBytes = numRecords * SIZE_WIDTH; + offsetAllocationSizeInBytes = (long) numRecords * OFFSET_WIDTH; + sizeAllocationSizeInBytes = (long) numRecords * SIZE_WIDTH; vector.setInitialCapacity(totalNumberOfElements); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java index 84c6f03edb25d..1d3f09ae4c3a5 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/LargeListViewVector.java @@ -739,8 +739,8 @@ public List getObject(int index) { return null; } final List vals = new JsonStringArrayList<>(); - final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); - final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH); + final int start = offsetBuffer.getInt((long) index * OFFSET_WIDTH); + final int end = start + sizeBuffer.getInt((long) index * SIZE_WIDTH); final ValueVector vv = getDataVector(); for (int i = start; i < end; i++) { vals.add(vv.getObject(checkedCastToInt(i))); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java index 9b4e6b4c0cd4a..53189278694af 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java @@ -499,7 +499,6 @@ public void splitAndTransfer(int startIndex, int length) { valueCount); to.clear(); if (length > 0) { - final int startPoint = offsetBuffer.getInt((long) startIndex * OFFSET_WIDTH); // we have to scan by index since there are out-of-order offsets to.offsetBuffer = to.allocateBuffers((long) length * OFFSET_WIDTH); to.sizeBuffer = to.allocateBuffers((long) length * SIZE_WIDTH); @@ -744,8 +743,8 @@ public List getObject(int index) { return null; } final List vals = new JsonStringArrayList<>(); - final int start = offsetBuffer.getInt(index * OFFSET_WIDTH); - final int end = start + sizeBuffer.getInt((index) * SIZE_WIDTH); + final int start = offsetBuffer.getInt((long) index * OFFSET_WIDTH); + final int end = start + sizeBuffer.getInt((long) index * SIZE_WIDTH); final ValueVector vv = getDataVector(); for (int i = start; i < end; i++) { vals.add(vv.getObject(i)); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionCodec.java b/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionCodec.java index dd62108a84a6b..b1e00ee6ef687 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionCodec.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compression/CompressionCodec.java @@ -80,7 +80,8 @@ private static Factory bestEffort() { try { factory.createCodec(codecType); // will throw if not supported factories.putIfAbsent(codecType, factory); - } catch (Throwable ignored) { + } catch (Throwable expected) { + // ignore } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 6b2c56de01c40..5859f245ef955 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -787,7 +787,6 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { } }, ; - private final ArrowType type; MinorType(ArrowType type) { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java index 293f1499df218..7b2cc1859a383 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; +import com.google.errorprone.annotations.InlineMe; import com.google.flatbuffers.FlatBufferBuilder; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -90,6 +91,9 @@ public static Schema fromJSON(String json) throws IOException { * @return The deserialized schema. */ @Deprecated + @InlineMe( + replacement = "Schema.convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(buffer))", + imports = "org.apache.arrow.vector.types.pojo.Schema") public static Schema deserialize(ByteBuffer buffer) { return convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(buffer)); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java b/java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java index b625f602caa9c..7b6a8a6d91b5a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java @@ -412,8 +412,8 @@ private boolean removeAt(final int index) { for (V value = values[i]; value != null; value = values[i = probeNext(i)]) { int key = keys[i]; int bucket = hashIndex(key); - if (i < bucket && (bucket <= nextFree || nextFree <= i) - || bucket <= nextFree && nextFree <= i) { + if ((i < bucket && (bucket <= nextFree || nextFree <= i)) + || (bucket <= nextFree && nextFree <= i)) { // Move the displaced entry "back" to the first available position. keys[nextFree] = key; values[nextFree] = value; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java index 35d810abbb633..4fd2518687264 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Text.java @@ -200,8 +200,12 @@ public int find(String what, int start) { public void set(String string) { try { ByteBuffer bb = encode(string, true); - bytes = bb.array(); - length = bb.limit(); + byte[] bytes = new byte[bb.remaining()]; + int originalPosition = bb.position(); + bb.get(bytes); + bb.position(originalPosition); // Restores the buffer position + this.bytes = bytes; + this.length = bb.limit(); } catch (CharacterCodingException e) { throw new RuntimeException("Should not have happened ", e); } From de95e9c7f191fdb411af495c1f840e8224be86e7 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Wed, 11 Sep 2024 14:02:13 +0530 Subject: [PATCH 05/13] fix: cleaning vector module v2 --- .../org/apache/arrow/vector/ZeroVector.java | 1 + .../vector/compare/RangeEqualsVisitor.java | 22 ++++++++++++------- .../vector/compare/TypeEqualsVisitor.java | 19 +++++++++++++--- .../complex/impl/ComplexWriterImpl.java | 3 +++ .../complex/impl/StructOrListWriterImpl.java | 15 +++++++++++++ .../dictionary/DictionaryHashTable.java | 1 + .../dictionary/StructSubfieldEncoder.java | 2 +- .../arrow/vector/ipc/ArrowStreamReader.java | 1 + .../apache/arrow/vector/ipc/ArrowWriter.java | 9 +------- .../arrow/vector/ipc/JsonFileWriter.java | 14 ++++++------ .../apache/arrow/vector/ipc/WriteChannel.java | 3 --- .../arrow/vector/ipc/message/ArrowBlock.java | 2 +- .../arrow/vector/ipc/message/ArrowBuffer.java | 2 +- .../ipc/message/ArrowDictionaryBatch.java | 3 +++ .../arrow/vector/ipc/message/ArrowFooter.java | 2 +- .../vector/ipc/message/ArrowRecordBatch.java | 8 ++++--- 16 files changed, 71 insertions(+), 36 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index c838de60d841a..8e3d487bbaa09 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -57,6 +57,7 @@ public ZeroVector(Field field) { } @Deprecated + @SuppressWarnings("InlineMeInliner") public ZeroVector() {} @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java index ed51f748af577..23157528a6197 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java @@ -478,12 +478,15 @@ protected boolean compareBaseVariableWidthVectors(Range range) { int offsetWidth = BaseVariableWidthVector.OFFSET_WIDTH; if (!isNull) { - final int startIndexLeft = leftVector.getOffsetBuffer().getInt(leftIndex * offsetWidth); - final int endIndexLeft = leftVector.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + final int startIndexLeft = + leftVector.getOffsetBuffer().getInt((long) leftIndex * offsetWidth); + final int endIndexLeft = + leftVector.getOffsetBuffer().getInt((long) (leftIndex + 1) * offsetWidth); - final int startIndexRight = rightVector.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int startIndexRight = + rightVector.getOffsetBuffer().getInt((long) rightIndex * offsetWidth); final int endIndexRight = - rightVector.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + rightVector.getOffsetBuffer().getInt((long) (rightIndex + 1) * offsetWidth); int ret = ByteFunctionHelpers.equal( @@ -657,12 +660,15 @@ protected boolean compareListVectors(Range range) { int offsetWidth = BaseRepeatedValueVector.OFFSET_WIDTH; if (!isNull) { - final int startIndexLeft = leftVector.getOffsetBuffer().getInt(leftIndex * offsetWidth); - final int endIndexLeft = leftVector.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth); + final int startIndexLeft = + leftVector.getOffsetBuffer().getInt((long) leftIndex * offsetWidth); + final int endIndexLeft = + leftVector.getOffsetBuffer().getInt((long) (leftIndex + 1) * offsetWidth); - final int startIndexRight = rightVector.getOffsetBuffer().getInt(rightIndex * offsetWidth); + final int startIndexRight = + rightVector.getOffsetBuffer().getInt((long) rightIndex * offsetWidth); final int endIndexRight = - rightVector.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth); + rightVector.getOffsetBuffer().getInt((long) (rightIndex + 1) * offsetWidth); if ((endIndexLeft - startIndexLeft) != (endIndexRight - startIndexRight)) { return false; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java index ce92b22ef61c9..ce9cb117ed81f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java @@ -62,10 +62,23 @@ public TypeEqualsVisitor(ValueVector right, boolean checkName, boolean checkMeta } /** Check type equals without passing IN param in VectorVisitor. */ - public boolean equals(ValueVector left) { + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof ValueVector)) { + return false; + } + ValueVector left = (ValueVector) obj; return left.accept(this, null); } + @Override + public int hashCode() { + return Objects.hash(right, checkName, checkMetadata); + } + @Override public Boolean visit(BaseFixedWidthVector left, Void value) { return compareField(left.getField(), right.getField()); @@ -138,12 +151,12 @@ public Boolean visit(LargeListViewVector left, Void value) { private boolean compareField(Field leftField, Field rightField) { - if (leftField == rightField) { + if (leftField.equals(rightField)) { return true; } return (!checkName || Objects.equals(leftField.getName(), rightField.getName())) - && Objects.equals(leftField.isNullable(), rightField.isNullable()) + && (leftField.isNullable() == rightField.isNullable()) && Objects.equals(leftField.getType(), rightField.getType()) && Objects.equals(leftField.getDictionary(), rightField.getDictionary()) && (!checkMetadata || Objects.equals(leftField.getMetadata(), rightField.getMetadata())) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java index f3e48aa050e30..07eb5ad0dcdf9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java @@ -37,7 +37,10 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri Mode mode = Mode.INIT; private final String name; + + @SuppressWarnings("UnusedVariable") private final boolean unionEnabled; + private final NullableStructWriterFactory nullableStructWriterFactory; private enum Mode { diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java index 7dbcbf8babe00..74318222c5fdb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/StructOrListWriterImpl.java @@ -51,6 +51,7 @@ public StructOrListWriterImpl(final BaseWriter.ListWriter writer) { } /** Start writing to either the list or the struct. */ + @Override public void start() { if (struct != null) { struct.start(); @@ -60,6 +61,7 @@ public void start() { } /** Finish writing to the list or struct. */ + @Override public void end() { if (struct != null) { struct.end(); @@ -69,6 +71,7 @@ public void end() { } /** Creates a new writer for a struct with the given name. */ + @Override public StructOrListWriter struct(final String name) { assert struct != null; return new StructOrListWriterImpl(struct.struct(name)); @@ -81,6 +84,7 @@ public StructOrListWriter struct(final String name) { * @deprecated use {@link #listOfStruct(String)} instead. */ @Deprecated + @SuppressWarnings({"InlineMeValidator", "InlineMeSuggester"}) public StructOrListWriter listoftstruct(final String name) { return listOfStruct(name); } @@ -90,48 +94,59 @@ public StructOrListWriter listoftstruct(final String name) { * * @param name Unused. */ + @Override public StructOrListWriter listOfStruct(final String name) { assert list != null; return new StructOrListWriterImpl(list.struct()); } + @Override public StructOrListWriter list(final String name) { assert struct != null; return new StructOrListWriterImpl(struct.list(name)); } + @Override public boolean isStructWriter() { return struct != null; } + @Override public boolean isListWriter() { return list != null; } + @Override public VarCharWriter varChar(final String name) { return (struct != null) ? struct.varChar(name) : list.varChar(); } + @Override public IntWriter integer(final String name) { return (struct != null) ? struct.integer(name) : list.integer(); } + @Override public BigIntWriter bigInt(final String name) { return (struct != null) ? struct.bigInt(name) : list.bigInt(); } + @Override public Float4Writer float4(final String name) { return (struct != null) ? struct.float4(name) : list.float4(); } + @Override public Float8Writer float8(final String name) { return (struct != null) ? struct.float8(name) : list.float8(); } + @Override public BitWriter bit(final String name) { return (struct != null) ? struct.bit(name) : list.bit(); } + @Override public VarBinaryWriter binary(final String name) { return (struct != null) ? struct.varBinary(name) : list.varBinary(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java index 57faf51845c4e..73ec406c7fbbb 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java @@ -246,6 +246,7 @@ public int hashCode() { return hash; } + @Override public final boolean equals(Object o) { if (!(o instanceof DictionaryHashTable.Entry)) { return false; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java index dc25bc32685dd..359583ac483a7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/StructSubfieldEncoder.java @@ -80,7 +80,7 @@ private static StructVector cloneVector(StructVector vector, BufferAllocator all StructVector cloned = (StructVector) fieldType.createNewSingleVector( - vector.getField().getName(), allocator, /*schemaCallback=*/ null); + vector.getField().getName(), allocator, /* schemaCallBack= */ null); final ArrowFieldNode fieldNode = new ArrowFieldNode(vector.getValueCount(), vector.getNullCount()); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java index 69811dc71727c..94eec4be94050 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamReader.java @@ -139,6 +139,7 @@ protected void closeReadSource() throws IOException { * @return true if a batch was read, false on EOS * @throws IOException on error */ + @Override public boolean loadNextBatch() throws IOException { prepareLoadNextBatch(); MessageResult result = messageReader.readNext(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java index c0f2b113bcb54..e8ea452c0322b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java @@ -155,15 +155,8 @@ protected void writeDictionaryBatch(Dictionary dictionary) throws IOException { VectorUnloader unloader = new VectorUnloader(dictRoot, /*includeNullCount*/ true, this.codec, /*alignBuffers*/ true); ArrowRecordBatch batch = unloader.getRecordBatch(); - ArrowDictionaryBatch dictionaryBatch = new ArrowDictionaryBatch(id, batch, false); - try { + try (ArrowDictionaryBatch dictionaryBatch = new ArrowDictionaryBatch(id, batch, false)) { writeDictionaryBatch(dictionaryBatch); - } finally { - try { - dictionaryBatch.close(); - } catch (Exception e) { - throw new RuntimeException("Error occurred while closing dictionary.", e); - } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index 68700fe6afd25..a3fb2be1a57c8 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.IOException; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -248,14 +249,14 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE // writing views ArrowBuf viewBuffer = vectorBuffers.get(1); List dataBuffers = vectorBuffers.subList(v + 1, vectorBuffers.size()); - writeValueToViewGenerator(bufferType, viewBuffer, dataBuffers, vector, i); + writeValueToViewGenerator(viewBuffer, dataBuffers, vector, i); } else if (bufferType.equals(VARIADIC_DATA_BUFFERS) && (vector.getMinorType() == MinorType.VIEWVARCHAR || vector.getMinorType() == MinorType.VIEWVARBINARY)) { ArrowBuf viewBuffer = vectorBuffers.get(1); // check if this is v-1 List dataBuffers = vectorBuffers.subList(v, vectorBuffers.size()); if (!dataBuffers.isEmpty()) { - writeValueToDataBufferGenerator(bufferType, viewBuffer, dataBuffers, vector); + writeValueToDataBufferGenerator(bufferType, viewBuffer, dataBuffers); // The variadic buffers are written at once and doesn't require iterating for // each index. // So, break the loop. @@ -350,7 +351,6 @@ private byte[] getView(final ArrowBuf viewBuffer, final List dataBuffe } private void writeValueToViewGenerator( - BufferType bufferType, ArrowBuf viewBuffer, List dataBuffers, FieldVector vector, @@ -383,7 +383,7 @@ private void writeValueToViewGenerator( } else { generator.writeFieldName("INLINED"); if (vector.getMinorType() == MinorType.VIEWVARCHAR) { - generator.writeString(new String(b, "UTF-8")); + generator.writeString(new String(b, StandardCharsets.UTF_8)); } else { generator.writeString(Hex.encodeHexString(b)); } @@ -392,7 +392,7 @@ private void writeValueToViewGenerator( } private void writeValueToDataBufferGenerator( - BufferType bufferType, ArrowBuf viewBuffer, List dataBuffers, FieldVector vector) + BufferType bufferType, ArrowBuf viewBuffer, List dataBuffers) throws IOException { if (bufferType.equals(VARIADIC_DATA_BUFFERS)) { Preconditions.checkNotNull(viewBuffer); @@ -560,8 +560,8 @@ private void writeValueToGenerator( case VARCHAR: { Preconditions.checkNotNull(offsetBuffer); - byte[] b = (BaseVariableWidthVector.get(buffer, offsetBuffer, index)); - generator.writeString(new String(b, "UTF-8")); + byte[] b = BaseVariableWidthVector.get(buffer, offsetBuffer, index); + generator.writeString(new String(b, StandardCharsets.UTF_8)); break; } case DECIMAL: diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java index eeb2eaf566d6e..fca6b13cd542e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java @@ -23,8 +23,6 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.vector.ipc.message.FBSerializable; import org.apache.arrow.vector.ipc.message.MessageSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Wrapper around a WritableByteChannel that maintains the position as well adding some common @@ -37,7 +35,6 @@ *

Please note that objects of this class are not thread-safe. */ public class WriteChannel implements AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(WriteChannel.class); private static final byte[] ZERO_BYTES = new byte[8]; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java index 455229cc6dda5..ff002794ed50f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBlock.java @@ -75,7 +75,7 @@ public boolean equals(Object obj) { if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (!(obj instanceof ArrowBlock)) { return false; } ArrowBlock other = (ArrowBlock) obj; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java index cebddeb660e25..1ec9888bd798d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowBuffer.java @@ -62,7 +62,7 @@ public boolean equals(Object obj) { if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (!(obj instanceof ArrowBuffer)) { return false; } ArrowBuffer other = (ArrowBuffer) obj; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java index cee76433ea4c7..208092c9461af 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowDictionaryBatch.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector.ipc.message; +import com.google.errorprone.annotations.InlineMe; import com.google.flatbuffers.FlatBufferBuilder; import org.apache.arrow.flatbuf.DictionaryBatch; import org.apache.arrow.flatbuf.MessageHeader; @@ -31,6 +32,7 @@ public class ArrowDictionaryBatch implements ArrowMessage { private final boolean isDelta; @Deprecated + @InlineMe(replacement = "this(dictionaryId, dictionary, false)") public ArrowDictionaryBatch(long dictionaryId, ArrowRecordBatch dictionary) { this(dictionaryId, dictionary, false); } @@ -46,6 +48,7 @@ public boolean isDelta() { return isDelta; } + @Override public byte getMessageType() { return MessageHeader.DictionaryBatch; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java index bb2b87113faca..27ef82a603b2a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowFooter.java @@ -190,7 +190,7 @@ public boolean equals(Object obj) { if (obj == null) { return false; } - if (getClass() != obj.getClass()) { + if (!(obj instanceof ArrowFooter)) { return false; } ArrowFooter other = (ArrowFooter) obj; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java index bc6bfa8c868f7..73b4c5313ffe4 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java @@ -182,6 +182,7 @@ public ArrowRecordBatch( // this constructor is different from the public ones in that the reference manager's // retain method is not called, so the first dummy parameter is used // to distinguish this from the public constructor. + @SuppressWarnings("UnusedVariable") private ArrowRecordBatch( boolean dummy, int length, @@ -206,6 +207,7 @@ private ArrowRecordBatch( this.buffersLayout = Collections.unmodifiableList(arrowBuffers); } + @Override public byte getMessageType() { return org.apache.arrow.flatbuf.MessageHeader.RecordBatch; } @@ -261,9 +263,9 @@ public ArrowRecordBatch cloneWithTransfer(final BufferAllocator allocator) { buffers.stream() .map( buf -> - (buf.getReferenceManager() - .transferOwnership(buf, allocator) - .getTransferredBuffer()) + buf.getReferenceManager() + .transferOwnership(buf, allocator) + .getTransferredBuffer() .writerIndex(buf.writerIndex())) .collect(Collectors.toList()); close(); From a99a70171878c8e4e5b5f865e71fd03f91531be1 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Thu, 12 Sep 2024 08:42:45 +0530 Subject: [PATCH 06/13] fix: cleaning vector module v3 --- .../arrow/vector/ipc/JsonFileWriter.java | 8 ++--- .../vector/ipc/message/MessageSerializer.java | 29 +++++++++++++++---- .../arrow/vector/util/DecimalUtility.java | 8 ++--- .../arrow/vector/util/VectorAppender.java | 16 ++++++---- .../arrow/vector/validate/ValidateUtil.java | 3 ++ .../validate/ValidateVectorBufferVisitor.java | 7 +++-- .../validate/ValidateVectorDataVisitor.java | 3 +- .../validate/ValidateVectorVisitor.java | 9 +++--- .../org/apache/arrow/vector/TestCopyFrom.java | 2 +- .../arrow/vector/TestDecimal256Vector.java | 4 +-- .../arrow/vector/TestDecimalVector.java | 4 +-- .../arrow/vector/TestLargeListVector.java | 8 ++--- .../arrow/vector/TestLargeListViewVector.java | 4 +-- .../arrow/vector/TestListViewVector.java | 10 ++----- 14 files changed, 66 insertions(+), 49 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index a3fb2be1a57c8..e57e11e6cc6dd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -351,10 +351,7 @@ private byte[] getView(final ArrowBuf viewBuffer, final List dataBuffe } private void writeValueToViewGenerator( - ArrowBuf viewBuffer, - List dataBuffers, - FieldVector vector, - final int index) + ArrowBuf viewBuffer, List dataBuffers, FieldVector vector, final int index) throws IOException { Preconditions.checkNotNull(viewBuffer); byte[] b = getView(viewBuffer, dataBuffers, index); @@ -392,8 +389,7 @@ private void writeValueToViewGenerator( } private void writeValueToDataBufferGenerator( - BufferType bufferType, ArrowBuf viewBuffer, List dataBuffers) - throws IOException { + BufferType bufferType, ArrowBuf viewBuffer, List dataBuffers) throws IOException { if (bufferType.equals(VARIADIC_DATA_BUFFERS)) { Preconditions.checkNotNull(viewBuffer); Preconditions.checkArgument(!dataBuffers.isEmpty()); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java index 36f6ea449b03c..8412af326e58c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java @@ -18,6 +18,7 @@ import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; +import com.google.errorprone.annotations.InlineMe; import com.google.flatbuffers.FlatBufferBuilder; import java.io.IOException; import java.nio.ByteBuffer; @@ -63,7 +64,7 @@ public static int bytesToInt(byte[] bytes) { return ((bytes[3] & 255) << 24) + ((bytes[2] & 255) << 16) + ((bytes[1] & 255) << 8) - + ((bytes[0] & 255)); + + (bytes[0] & 255); } /** @@ -76,7 +77,7 @@ public static void intToBytes(int value, byte[] bytes) { bytes[3] = (byte) (value >>> 24); bytes[2] = (byte) (value >>> 16); bytes[1] = (byte) (value >>> 8); - bytes[0] = (byte) (value); + bytes[0] = (byte) value; } /** @@ -93,7 +94,7 @@ public static void longToBytes(long value, byte[] bytes) { bytes[3] = (byte) (value >>> 24); bytes[2] = (byte) (value >>> 16); bytes[1] = (byte) (value >>> 8); - bytes[0] = (byte) (value); + bytes[0] = (byte) value; } public static int writeMessageBuffer( @@ -166,6 +167,12 @@ public static long serialize(WriteChannel out, Schema schema, IpcOption option) /** Returns the serialized flatbuffer bytes of the schema wrapped in a message table. */ @Deprecated + @InlineMe( + replacement = "MessageSerializer.serializeMetadata(schema, IpcOption.DEFAULT)", + imports = { + "org.apache.arrow.vector.ipc.message.IpcOption", + "org.apache.arrow.vector.ipc.message.MessageSerializer" + }) public static ByteBuffer serializeMetadata(Schema schema) { return serializeMetadata(schema, IpcOption.DEFAULT); } @@ -314,6 +321,12 @@ public static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) * org.apache.arrow.flatbuf.Message}. */ @Deprecated + @InlineMe( + replacement = "MessageSerializer.serializeMetadata(message, IpcOption.DEFAULT)", + imports = { + "org.apache.arrow.vector.ipc.message.IpcOption", + "org.apache.arrow.vector.ipc.message.MessageSerializer" + }) public static ByteBuffer serializeMetadata(ArrowMessage message) { return serializeMetadata(message, IpcOption.DEFAULT); } @@ -655,6 +668,13 @@ public static ArrowMessage deserializeMessageBatch(ReadChannel in, BufferAllocat } @Deprecated + @InlineMe( + replacement = + "MessageSerializer.serializeMessage(builder, headerType, headerOffset, bodyLength, IpcOption.DEFAULT)", + imports = { + "org.apache.arrow.vector.ipc.message.IpcOption", + "org.apache.arrow.vector.ipc.message.MessageSerializer" + }) public static ByteBuffer serializeMessage( FlatBufferBuilder builder, byte headerType, int headerOffset, long bodyLength) { return serializeMessage(builder, headerType, headerOffset, bodyLength, IpcOption.DEFAULT); @@ -719,9 +739,6 @@ public static MessageMetadataResult readMessage(ReadChannel in) throws IOExcepti if (in.readFully(messageBuffer) != messageLength) { throw new IOException("Unexpected end of stream trying to read message."); } - // see https://github.com/apache/arrow/issues/41717 for reason why we cast to - // java.nio.Buffer - ByteBuffer rewindBuffer = (ByteBuffer) ((java.nio.Buffer) messageBuffer).rewind(); // Load the message. Message message = Message.getRootAsMessage(messageBuffer); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java index 31b79fe53a4a5..0bd29526fd0e6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java @@ -27,12 +27,12 @@ public class DecimalUtility { private DecimalUtility() {} - public static final byte[] zeroes = + private static final byte[] zeroes = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - public static final byte[] minus_one = + private static final byte[] minus_one = new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 @@ -172,11 +172,11 @@ public static void writeLongToArrowBuf(long value, ArrowBuf bytebuf, int index, if (LITTLE_ENDIAN) { MemoryUtil.putLong(addressOfValue, value); for (int i = 1; i <= (byteWidth - 8) / 8; i++) { - MemoryUtil.putLong(addressOfValue + Long.BYTES * i, padValue); + MemoryUtil.putLong(addressOfValue + Long.BYTES * ((long) i), padValue); } } else { for (int i = 0; i < (byteWidth - 8) / 8; i++) { - MemoryUtil.putLong(addressOfValue + Long.BYTES * i, padValue); + MemoryUtil.putLong(addressOfValue + Long.BYTES * ((long) i), padValue); } MemoryUtil.putLong(addressOfValue + Long.BYTES * (byteWidth - 8) / 8, value); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java index e703571b374eb..9dcf95d8a8147 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java @@ -96,8 +96,8 @@ public ValueVector visit(BaseFixedWidthVector deltaVector, Void value) { MemoryUtil.copyMemory( deltaVector.getDataBuffer().memoryAddress(), targetVector.getDataBuffer().memoryAddress() - + deltaVector.getTypeWidth() * targetVector.getValueCount(), - deltaVector.getTypeWidth() * deltaVector.getValueCount()); + + (long) deltaVector.getTypeWidth() * targetVector.getValueCount(), + (long) deltaVector.getTypeWidth() * deltaVector.getValueCount()); } targetVector.setValueCount(newValueCount); return targetVector; @@ -151,8 +151,8 @@ public ValueVector visit(BaseVariableWidthVector deltaVector, Void value) { MemoryUtil.copyMemory( deltaVector.getOffsetBuffer().memoryAddress() + BaseVariableWidthVector.OFFSET_WIDTH, targetVector.getOffsetBuffer().memoryAddress() - + (targetVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, - deltaVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); + + (long) (targetVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, + (long) deltaVector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); // increase each offset from the second buffer for (int i = 0; i < deltaVector.getValueCount(); i++) { @@ -223,8 +223,8 @@ public ValueVector visit(BaseLargeVariableWidthVector deltaVector, Void value) { MemoryUtil.copyMemory( deltaVector.getOffsetBuffer().memoryAddress() + BaseLargeVariableWidthVector.OFFSET_WIDTH, targetVector.getOffsetBuffer().memoryAddress() - + (targetVector.getValueCount() + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH, - deltaVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH); + + (long) (targetVector.getValueCount() + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH, + (long) deltaVector.getValueCount() * BaseLargeVariableWidthVector.OFFSET_WIDTH); // increase each offset from the second buffer for (int i = 0; i < deltaVector.getValueCount(); i++) { @@ -252,6 +252,7 @@ public ValueVector visit(BaseVariableWidthViewVector left, Void value) { } @Override + @SuppressWarnings("EqualsIncompatibleType") public ValueVector visit(ListVector deltaVector, Void value) { Preconditions.checkArgument( typeVisitor.equals(deltaVector), @@ -321,6 +322,7 @@ public ValueVector visit(ListVector deltaVector, Void value) { } @Override + @SuppressWarnings("EqualsIncompatibleType") public ValueVector visit(LargeListVector deltaVector, Void value) { Preconditions.checkArgument( typeVisitor.equals(deltaVector), @@ -392,6 +394,7 @@ public ValueVector visit(LargeListVector deltaVector, Void value) { } @Override + @SuppressWarnings("EqualsIncompatibleType") public ValueVector visit(FixedSizeListVector deltaVector, Void value) { Preconditions.checkArgument( typeVisitor.equals(deltaVector), @@ -438,6 +441,7 @@ public ValueVector visit(FixedSizeListVector deltaVector, Void value) { } @Override + @SuppressWarnings("EqualsIncompatibleType") public ValueVector visit(NonNullableStructVector deltaVector, Void value) { Preconditions.checkArgument( typeVisitor.equals(deltaVector), diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateUtil.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateUtil.java index 015ee265d1423..b6490a84ab42d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateUtil.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateUtil.java @@ -16,6 +16,8 @@ */ package org.apache.arrow.vector.validate; +import com.google.errorprone.annotations.FormatMethod; + /** Utilities for vector validation. */ public class ValidateUtil { @@ -42,6 +44,7 @@ public static void validateOrThrow(boolean expression, String errorMessage) { * @param args the error message arguments. * @throws ValidateException if the expression evaluates to false. */ + @FormatMethod public static void validateOrThrow(boolean expression, String errorMessage, Object... args) { if (!expression) { throw new ValidateException(String.format(errorMessage, args)); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java index 0c9140c360d15..c5cb8f339b5a9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java @@ -131,7 +131,7 @@ public Void visit(BaseVariableWidthVector vector, Void value) { int lastOffset = valueCount == 0 ? 0 - : vector.getOffsetBuffer().getInt(valueCount * BaseVariableWidthVector.OFFSET_WIDTH); + : vector.getOffsetBuffer().getInt((long) valueCount * BaseVariableWidthVector.OFFSET_WIDTH); validateDataBuffer(vector, lastOffset); return null; } @@ -172,7 +172,7 @@ public Void visit(ListVector vector, Void value) { int lastOffset = valueCount == 0 ? 0 - : vector.getOffsetBuffer().getInt(valueCount * BaseVariableWidthVector.OFFSET_WIDTH); + : vector.getOffsetBuffer().getInt((long) valueCount * BaseVariableWidthVector.OFFSET_WIDTH); int dataVectorLength = dataVector == null ? 0 : dataVector.getValueCount(); validateOrThrow( dataVectorLength >= lastOffset, @@ -219,7 +219,7 @@ public Void visit(LargeListVector vector, Void value) { ? 0 : vector .getOffsetBuffer() - .getLong(valueCount * BaseLargeVariableWidthVector.OFFSET_WIDTH); + .getLong((long) valueCount * BaseLargeVariableWidthVector.OFFSET_WIDTH); int dataVectorLength = dataVector == null ? 0 : dataVector.getValueCount(); validateOrThrow( dataVectorLength >= lastOffset, @@ -283,6 +283,7 @@ public Void visit(NullVector vector, Void value) { } @Override + @SuppressWarnings("VoidUsed") public Void visit(ExtensionTypeVector vector, Void value) { vector.getUnderlyingVector().accept(this, value); return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java index c62bff79f7710..2cf0871fcc818 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java @@ -46,7 +46,7 @@ private void validateOffsetBuffer(ValueVector vector, int valueCount) { // verify that the values in the offset buffer is non-decreasing int prevValue = offsetBuffer.getInt(0); for (int i = 1; i <= valueCount; i++) { - int curValue = offsetBuffer.getInt(i * 4); + int curValue = offsetBuffer.getInt(i * 4L); validateOrThrow( curValue >= 0, "The value at position %s of the offset buffer is negative: %s.", @@ -202,6 +202,7 @@ public Void visit(NullVector vector, Void value) { } @Override + @SuppressWarnings("VoidUsed") public Void visit(ExtensionTypeVector vector, Void value) { vector.getUnderlyingVector().accept(this, value); return null; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java index 5004ba488cacd..dce21910f3e56 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java @@ -79,7 +79,7 @@ public Void visit(BaseVariableWidthVector vector, Void value) { int lastOffset = vector .getOffsetBuffer() - .getInt(vector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); + .getInt((long) vector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); if (firstOffset < 0 || lastOffset < 0) { throw new IllegalArgumentException("Negative offsets in vector"); @@ -136,7 +136,7 @@ public Void visit(ListVector vector, Void value) { int lastOffset = vector .getOffsetBuffer() - .getInt(vector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); + .getInt((long) vector.getValueCount() * BaseVariableWidthVector.OFFSET_WIDTH); if (firstOffset < 0 || lastOffset < 0) { throw new IllegalArgumentException("Negative offsets in list vector"); @@ -169,7 +169,7 @@ public Void visit(LargeListVector vector, Void value) { if (vector.getValueCount() > 0) { ArrowBuf offsetBuf = vector.getOffsetBuffer(); - long minBufferSize = (vector.getValueCount() + 1) * LargeListVector.OFFSET_WIDTH; + long minBufferSize = (long) (vector.getValueCount() + 1) * LargeListVector.OFFSET_WIDTH; if (offsetBuf.capacity() < minBufferSize) { throw new IllegalArgumentException( @@ -184,7 +184,7 @@ public Void visit(LargeListVector vector, Void value) { long firstOffset = vector.getOffsetBuffer().getLong(0); long lastOffset = - vector.getOffsetBuffer().getLong(vector.getValueCount() * LargeListVector.OFFSET_WIDTH); + vector.getOffsetBuffer().getLong((long) vector.getValueCount() * LargeListVector.OFFSET_WIDTH); if (firstOffset < 0 || lastOffset < 0) { throw new IllegalArgumentException("Negative offsets in list vector"); @@ -314,6 +314,7 @@ public Void visit(NullVector vector, Void value) { } @Override + @SuppressWarnings("VoidUsed") public Void visit(ExtensionTypeVector vector, Void value) { vector.getUnderlyingVector().accept(this, value); return null; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java index bd0b44a14452a..abcc932302146 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java @@ -180,7 +180,7 @@ public void testCopyFromWithNulls1() { /* set lesser initial capacity than actually needed * to trigger reallocs in copyFromSafe() */ - vector2.allocateNew((initialCapacity / 4) * 10, initialCapacity / 4); + vector2.allocateNew((initialCapacity / 4) * 10L, initialCapacity / 4); capacity = vector2.getValueCapacity(); assertTrue(capacity >= initialCapacity / 4); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java index c155ab98fa2e9..dbbff1aae6ade 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java @@ -380,8 +380,8 @@ private void verifyWritingArrowBufWithBigEndianBytes( decimalVector.allocateNew(); for (int i = 0; i < expectedValues.length; i++) { byte[] bigEndianBytes = expectedValues[i].unscaledValue().toByteArray(); - buf.setBytes(length * i, bigEndianBytes, 0, bigEndianBytes.length); - decimalVector.setBigEndianSafe(i, length * i, buf, bigEndianBytes.length); + buf.setBytes((long) length * i, bigEndianBytes, 0, bigEndianBytes.length); + decimalVector.setBigEndianSafe(i, (long) length * i, buf, bigEndianBytes.length); } decimalVector.setValueCount(3); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index d5310bad0ea0e..8f839a693ebe7 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -376,8 +376,8 @@ private void verifyWritingArrowBufWithBigEndianBytes( decimalVector.allocateNew(); for (int i = 0; i < expectedValues.length; i++) { byte[] bigEndianBytes = expectedValues[i].unscaledValue().toByteArray(); - buf.setBytes(length * i, bigEndianBytes, 0, bigEndianBytes.length); - decimalVector.setBigEndianSafe(i, length * i, buf, bigEndianBytes.length); + buf.setBytes((long) length * i, bigEndianBytes, 0, bigEndianBytes.length); + decimalVector.setBigEndianSafe(i, (long) length * i, buf, bigEndianBytes.length); } decimalVector.setValueCount(3); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index 101d942d2a4c5..0f45d80b7488f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -773,7 +773,7 @@ public void testSetInitialCapacity() { try (final LargeListVector vector = LargeListVector.empty("", allocator)) { vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); - /** + /* * use the default multiplier of 5, 512 * 5 => 2560 * 4 => 10240 bytes => 16KB => 4096 value * capacity. */ @@ -788,7 +788,7 @@ public void testSetInitialCapacity() { assertEquals(512, vector.getValueCapacity()); assertTrue(vector.getDataVector().getValueCapacity() >= 512 * 4); - /** + /* * inner value capacity we pass to data vector is 512 * 0.1 => 51 For an int vector this is * 204 bytes of memory for data buffer and 7 bytes for validity buffer. and with power of 2 * allocation, we allocate 256 bytes and 8 bytes for the data buffer and validity buffer of @@ -799,7 +799,7 @@ public void testSetInitialCapacity() { assertEquals(512, vector.getValueCapacity()); assertTrue(vector.getDataVector().getValueCapacity() >= 51); - /** + /* * inner value capacity we pass to data vector is 512 * 0.01 => 5 For an int vector this is 20 * bytes of memory for data buffer and 1 byte for validity buffer. and with power of 2 * allocation, we allocate 32 bytes and 1 bytes for the data buffer and validity buffer of the @@ -810,7 +810,7 @@ public void testSetInitialCapacity() { assertEquals(512, vector.getValueCapacity()); assertTrue(vector.getDataVector().getValueCapacity() >= 5); - /** + /* * inner value capacity we pass to data vector is 5 * 0.1 => 0 which is then rounded off to 1. * So we pass value count as 1 to the inner int vector. the offset buffer of the list vector * is allocated for 6 values which is 24 bytes and then rounded off to 32 bytes (8 values) the diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java index 26e7bb4a0d3b2..44f6b56d3789f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListViewVector.java @@ -1653,7 +1653,7 @@ private int validateSizeBufferAndCalculateMinOffset( for (int i = 0; i < splitLength; i++) { fromDataLength = fromSizeBuffer.getInt((long) (start + i) * LargeListViewVector.SIZE_WIDTH); - toDataLength = toSizeBuffer.getInt((long) (i) * LargeListViewVector.SIZE_WIDTH); + toDataLength = toSizeBuffer.getInt((long) i * LargeListViewVector.SIZE_WIDTH); /* validate size */ assertEquals( @@ -1683,7 +1683,7 @@ private void validateOffsetBuffer( for (int i = 0; i < splitLength; i++) { offset1 = fromOffsetBuffer.getInt((long) (start + i) * LargeListViewVector.OFFSET_WIDTH); - offset2 = toOffsetBuffer.getInt((long) (i) * LargeListViewVector.OFFSET_WIDTH); + offset2 = toOffsetBuffer.getInt((long) i * LargeListViewVector.OFFSET_WIDTH); assertEquals( offset1 - minOffset, offset2, diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java index 639585fc48d0a..490090177c0d0 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java @@ -384,12 +384,6 @@ public void testNestedListVector() throws Exception { } } - private void setValuesInBuffer(int[] bufValues, ArrowBuf buffer, long bufWidth) { - for (int i = 0; i < bufValues.length; i++) { - buffer.setInt(i * bufWidth, bufValues[i]); - } - } - /* * Setting up the buffers directly needs to be validated with the base method used in * the ListVector class where we use the approach of startListView(), @@ -1667,7 +1661,7 @@ private int validateSizeBufferAndCalculateMinOffset( for (int i = 0; i < splitLength; i++) { fromDataLength = fromSizeBuffer.getInt((long) (start + i) * ListViewVector.SIZE_WIDTH); - toDataLength = toSizeBuffer.getInt((long) (i) * ListViewVector.SIZE_WIDTH); + toDataLength = toSizeBuffer.getInt((long) i * ListViewVector.SIZE_WIDTH); /* validate size */ assertEquals( @@ -1696,7 +1690,7 @@ private void validateOffsetBuffer( for (int i = 0; i < splitLength; i++) { offset1 = fromOffsetBuffer.getInt((long) (start + i) * ListViewVector.OFFSET_WIDTH); - offset2 = toOffsetBuffer.getInt((long) (i) * ListViewVector.OFFSET_WIDTH); + offset2 = toOffsetBuffer.getInt((long) i * ListViewVector.OFFSET_WIDTH); assertEquals( offset1 - minOffset, offset2, From f8bc62359be4ec47112ac8c400a0e03ad537d6e7 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Thu, 12 Sep 2024 09:33:52 +0530 Subject: [PATCH 07/13] fix: cleaning vector module v4 --- .../arrow/vector/compare/TypeEqualsVisitor.java | 11 ++--------- .../apache/arrow/vector/util/VectorAppender.java | 4 ---- .../apache/arrow/vector/TestPeriodDuration.java | 15 +++++++-------- .../org/apache/arrow/vector/TestTypeLayout.java | 13 +++++++------ .../org/apache/arrow/vector/TestUnionVector.java | 1 + .../org/apache/arrow/vector/TestValueVector.java | 10 +++++----- .../arrow/vector/TestValueVectorIterable.java | 2 +- .../arrow/vector/TestVarCharViewVector.java | 10 +++++----- .../apache/arrow/vector/TestVectorUnloadLoad.java | 8 ++++---- 9 files changed, 32 insertions(+), 42 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java index ce9cb117ed81f..ae49e5380bea3 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java @@ -62,15 +62,8 @@ public TypeEqualsVisitor(ValueVector right, boolean checkName, boolean checkMeta } /** Check type equals without passing IN param in VectorVisitor. */ - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof ValueVector)) { - return false; - } - ValueVector left = (ValueVector) obj; + @SuppressWarnings("NonOverridingEquals") + public boolean equals(ValueVector left) { return left.accept(this, null); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java index 9dcf95d8a8147..f97310ac736cf 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java @@ -252,7 +252,6 @@ public ValueVector visit(BaseVariableWidthViewVector left, Void value) { } @Override - @SuppressWarnings("EqualsIncompatibleType") public ValueVector visit(ListVector deltaVector, Void value) { Preconditions.checkArgument( typeVisitor.equals(deltaVector), @@ -322,7 +321,6 @@ public ValueVector visit(ListVector deltaVector, Void value) { } @Override - @SuppressWarnings("EqualsIncompatibleType") public ValueVector visit(LargeListVector deltaVector, Void value) { Preconditions.checkArgument( typeVisitor.equals(deltaVector), @@ -394,7 +392,6 @@ public ValueVector visit(LargeListVector deltaVector, Void value) { } @Override - @SuppressWarnings("EqualsIncompatibleType") public ValueVector visit(FixedSizeListVector deltaVector, Void value) { Preconditions.checkArgument( typeVisitor.equals(deltaVector), @@ -441,7 +438,6 @@ public ValueVector visit(FixedSizeListVector deltaVector, Void value) { } @Override - @SuppressWarnings("EqualsIncompatibleType") public ValueVector visit(NonNullableStructVector deltaVector, Void value) { Preconditions.checkArgument( typeVisitor.equals(deltaVector), diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java b/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java index 37f59f82cbd2f..6b1d8324f9200 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java @@ -54,19 +54,19 @@ public void testToISO8601IntervalString() { new PeriodDuration(Period.ZERO, Duration.ofNanos(123)).toISO8601IntervalString()); assertEquals( "PT1.000000123S", - new PeriodDuration(Period.ZERO, Duration.ofSeconds(1).withNanos(123)) + new PeriodDuration(Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(1).getSeconds(), 123)) .toISO8601IntervalString()); assertEquals( "PT1H1.000000123S", - new PeriodDuration(Period.ZERO, Duration.ofSeconds(3601).withNanos(123)) + new PeriodDuration(Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(3601).getSeconds(), 123)) .toISO8601IntervalString()); assertEquals( "PT24H1M1.000000123S", - new PeriodDuration(Period.ZERO, Duration.ofSeconds(86461).withNanos(123)) + new PeriodDuration(Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(86461).getSeconds(), 123)) .toISO8601IntervalString()); assertEquals( "P1Y2M3DT24H1M1.000000123S", - new PeriodDuration(Period.of(1, 2, 3), Duration.ofSeconds(86461).withNanos(123)) + new PeriodDuration(Period.of(1, 2, 3), Duration.ofSeconds(Duration.ofSeconds(86461).getSeconds(), 123)) .toISO8601IntervalString()); assertEquals( @@ -77,11 +77,11 @@ public void testToISO8601IntervalString() { new PeriodDuration(Period.ZERO, Duration.ofNanos(-123)).toISO8601IntervalString()); assertEquals( "PT-24H-1M-0.999999877S", - new PeriodDuration(Period.ZERO, Duration.ofSeconds(-86461).withNanos(123)) + new PeriodDuration(Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(-86461).getSeconds(), 123)) .toISO8601IntervalString()); assertEquals( "P-1Y-2M-3DT-0.999999877S", - new PeriodDuration(Period.of(-1, -2, -3), Duration.ofSeconds(-1).withNanos(123)) + new PeriodDuration(Period.of(-1, -2, -3), Duration.ofSeconds(Duration.ofSeconds(-1).getSeconds(), 123)) .toISO8601IntervalString()); } @@ -95,8 +95,7 @@ public void testTemporalAccessor() { PeriodDuration pd2 = new PeriodDuration(Period.ZERO, Duration.ofMinutes(1)); assertEquals(LocalDateTime.of(2024, 1, 2, 3, 3), pd2.subtractFrom(dateTime)); - PeriodDuration pd3 = - new PeriodDuration(Period.of(1, 2, 3), Duration.ofSeconds(86461).withNanos(123)); + PeriodDuration pd3 = new PeriodDuration(Period.of(1, 2, 3), Duration.ofSeconds(Duration.ofSeconds(86461).getSeconds(), 123)); assertEquals(pd3.get(ChronoUnit.YEARS), 1); assertEquals(pd3.get(ChronoUnit.MONTHS), 2); assertEquals(pd3.get(ChronoUnit.DAYS), 3); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java b/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java index 75c6df1d6a496..d56487d8763b2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import java.nio.charset.StandardCharsets; import java.util.Random; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -175,12 +176,12 @@ public void testTypeBufferCountInVectorsWithVariadicBuffers() { try (ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { viewVarCharVector.allocateNew(32, 6); - viewVarCharVector.setSafe(0, generateRandomString(8).getBytes()); - viewVarCharVector.setSafe(1, generateRandomString(12).getBytes()); - viewVarCharVector.setSafe(2, generateRandomString(14).getBytes()); - viewVarCharVector.setSafe(3, generateRandomString(18).getBytes()); - viewVarCharVector.setSafe(4, generateRandomString(22).getBytes()); - viewVarCharVector.setSafe(5, generateRandomString(24).getBytes()); + viewVarCharVector.setSafe(0, generateRandomString(8).getBytes(StandardCharsets.UTF_8)); + viewVarCharVector.setSafe(1, generateRandomString(12).getBytes(StandardCharsets.UTF_8)); + viewVarCharVector.setSafe(2, generateRandomString(14).getBytes(StandardCharsets.UTF_8)); + viewVarCharVector.setSafe(3, generateRandomString(18).getBytes(StandardCharsets.UTF_8)); + viewVarCharVector.setSafe(4, generateRandomString(22).getBytes(StandardCharsets.UTF_8)); + viewVarCharVector.setSafe(5, generateRandomString(24).getBytes(StandardCharsets.UTF_8)); viewVarCharVector.setValueCount(6); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index 6c05073c16844..972c2c62cba94 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -381,6 +381,7 @@ public void testSplitAndTransferWithMixedVectors() throws Exception { } @Test + @SuppressWarnings("EnumOrdinal") public void testGetFieldTypeInfo() throws Exception { Map metadata = new HashMap<>(); metadata.put("key1", "value1"); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 83e470ae2581d..9220e4c6c911c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -1908,7 +1908,7 @@ public void testSetSafeWithArrowBufNoExcessAllocs() { } fromVector.setValueCount(numValues); ArrowBuf fromDataBuffer = fromVector.getDataBuffer(); - assertTrue(numValues * valueBytesLength <= fromDataBuffer.capacity()); + assertTrue((long) numValues * valueBytesLength <= fromDataBuffer.capacity()); /* * Copy the entries one-by-one from 'fromVector' to 'toVector', but use the setSafe with @@ -2398,11 +2398,11 @@ public void testMultipleClose() { */ public static void setBytes(int index, byte[] bytes, VarCharVector vector) { final int currentOffset = - vector.offsetBuffer.getInt(index * BaseVariableWidthVector.OFFSET_WIDTH); + vector.offsetBuffer.getInt((long) index * BaseVariableWidthVector.OFFSET_WIDTH); BitVectorHelper.setBit(vector.validityBuffer, index); vector.offsetBuffer.setInt( - (index + 1) * BaseVariableWidthVector.OFFSET_WIDTH, currentOffset + bytes.length); + (long) (index + 1) * BaseVariableWidthVector.OFFSET_WIDTH, currentOffset + bytes.length); vector.valueBuffer.setBytes(currentOffset, bytes, 0, bytes.length); } @@ -2669,8 +2669,8 @@ public void testGetPointerVariableWidth() { try (VarCharVector vec1 = new VarCharVector("vec1", allocator); VarCharVector vec2 = new VarCharVector("vec2", allocator)) { - vec1.allocateNew(sampleData.length * 10, sampleData.length); - vec2.allocateNew(sampleData.length * 10, sampleData.length); + vec1.allocateNew(sampleData.length * 10L, sampleData.length); + vec2.allocateNew(sampleData.length * 10L, sampleData.length); for (int i = 0; i < sampleData.length; i++) { String str = sampleData[i]; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVectorIterable.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVectorIterable.java index e6b79e89b9942..3d75b56640b71 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVectorIterable.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVectorIterable.java @@ -172,7 +172,7 @@ public void testDurationVectorIterable() { durationVector.setSafe(2, 555); durationVector.setValueCount(3); - final Duration value1 = Duration.ofMillis(30000); + final Duration value1 = Duration.ofSeconds(30); final Duration value3 = Duration.ofMillis(555); assertThat( durationVector.getValueIterable(), diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java index 308431fdeb9cf..d418fa92ca9e9 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java @@ -544,7 +544,7 @@ public void testSetSafeWithArrowBufNoExcessAllocs() { } fromVector.setValueCount(numValues); ArrowBuf fromDataBuffer = fromVector.getDataBuffer(); - assertTrue(numValues * valueBytesLength <= fromDataBuffer.capacity()); + assertTrue((long) numValues * valueBytesLength <= fromDataBuffer.capacity()); /* * Copy the entries one-by-one from 'fromVector' to 'toVector', but use the setSafe with @@ -1668,20 +1668,20 @@ static Stream vectorCreatorProvider() { return Stream.of( Arguments.of( (Function) - (allocator -> + allocator -> newVector( ViewVarBinaryVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VIEWVARBINARY, - allocator))), + allocator)), Arguments.of( (Function) - (allocator -> + allocator -> newVector( ViewVarCharVector.class, EMPTY_SCHEMA_PATH, Types.MinorType.VIEWVARCHAR, - allocator)))); + allocator))); } @ParameterizedTest diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index 6121fb67fefb7..4638c9fb7a974 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -216,7 +216,7 @@ public void testLoadValidityBuffer() throws IOException { ArrowBuf[] values = new ArrowBuf[4]; for (int i = 0; i < 4; i += 2) { ArrowBuf buf1 = allocator.buffer(BitVectorHelper.getValidityBufferSize(count)); - ArrowBuf buf2 = allocator.buffer(count * 4); // integers + ArrowBuf buf2 = allocator.buffer(count * 4L); // integers buf1.setZero(0, buf1.capacity()); buf2.setZero(0, buf2.capacity()); values[i] = buf1; @@ -228,10 +228,10 @@ public void testLoadValidityBuffer() throws IOException { BitVectorHelper.setBit(buf1, j); } - buf2.setInt(j * 4, j); + buf2.setInt(j * 4L, j); } - buf1.writerIndex((int) Math.ceil(count / 8)); - buf2.writerIndex(count * 4); + buf1.writerIndex((int) Math.ceil(count / 8.0)); + buf2.writerIndex(count * 4L); } /* From ab93ca800ffd003d44498609dd1bc5053d06838c Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Thu, 12 Sep 2024 18:15:06 +0530 Subject: [PATCH 08/13] fix: cleaning vector module v5 --- java/memory/pom.xml | 17 +++++++++++++ .../vector/ipc/message/MessageSerializer.java | 5 ++++ .../validate/ValidateVectorBufferVisitor.java | 8 +++++-- .../validate/ValidateVectorVisitor.java | 4 +++- .../arrow/vector/TestPeriodDuration.java | 19 ++++++++++----- .../complex/impl/TestComplexCopier.java | 24 +++++++++---------- .../apache/arrow/vector/ipc/BaseFileTest.java | 4 ++-- .../vector/ipc/TestArrowReaderWriter.java | 4 ++-- .../apache/arrow/vector/pojo/TestConvert.java | 6 ++++- .../apache/arrow/vector/table/RowTest.java | 1 + .../testing/TestValueVectorPopulator.java | 14 +++++------ .../vector/types/pojo/TestExtensionType.java | 1 + .../arrow/vector/util/TestValidator.java | 1 + 13 files changed, 75 insertions(+), 33 deletions(-) diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 0ca357beaa781..15761ae50b26c 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -34,4 +34,21 @@ under the License. memory-netty-buffer-patch memory-netty + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + 11 + + -XDcompilePolicy=simple + + + + + diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java index 8412af326e58c..6c6a5893ccc9c 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java @@ -715,6 +715,7 @@ public static ByteBuffer serializeMessage( * valid Message was read, or null if end-of-stream * @throws IOException on error */ + @SuppressWarnings("UnusedVariable") public static MessageMetadataResult readMessage(ReadChannel in) throws IOException { // Read the message size. There is an i32 little endian prefix. @@ -740,6 +741,10 @@ public static MessageMetadataResult readMessage(ReadChannel in) throws IOExcepti throw new IOException("Unexpected end of stream trying to read message."); } + // see https://github.com/apache/arrow/issues/41717 for reason why we cast to + // java.nio.Buffer + ByteBuffer rewindBuffer = (ByteBuffer) ((java.nio.Buffer) messageBuffer).rewind(); + // Load the message. Message message = Message.getRootAsMessage(messageBuffer); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java index c5cb8f339b5a9..e46f5f9b981e9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java @@ -131,7 +131,9 @@ public Void visit(BaseVariableWidthVector vector, Void value) { int lastOffset = valueCount == 0 ? 0 - : vector.getOffsetBuffer().getInt((long) valueCount * BaseVariableWidthVector.OFFSET_WIDTH); + : vector + .getOffsetBuffer() + .getInt((long) valueCount * BaseVariableWidthVector.OFFSET_WIDTH); validateDataBuffer(vector, lastOffset); return null; } @@ -172,7 +174,9 @@ public Void visit(ListVector vector, Void value) { int lastOffset = valueCount == 0 ? 0 - : vector.getOffsetBuffer().getInt((long) valueCount * BaseVariableWidthVector.OFFSET_WIDTH); + : vector + .getOffsetBuffer() + .getInt((long) valueCount * BaseVariableWidthVector.OFFSET_WIDTH); int dataVectorLength = dataVector == null ? 0 : dataVector.getValueCount(); validateOrThrow( dataVectorLength >= lastOffset, diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java index dce21910f3e56..71f49672bd2fe 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java @@ -184,7 +184,9 @@ public Void visit(LargeListVector vector, Void value) { long firstOffset = vector.getOffsetBuffer().getLong(0); long lastOffset = - vector.getOffsetBuffer().getLong((long) vector.getValueCount() * LargeListVector.OFFSET_WIDTH); + vector + .getOffsetBuffer() + .getLong((long) vector.getValueCount() * LargeListVector.OFFSET_WIDTH); if (firstOffset < 0 || lastOffset < 0) { throw new IllegalArgumentException("Negative offsets in list vector"); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java b/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java index 6b1d8324f9200..6864102713850 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java @@ -58,15 +58,18 @@ public void testToISO8601IntervalString() { .toISO8601IntervalString()); assertEquals( "PT1H1.000000123S", - new PeriodDuration(Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(3601).getSeconds(), 123)) + new PeriodDuration( + Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(3601).getSeconds(), 123)) .toISO8601IntervalString()); assertEquals( "PT24H1M1.000000123S", - new PeriodDuration(Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(86461).getSeconds(), 123)) + new PeriodDuration( + Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(86461).getSeconds(), 123)) .toISO8601IntervalString()); assertEquals( "P1Y2M3DT24H1M1.000000123S", - new PeriodDuration(Period.of(1, 2, 3), Duration.ofSeconds(Duration.ofSeconds(86461).getSeconds(), 123)) + new PeriodDuration( + Period.of(1, 2, 3), Duration.ofSeconds(Duration.ofSeconds(86461).getSeconds(), 123)) .toISO8601IntervalString()); assertEquals( @@ -77,11 +80,13 @@ public void testToISO8601IntervalString() { new PeriodDuration(Period.ZERO, Duration.ofNanos(-123)).toISO8601IntervalString()); assertEquals( "PT-24H-1M-0.999999877S", - new PeriodDuration(Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(-86461).getSeconds(), 123)) + new PeriodDuration( + Period.ZERO, Duration.ofSeconds(Duration.ofSeconds(-86461).getSeconds(), 123)) .toISO8601IntervalString()); assertEquals( "P-1Y-2M-3DT-0.999999877S", - new PeriodDuration(Period.of(-1, -2, -3), Duration.ofSeconds(Duration.ofSeconds(-1).getSeconds(), 123)) + new PeriodDuration( + Period.of(-1, -2, -3), Duration.ofSeconds(Duration.ofSeconds(-1).getSeconds(), 123)) .toISO8601IntervalString()); } @@ -95,7 +100,9 @@ public void testTemporalAccessor() { PeriodDuration pd2 = new PeriodDuration(Period.ZERO, Duration.ofMinutes(1)); assertEquals(LocalDateTime.of(2024, 1, 2, 3, 3), pd2.subtractFrom(dateTime)); - PeriodDuration pd3 = new PeriodDuration(Period.of(1, 2, 3), Duration.ofSeconds(Duration.ofSeconds(86461).getSeconds(), 123)); + PeriodDuration pd3 = + new PeriodDuration( + Period.of(1, 2, 3), Duration.ofSeconds(Duration.ofSeconds(86461).getSeconds(), 123)); assertEquals(pd3.get(ChronoUnit.YEARS), 1); assertEquals(pd3.get(ChronoUnit.MONTHS), 2); assertEquals(pd3.get(ChronoUnit.DAYS), 3); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 3bc02c602983b..2c4b06952c6c9 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -136,8 +136,8 @@ public void testCopyMapVector() { mapWriter.value().integer().writeInt(i); mapWriter.endEntry(); mapWriter.startEntry(); - mapWriter.key().decimal().writeDecimal(BigDecimal.valueOf(i * 2)); - mapWriter.value().decimal().writeDecimal(BigDecimal.valueOf(i * 2)); + mapWriter.key().decimal().writeDecimal(BigDecimal.valueOf(i * 2L)); + mapWriter.value().decimal().writeDecimal(BigDecimal.valueOf(i * 2L)); mapWriter.endEntry(); mapWriter.endMap(); } @@ -176,13 +176,13 @@ public void testCopyListVector() { listWriter.list().startList(); listWriter.list().bigInt().writeBigInt(i); - listWriter.list().bigInt().writeBigInt(i * 2); - listWriter.list().bigInt().writeBigInt(i * 3); + listWriter.list().bigInt().writeBigInt(i * 2L); + listWriter.list().bigInt().writeBigInt(i * 3L); listWriter.list().endList(); listWriter.list().startList(); - listWriter.list().decimal().writeDecimal(BigDecimal.valueOf(i * 4)); - listWriter.list().decimal().writeDecimal(BigDecimal.valueOf(i * 5)); + listWriter.list().decimal().writeDecimal(BigDecimal.valueOf(i * 4L)); + listWriter.list().decimal().writeDecimal(BigDecimal.valueOf(i * 5L)); listWriter.list().endList(); listWriter.endList(); } @@ -597,7 +597,7 @@ public void testCopyFixedSizedListOfDecimalsVector() { writer .decimal() .writeBigEndianBytesToDecimal( - BigDecimal.valueOf(i * 4).unscaledValue().toByteArray(), arrowType); + BigDecimal.valueOf(i * 4L).unscaledValue().toByteArray(), arrowType); writer.endList(); } @@ -631,12 +631,12 @@ public void testCopyUnionListWithDecimal() { listWriter.setPosition(i); listWriter.startList(); - listWriter.decimal().writeDecimal(BigDecimal.valueOf(i * 2)); + listWriter.decimal().writeDecimal(BigDecimal.valueOf(i * 2L)); listWriter.integer().writeInt(i); listWriter .decimal() .writeBigEndianBytesToDecimal( - BigDecimal.valueOf(i * 3).unscaledValue().toByteArray(), + BigDecimal.valueOf(i * 3L).unscaledValue().toByteArray(), new ArrowType.Decimal(3, 0, 128)); listWriter.endList(); @@ -671,15 +671,15 @@ public void testCopyStructVector() { structWriter.setPosition(i); structWriter.start(); structWriter.integer("int").writeInt(i); - structWriter.decimal("dec", 0, 38).writeDecimal(BigDecimal.valueOf(i * 2)); + structWriter.decimal("dec", 0, 38).writeDecimal(BigDecimal.valueOf(i * 2L)); StructWriter innerStructWriter = structWriter.struct("struc"); innerStructWriter.start(); innerStructWriter.integer("innerint").writeInt(i * 3); - innerStructWriter.decimal("innerdec", 0, 38).writeDecimal(BigDecimal.valueOf(i * 4)); + innerStructWriter.decimal("innerdec", 0, 38).writeDecimal(BigDecimal.valueOf(i * 4L)); innerStructWriter .decimal("innerdec", 0, 38) .writeBigEndianBytesToDecimal( - BigDecimal.valueOf(i * 4).unscaledValue().toByteArray(), + BigDecimal.valueOf(i * 4L).unscaledValue().toByteArray(), new ArrowType.Decimal(3, 0, 128)); innerStructWriter.end(); structWriter.end(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java index 281f050dfb662..d7fba5c3b2751 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java @@ -510,7 +510,7 @@ protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { for (int i = 0; i < count; i++) { decimalVector1.setSafe(i, new BigDecimal(BigInteger.valueOf(i), 3)); - decimalVector2.setSafe(i, new BigDecimal(BigInteger.valueOf(i * (1 << 10)), 2)); + decimalVector2.setSafe(i, new BigDecimal(BigInteger.valueOf(i * ((long) (1 << 10))), 2)); decimalVector3.setSafe(i, new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), 8)); } @@ -543,7 +543,7 @@ protected void validateDecimalData(VectorSchemaRoot root) { // Verify decimal 2 vector readValue = decimalVector2.getObject(i); type = (ArrowType.Decimal) decimalVector2.getField().getType(); - genValue = new BigDecimal(BigInteger.valueOf(i * (1 << 10)), type.getScale()); + genValue = new BigDecimal(BigInteger.valueOf(i * ((long) (1 << 10))), type.getScale()); assertEquals(genValue, readValue); // Verify decimal 3 vector diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java index 74ff95d41d69c..c686b124c81ad 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java @@ -783,7 +783,7 @@ public void testReadInterleavedData() throws IOException { Collections.singletonList(dictVector1), dictVector1.getValueCount()); ArrowDictionaryBatch dictionaryBatch1 = - new ArrowDictionaryBatch(1, new VectorUnloader(dictRoot1).getRecordBatch()); + new ArrowDictionaryBatch(1, new VectorUnloader(dictRoot1).getRecordBatch(), false); MessageSerializer.serialize(out, dictionaryBatch1); dictionaryBatch1.close(); dictRoot1.close(); @@ -799,7 +799,7 @@ public void testReadInterleavedData() throws IOException { Collections.singletonList(dictVector2), dictVector2.getValueCount()); ArrowDictionaryBatch dictionaryBatch2 = - new ArrowDictionaryBatch(2, new VectorUnloader(dictRoot2).getRecordBatch()); + new ArrowDictionaryBatch(2, new VectorUnloader(dictRoot2).getRecordBatch(), false); MessageSerializer.serialize(out, dictionaryBatch2); dictionaryBatch2.close(); dictRoot2.close(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index 62e5355d05c8b..12d49206c05d4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -101,7 +101,10 @@ public void list() throws Exception { org.apache.arrow.vector.types.pojo.Schema schema = new org.apache.arrow.vector.types.pojo.Schema(tempSchema.getFields()); schemaBuilder.finish(schema.getSchema(schemaBuilder)); - Schema finalSchema = Schema.deserialize(ByteBuffer.wrap(schemaBuilder.sizedByteArray())); + Schema finalSchema = + Schema.convertSchema( + org.apache.arrow.flatbuf.Schema.getRootAsSchema( + ByteBuffer.wrap(schemaBuilder.sizedByteArray()))); assertFalse(finalSchema.toString().contains("[DEFAULT]")); } @@ -131,6 +134,7 @@ public void schemaMetadata() { } @Test + @SuppressWarnings("EnumOrdinal") public void nestedSchema() { java.util.List children = new ArrayList<>(); children.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null)); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/table/RowTest.java b/java/vector/src/test/java/org/apache/arrow/vector/table/RowTest.java index c1125d407e145..cc9a6ad8fc6c2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/table/RowTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/table/RowTest.java @@ -790,6 +790,7 @@ void testSimpleDenseUnionVector() { } @Test + @SuppressWarnings("FloatingPointLiteralPrecision") void testExtensionTypeVector() { TestExtensionType.LocationVector vector = new TestExtensionType.LocationVector("location", allocator); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java b/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java index c0e2ae252e02c..727737c260af9 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java @@ -141,7 +141,7 @@ public void testPopulateDateMilliVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, i * 1000); + vector1.set(i, i * 1000L); } } vector1.setValueCount(10); @@ -334,7 +334,7 @@ public void testPopulateTimeMicroVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, i * 10000); + vector1.set(i, i * 10000L); } } vector1.setValueCount(10); @@ -372,7 +372,7 @@ public void testPopulateTimeNanoVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, i * 10000); + vector1.set(i, i * 10000L); } } vector1.setValueCount(10); @@ -410,7 +410,7 @@ public void testPopulateTimeStampMicroVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, i * 10000); + vector1.set(i, i * 10000L); } } vector1.setValueCount(10); @@ -429,7 +429,7 @@ public void testPopulateTimeStampMilliVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, i * 10000); + vector1.set(i, i * 10000L); } } vector1.setValueCount(10); @@ -448,7 +448,7 @@ public void testPopulateTimeStampNanoVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, i * 10000); + vector1.set(i, i * 10000L); } } vector1.setValueCount(10); @@ -467,7 +467,7 @@ public void testPopulateTimeStampSecVector() { if (i % 2 == 0) { vector1.setNull(i); } else { - vector1.set(i, i * 100); + vector1.set(i, i * 100L); } } vector1.setValueCount(10); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 8f54a6e5d741f..19eeea1dd301f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -194,6 +194,7 @@ public void testNullCheck() { /** Test that a custom Location type can be round-tripped through a temporary file. */ @Test + @SuppressWarnings("FloatingPointLiteralPrecision") public void roundtripLocation() throws IOException { ExtensionTypeRegistry.register(new LocationType()); final Schema schema = diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java index a7a4035550c21..258a83142f403 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java @@ -25,6 +25,7 @@ public class TestValidator { @Test + @SuppressWarnings("FloatingPointLiteralPrecision") public void testFloatComp() { assertTrue(equalEnough(912.4140000000002F, 912.414F)); assertTrue(equalEnough(912.4140000000002D, 912.414D)); From 7c1d82901664662114a2cb61ce6309b5e12f90cf Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Fri, 13 Sep 2024 07:21:54 +0530 Subject: [PATCH 09/13] fix: cleaning vector module v6 --- .../jdbc/h2/JdbcToArrowVectorIteratorTest.java | 6 +++--- .../apache/arrow/flight/CloseSessionResult.java | 1 + .../org/apache/arrow/flight/DictionaryUtils.java | 2 +- .../arrow/flight/FlightRuntimeException.java | 5 +++++ .../flight/NoOpSessionOptionValueVisitor.java | 6 ++++++ .../arrow/flight/ServerSessionMiddleware.java | 12 +++++++----- .../apache/arrow/flight/SessionOptionValue.java | 3 ++- .../arrow/flight/SessionOptionValueFactory.java | 12 ++++++------ .../arrow/flight/SetSessionOptionsResult.java | 4 +++- .../arrow/flight/grpc/AddWritableBuffer.java | 4 ++-- .../arrow/flight/grpc/GetReadableBuffer.java | 4 ++-- .../org/apache/arrow/flight/TestFlightService.java | 8 ++++++-- .../arrow/driver/jdbc/ArrowFlightJdbcDriver.java | 4 +++- .../ArrowFlightJdbcDecimalVectorAccessor.java | 1 + .../utils/ArrowFlightConnectionConfigImpl.java | 4 +++- .../apache/arrow/driver/jdbc/ResultSetTest.java | 3 +-- .../ArrowFlightJdbcIntervalVectorAccessorTest.java | 14 ++++++++------ ...ractArrowFlightJdbcUnionVectorAccessorTest.java | 7 ++++++- .../apache/arrow/flight/sql/FlightSqlClient.java | 2 +- .../flight/sql/test/TestFlightSqlStreams.java | 1 + .../arrow/flight/sql/util/AdhocTestOption.java | 1 + java/memory/pom.xml | 14 +++++++++++--- java/pom.xml | 2 +- .../org/apache/arrow/tools/EchoServerTest.java | 3 ++- .../org/apache/arrow/vector/Decimal256Vector.java | 8 ++++---- .../vector/ipc/message/MessageSerializer.java | 1 - .../java/org/apache/arrow/vector/types/Types.java | 1 + 27 files changed, 88 insertions(+), 45 deletions(-) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java index de9eff327ef6f..c77c91e730b3e 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java @@ -27,10 +27,10 @@ import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getLongValues; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; @@ -146,10 +146,10 @@ public void testVectorSchemaRootReuse(Table table, boolean reuseVectorSchemaRoot if (reuseVectorSchemaRoot) { // when reuse is enabled, different iterations are based on the same vector schema root. - assertTrue(prev == cur); + assertSame(prev, cur); } else { // when reuse is enabled, a new vector schema root is created in each iteration. - assertFalse(prev == cur); + assertNotEquals(prev, cur); if (batchCount < 3) { cur.close(); } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CloseSessionResult.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CloseSessionResult.java index 73d7bc16a1a7d..78350b63420fe 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CloseSessionResult.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/CloseSessionResult.java @@ -41,6 +41,7 @@ public static Status fromProtocol(Flight.CloseSessionResult.Status proto) { return values()[proto.getNumber()]; } + @SuppressWarnings("EnumOrdinal") public Flight.CloseSessionResult.Status toProtocol() { return Flight.CloseSessionResult.Status.values()[ordinal()]; } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/DictionaryUtils.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/DictionaryUtils.java index cecc1b876ecd9..5d01b98d86758 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/DictionaryUtils.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/DictionaryUtils.java @@ -79,7 +79,7 @@ static Schema generateSchemaMessages( count); final VectorUnloader unloader = new VectorUnloader(dictRoot); try (final ArrowDictionaryBatch dictionaryBatch = - new ArrowDictionaryBatch(id, unloader.getRecordBatch()); + new ArrowDictionaryBatch(id, unloader.getRecordBatch(), false); final ArrowMessage message = new ArrowMessage(dictionaryBatch, option)) { messageCallback.accept(message); } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightRuntimeException.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightRuntimeException.java index d002dd4867f8c..fcb3b82cf025f 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightRuntimeException.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/FlightRuntimeException.java @@ -37,6 +37,11 @@ public CallStatus status() { @Override public String toString() { + return getMessage(); + } + + @Override + public String getMessage() { String s = getClass().getName(); return String.format("%s: %s: %s", s, status.code(), status.description()); } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/NoOpSessionOptionValueVisitor.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/NoOpSessionOptionValueVisitor.java index 34695139107eb..2e19437f11431 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/NoOpSessionOptionValueVisitor.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/NoOpSessionOptionValueVisitor.java @@ -25,26 +25,31 @@ */ public class NoOpSessionOptionValueVisitor implements SessionOptionValueVisitor { /** A callback to handle SessionOptionValue containing a String. */ + @Override public T visit(String value) { return null; } /** A callback to handle SessionOptionValue containing a boolean. */ + @Override public T visit(boolean value) { return null; } /** A callback to handle SessionOptionValue containing a long. */ + @Override public T visit(long value) { return null; } /** A callback to handle SessionOptionValue containing a double. */ + @Override public T visit(double value) { return null; } /** A callback to handle SessionOptionValue containing an array of String. */ + @Override public T visit(String[] value) { return null; } @@ -55,6 +60,7 @@ public T visit(String[] value) { *

By convention, an attempt to set a valueless SessionOptionValue should attempt to unset or * clear the named option value on the server. */ + @Override public T visit(Void value) { return null; } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java index 47fd6f1366272..d1f97118d02a3 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java @@ -16,8 +16,10 @@ */ package org.apache.arrow.flight; +import com.google.common.base.Splitter; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -83,15 +85,15 @@ public ServerSessionMiddleware onCallStarted( if (it != null) { findIdCookie: for (final String headerValue : it) { - for (final String cookie : headerValue.split(" ;")) { - final String[] cookiePair = cookie.split("="); - if (cookiePair.length != 2) { + for (final String cookie : Splitter.on(" ;").split(headerValue)) { + final List cookiePair = Splitter.on('=').splitToList(cookie); + if (cookiePair.size() != 2) { // Soft failure: Ignore invalid cookie list field break; } - if (sessionCookieName.equals(cookiePair[0]) && cookiePair[1].length() > 0) { - sessionId = cookiePair[1]; + if (sessionCookieName.equals(cookiePair.get(0)) && !cookiePair.get(1).isEmpty()) { + sessionId = cookiePair.get(1); break findIdCookie; } } diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SessionOptionValue.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SessionOptionValue.java index 5f342501fb0ad..bc5d7cad31859 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SessionOptionValue.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SessionOptionValue.java @@ -38,7 +38,8 @@ public boolean isEmpty() { return false; } - private class SessionOptionValueToProtocolVisitor implements SessionOptionValueVisitor { + private static class SessionOptionValueToProtocolVisitor + implements SessionOptionValueVisitor { final Flight.SessionOptionValue.Builder b; SessionOptionValueToProtocolVisitor(Flight.SessionOptionValue.Builder b) { diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SessionOptionValueFactory.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SessionOptionValueFactory.java index aff7dffddd487..f55889739fb86 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SessionOptionValueFactory.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SessionOptionValueFactory.java @@ -89,7 +89,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof SessionOptionValueString)) { return false; } SessionOptionValueString that = (SessionOptionValueString) o; @@ -124,7 +124,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof SessionOptionValueBoolean)) { return false; } SessionOptionValueBoolean that = (SessionOptionValueBoolean) o; @@ -159,7 +159,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof SessionOptionValueLong)) { return false; } SessionOptionValueLong that = (SessionOptionValueLong) o; @@ -194,7 +194,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof SessionOptionValueDouble)) { return false; } SessionOptionValueDouble that = (SessionOptionValueDouble) o; @@ -229,7 +229,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof SessionOptionValueStringList)) { return false; } SessionOptionValueStringList that = (SessionOptionValueStringList) o; @@ -266,7 +266,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof SessionOptionValueEmpty)) { return false; } return true; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SetSessionOptionsResult.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SetSessionOptionsResult.java index 8ea953fda3cd0..3d23cd7ba24e3 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SetSessionOptionsResult.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/SetSessionOptionsResult.java @@ -45,12 +45,14 @@ static ErrorValue fromProtocol(Flight.SetSessionOptionsResult.ErrorValue s) { return values()[s.getNumber()]; } + @SuppressWarnings("EnumOrdinal") Flight.SetSessionOptionsResult.ErrorValue toProtocol() { return Flight.SetSessionOptionsResult.ErrorValue.values()[ordinal()]; } } /** Per-option extensible error response container. */ + @SuppressWarnings("JavaLangClash") public static class Error { public ErrorValue value; @@ -74,7 +76,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof Error)) { return false; } Error that = (Error) o; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/AddWritableBuffer.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/AddWritableBuffer.java index 6415bd1a2537a..e4010b48f7df4 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/AddWritableBuffer.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/AddWritableBuffer.java @@ -70,8 +70,8 @@ public class AddWritableBuffer { tmpBufChainOut = tmpBufChainOut2; } catch (Exception ex) { - new RuntimeException("Failed to initialize AddWritableBuffer, falling back to slow path", ex) - .printStackTrace(); + throw new RuntimeException( + "Failed to initialize AddWritableBuffer, falling back to slow path", ex); } bufConstruct = tmpConstruct; diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java index 45c32a86c6a78..2f5470a8bef9c 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java @@ -47,8 +47,8 @@ public class GetReadableBuffer { tmpField = f; tmpClazz = clazz; } catch (Exception e) { - new RuntimeException("Failed to initialize GetReadableBuffer, falling back to slow path", e) - .printStackTrace(); + throw new RuntimeException( + "Failed to initialize GetReadableBuffer, falling back to slow path", e); } READABLE_BUFFER = tmpField; BUFFER_INPUT_STREAM = tmpClazz; diff --git a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java index fc3f83e4eafd3..37d48bbd5aaf7 100644 --- a/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java +++ b/java/flight/flight-core/src/test/java/org/apache/arrow/flight/TestFlightService.java @@ -150,7 +150,9 @@ public FlightInfo getFlightInfo(CallContext context, FlightDescriptor descriptor Exception e = assertThrows( FlightRuntimeException.class, () -> client.getSchema(FlightDescriptor.path("test"))); - assertEquals("No schema is present in FlightInfo", e.getMessage()); + assertEquals( + "org.apache.arrow.flight.FlightRuntimeException: INVALID_ARGUMENT: No schema is present in FlightInfo", + e.getMessage()); } } @@ -211,7 +213,9 @@ public FlightInfo getFlightInfo(CallContext context, FlightDescriptor descriptor FlightRuntimeException.class, () -> client.getInfo(FlightDescriptor.path("test"), new HeaderCallOption(callHeaders))); - assertEquals("http2 exception", e.getMessage()); + assertEquals( + "org.apache.arrow.flight.FlightRuntimeException: INTERNAL: http2 exception", + e.getMessage()); } } diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java index 53e6120f6271b..c1f1f2196bf00 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/ArrowFlightJdbcDriver.java @@ -26,6 +26,7 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.sql.SQLException; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -277,7 +278,8 @@ Optional> getUrlsArgs(String url) throws SQLException { static Properties lowerCasePropertyKeys(final Properties properties) { final Properties resultProperty = new Properties(); - properties.forEach((k, v) -> resultProperty.put(k.toString().toLowerCase(), v)); + properties.forEach( + (k, v) -> resultProperty.put(k.toString().toLowerCase(Locale.getDefault()), v)); return resultProperty; } } diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcDecimalVectorAccessor.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcDecimalVectorAccessor.java index 325e9bc700ad9..1d1caa5bb58bd 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcDecimalVectorAccessor.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcDecimalVectorAccessor.java @@ -71,6 +71,7 @@ public String getString() { } @Override + @SuppressWarnings("BigDecimalEquals") public boolean getBoolean() { final BigDecimal value = this.getBigDecimal(); diff --git a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java index e8bae2a207346..f122055d1a5d5 100644 --- a/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java +++ b/java/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/ArrowFlightConnectionConfigImpl.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Properties; @@ -196,6 +197,7 @@ public Map getHeaderAttributes() { } /** Custom {@link ConnectionProperty} for the {@link ArrowFlightConnectionConfigImpl}. */ + @SuppressWarnings("Immutable") public enum ArrowFlightConnectionProperty implements ConnectionProperty { HOST("host", null, Type.STRING, true), PORT("port", null, Type.NUMBER, true), @@ -241,7 +243,7 @@ public Object get(final Properties properties) { Preconditions.checkNotNull(properties, "Properties cannot be null."); Object value = properties.get(camelName); if (value == null) { - value = properties.get(camelName.toLowerCase()); + value = properties.get(camelName.toLowerCase(Locale.getDefault())); } if (required) { if (value == null) { diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java index a8d04dfc83ac0..2849e17124e52 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ResultSetTest.java @@ -156,8 +156,6 @@ public void testShouldRunSelectQuerySettingMaxRowLimit() throws Exception { /** * Tests whether the {@link ArrowFlightJdbcDriver} fails upon attempting to run an invalid query. - * - * @throws Exception If the connection fails to be established. */ @Test public void testShouldThrowExceptionUponAttemptingToExecuteAnInvalidSelectQuery() { @@ -336,6 +334,7 @@ public void testShouldInterruptFlightStreamsIfQueryIsCancelledMidQuerying() } @Test + @SuppressWarnings("ThreadPriorityCheck") public void testShouldInterruptFlightStreamsIfQueryIsCancelledMidProcessingForTimeConsumingQueries() throws SQLException, InterruptedException { diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessorTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessorTest.java index 521debc323e83..5fa3899443a82 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessorTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/calendar/ArrowFlightJdbcIntervalVectorAccessorTest.java @@ -22,9 +22,11 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.jupiter.api.Assertions.assertEquals; +import com.google.common.base.Splitter; import java.time.Duration; import java.time.Period; import java.time.format.DateTimeParseException; +import java.util.List; import java.util.function.Supplier; import java.util.stream.Stream; import org.apache.arrow.driver.jdbc.accessor.ArrowFlightJdbcAccessorFactory; @@ -110,7 +112,7 @@ public static Stream data() { int valueCount = 10; vector.setValueCount(valueCount); for (int i = 0; i < valueCount; i++) { - vector.set(i, i + 1, (i + 1) * 10, (i + 1) * 100); + vector.set(i, i + 1, (i + 1) * 10, (i + 1) * 100L); } return vector; }, @@ -172,11 +174,11 @@ private String getStringOnVector(ValueVector vector, int index) { return formatIntervalYear(Period.parse(object.toString())); } else if (vector instanceof IntervalMonthDayNanoVector) { String iso8601IntervalString = ((PeriodDuration) object).toISO8601IntervalString(); - String[] periodAndDuration = iso8601IntervalString.split("T"); - if (periodAndDuration.length == 1) { + List periodAndDuration = Splitter.on('T').splitToList(iso8601IntervalString); + if (periodAndDuration.size() == 1) { // If there is no 'T', then either Period or Duration is zero, and the other one will // successfully parse it - String periodOrDuration = periodAndDuration[0]; + String periodOrDuration = periodAndDuration.get(0); try { return new PeriodDuration(Period.parse(periodOrDuration), Duration.ZERO) .toISO8601IntervalString(); @@ -188,8 +190,8 @@ private String getStringOnVector(ValueVector vector, int index) { // If there is a 'T', both Period and Duration are non-zero, and we just need to prepend the // 'PT' to the // duration for both to parse successfully - Period parse = Period.parse(periodAndDuration[0]); - Duration duration = Duration.parse("PT" + periodAndDuration[1]); + Period parse = Period.parse(periodAndDuration.get(0)); + Duration duration = Duration.parse("PT" + periodAndDuration.get(1)); return new PeriodDuration(parse, duration).toISO8601IntervalString(); } } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcUnionVectorAccessorTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcUnionVectorAccessorTest.java index 6a143d2ee888f..d421eb8e41e8e 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcUnionVectorAccessorTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/complex/AbstractArrowFlightJdbcUnionVectorAccessorTest.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.driver.jdbc.accessor.impl.complex; +import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -27,6 +28,9 @@ import org.apache.arrow.driver.jdbc.accessor.impl.ArrowFlightJdbcNullVectorAccessor; import org.apache.arrow.vector.NullVector; import org.apache.arrow.vector.ValueVector; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -253,7 +257,8 @@ protected byte getCurrentTypeId() { @Override protected ValueVector getVectorByTypeId(byte typeId) { - return new NullVector(); + return new NullVector( + new Field(DATA_VECTOR_NAME, FieldType.nullable(new ArrowType.Null()), null)); } } } diff --git a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java index 9a6ffdfdca847..ac51e3fa249ff 100644 --- a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java +++ b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java @@ -1240,7 +1240,7 @@ public static class PreparedStatement implements AutoCloseable { * used in the {@code PreparedStatement} setters. */ public void setParameters(final VectorSchemaRoot parameterBindingRoot) { - if (parameterBindingRoot == this.parameterBindingRoot) { + if (parameterBindingRoot.equals(this.parameterBindingRoot)) { // Nothing to do if we're attempting to set the same parameters again. return; } diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java index 71c0dc88e4ec2..7c96e0aed5212 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStreams.java @@ -144,6 +144,7 @@ public void getStreamSqlInfo( } @Override + @SuppressWarnings("EnumOrdinal") public void getStreamTypeInfo( FlightSql.CommandGetXdbcTypeInfo request, CallContext context, diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/util/AdhocTestOption.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/util/AdhocTestOption.java index 82aa883935f27..6f5b2df1ad784 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/util/AdhocTestOption.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/util/AdhocTestOption.java @@ -26,6 +26,7 @@ enum AdhocTestOption implements ProtocolMessageEnum { OPTION_C; @Override + @SuppressWarnings("EnumOrdinal") public int getNumber() { return ordinal(); } diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 15761ae50b26c..3cd20b40f620f 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -40,12 +40,20 @@ under the License. org.apache.maven.plugins maven-compiler-plugin - 3.8.1 - 11 - 11 -XDcompilePolicy=simple + -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-source|target/generated-sources|format/src/main/java/org/apache/arrow/flatbuf)/.* + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED diff --git a/java/pom.xml b/java/pom.xml index ba2df05cbd976..2051caf572a24 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -923,7 +923,7 @@ under the License. -XDcompilePolicy=simple - -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-source|target/generated-sources|format/src/main/java/org/apache/arrow/flatbuf)/.* + -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-source|target/generated-sources|target/generated-test-sources|format/src/main/java/org/apache/arrow/flatbuf)/.* -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java index 239913ca4ea1e..fe727b83b2326 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableList; import java.io.IOException; +import java.io.UncheckedIOException; import java.net.Socket; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; @@ -77,7 +78,7 @@ public void run() { try { server.run(); } catch (IOException e) { - e.printStackTrace(); + throw new UncheckedIOException(e); } } }; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java index 42ad741c85f8b..1bcb7426779a1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java @@ -412,7 +412,7 @@ public void setBigEndianSafe(int index, byte[] value) { } /** - * Same as {@link #set(int, int, ArrowBuf)} except that it handles the case when index is greater + * Same as {@link #set(int, long, ArrowBuf)} except that it handles the case when index is greater * than or equal to existing value capacity {@link #getValueCapacity()}. * * @param index position of element @@ -449,7 +449,7 @@ public void setSafe(int index, long value) { } /** - * Same as {@link #set(int, NullableDecimalHolder)} except that it handles the case when index is + * Same as {@link #set(int, Decimal256Holder)} except that it handles the case when index is * greater than or equal to existing value capacity {@link #getValueCapacity()}. * * @param index position of element @@ -490,8 +490,8 @@ public void set(int index, int isSet, long start, ArrowBuf buffer) { } /** - * Same as {@link #setSafe(int, int, int, ArrowBuf)} except that it handles the case when the - * position of new value is beyond the current value capacity of the vector. + * Same as {@link #setSafe(int, long, ArrowBuf)} except that it handles the case when the position + * of new value is beyond the current value capacity of the vector. * * @param index position of the new value * @param isSet 0 for NULL value, 1 otherwise diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java index 6c6a5893ccc9c..7ba72ed2bb5c9 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/MessageSerializer.java @@ -740,7 +740,6 @@ public static MessageMetadataResult readMessage(ReadChannel in) throws IOExcepti if (in.readFully(messageBuffer) != messageLength) { throw new IOException("Unexpected end of stream trying to read message."); } - // see https://github.com/apache/arrow/issues/41717 for reason why we cast to // java.nio.Buffer ByteBuffer rewindBuffer = (ByteBuffer) ((java.nio.Buffer) messageBuffer).rewind(); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 5859f245ef955..6b2c56de01c40 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -787,6 +787,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { } }, ; + private final ArrowType type; MinorType(ArrowType type) { From 003e1ab519d952b27301dd27cd9d623b3ce6314d Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Fri, 13 Sep 2024 13:14:14 +0530 Subject: [PATCH 10/13] fix: cleaning vector module v7 --- .../org/apache/arrow/c/RoundtripTest.java | 5 +- .../tests/IntegrationTestServer.java | 2 +- .../tests/TestBufferAllocationListener.java | 2 + ...ArrowFlightStatementExecuteUpdateTest.java | 5 +- ...owFlightJdbcDecimalVectorAccessorTest.java | 1 + .../arrow/gandiva/evaluator/Filter.java | 5 +- .../gandiva/evaluator/FunctionSignature.java | 7 +- .../arrow/gandiva/evaluator/JniLoader.java | 9 +- .../arrow/gandiva/evaluator/Projector.java | 7 +- .../gandiva/evaluator/SelectionVector.java | 2 +- .../evaluator/SelectionVectorInt16.java | 3 +- .../evaluator/SelectionVectorInt32.java | 2 +- .../gandiva/exceptions/GandivaException.java | 7 +- .../gandiva/evaluator/BaseEvaluatorTest.java | 47 +++--- .../arrow/gandiva/evaluator/FilterTest.java | 8 +- .../evaluator/ProjectorDecimalTest.java | 7 +- .../gandiva/evaluator/ProjectorTest.java | 152 +++++++++--------- .../gandiva/expression/TreeBuilderTest.java | 14 +- .../adapter/jdbc/JdbcAdapterBenchmarks.java | 6 +- .../vector/BaseValueVectorBenchmarks.java | 2 +- .../arrow/vector/VarCharBenchmarks.java | 3 +- .../vector/VariableWidthVectorBenchmarks.java | 4 +- .../DictionaryEncoderBenchmarks.java | 2 +- 23 files changed, 167 insertions(+), 135 deletions(-) diff --git a/java/c/src/test/java/org/apache/arrow/c/RoundtripTest.java b/java/c/src/test/java/org/apache/arrow/c/RoundtripTest.java index d8286465e475f..ab6ac2d4ec45e 100644 --- a/java/c/src/test/java/org/apache/arrow/c/RoundtripTest.java +++ b/java/c/src/test/java/org/apache/arrow/c/RoundtripTest.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.c; +import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME; import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -965,7 +966,9 @@ public void testImportReleasedArray() { try (ArrowSchema arrowSchema = ArrowSchema.wrap(consumerArrowSchema.memoryAddress()); ArrowArray arrowArray = ArrowArray.wrap(consumerArrowArray.memoryAddress())) { // Producer exports vector into the C Data Interface structures - try (final NullVector vector = new NullVector()) { + try (final NullVector vector = + new NullVector( + new Field(DATA_VECTOR_NAME, FieldType.nullable(new ArrowType.Null()), null))) { Data.exportVector(allocator, vector, null, arrowArray, arrowSchema); } } diff --git a/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/IntegrationTestServer.java b/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/IntegrationTestServer.java index d6e53cd427c73..f91df1403702c 100644 --- a/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/IntegrationTestServer.java +++ b/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/IntegrationTestServer.java @@ -71,7 +71,7 @@ private void run(String[] args) throws Exception { System.out.println("\nExiting..."); AutoCloseables.close(server, allocator); } catch (Exception e) { - e.printStackTrace(); + LOGGER.error("Error during shutdown", e); } })); diff --git a/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/TestBufferAllocationListener.java b/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/TestBufferAllocationListener.java index 10594d4cf0962..e7e3e9ac6f7cb 100644 --- a/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/TestBufferAllocationListener.java +++ b/java/flight/flight-integration-tests/src/main/java/org/apache/arrow/flight/integration/tests/TestBufferAllocationListener.java @@ -35,10 +35,12 @@ public Entry(StackTraceElement[] stackTrace, long size, boolean forAllocation) { List trail = new ArrayList<>(); + @Override public void onAllocation(long size) { trail.add(new Entry(Thread.currentThread().getStackTrace(), size, true)); } + @Override public void onRelease(long size) { trail.add(new Entry(Thread.currentThread().getStackTrace(), size, false)); } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteUpdateTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteUpdateTest.java index f7c31c590c556..7f4870a74dc1c 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteUpdateTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/ArrowFlightStatementExecuteUpdateTest.java @@ -220,7 +220,10 @@ public void testShouldFailToPrepareStatementForBadStatement() { */ assertThat( e.getMessage(), - is(format("Error while executing SQL \"%s\": Query not found", badQuery))); + is( + format( + "Error while executing SQL \"%s\": org.apache.arrow.flight.FlightRuntimeException: INVALID_ARGUMENT: Query not found", + badQuery))); } assertThat(count, is(1)); } diff --git a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcDecimalVectorAccessorTest.java b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcDecimalVectorAccessorTest.java index 668c50a843386..a3ca62f3db3aa 100644 --- a/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcDecimalVectorAccessorTest.java +++ b/java/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/accessor/impl/numeric/ArrowFlightJdbcDecimalVectorAccessorTest.java @@ -176,6 +176,7 @@ public void testShouldGetStringMethodFromDecimalVector(Supplier vec @ParameterizedTest @MethodSource("data") + @SuppressWarnings("BigDecimalEquals") public void testShouldGetBooleanMethodFromDecimalVector(Supplier vectorSupplier) throws Exception { setup(vectorSupplier); diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java index f584564e0920c..b47e87bd4d9a2 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java @@ -42,7 +42,10 @@ public class Filter { private final JniWrapper wrapper; private final long moduleId; + + @SuppressWarnings("UnusedVariable") private final Schema schema; + private boolean closed; private Filter(JniWrapper wrapper, long moduleId, Schema schema) { @@ -98,7 +101,7 @@ public static Filter make(Schema schema, Condition condition, boolean optimize) schema, condition, JniLoader.getConfiguration( - (new ConfigurationBuilder.ConfigOptions()).withOptimize(optimize))); + new ConfigurationBuilder.ConfigOptions().withOptimize(optimize))); } /** diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/FunctionSignature.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/FunctionSignature.java index 164b6aaac6684..3652db5feb762 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/FunctionSignature.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/FunctionSignature.java @@ -19,6 +19,7 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import java.util.List; +import java.util.Locale; import org.apache.arrow.vector.types.pojo.ArrowType; /** POJO to define a function signature. */ @@ -58,11 +59,12 @@ public FunctionSignature(String name, ArrowType returnType, List para * @param signature - signature to compare * @return true if equal and false if not. */ + @Override public boolean equals(Object signature) { if (signature == null) { return false; } - if (getClass() != signature.getClass()) { + if (!(signature instanceof FunctionSignature)) { return false; } final FunctionSignature other = (FunctionSignature) signature; @@ -73,7 +75,8 @@ public boolean equals(Object signature) { @Override public int hashCode() { - return Objects.hashCode(this.name.toLowerCase(), this.returnType, this.paramTypes); + return Objects.hashCode( + this.name.toLowerCase(Locale.getDefault()), this.returnType, this.paramTypes); } @Override diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java index 6f4cdc58c5a98..e9670e035584d 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java @@ -127,7 +127,7 @@ JniWrapper getWrapper() throws GandivaException { static long getConfiguration(ConfigurationBuilder.ConfigOptions configOptions) throws GandivaException { if (!configurationMap.containsKey(configOptions)) { - synchronized (ConfigurationBuilder.class) { + synchronized (JniLoader.class) { if (!configurationMap.containsKey(configOptions)) { JniLoader.getInstance(); // setup long configInstance = new ConfigurationBuilder().buildConfigInstance(configOptions); @@ -150,7 +150,7 @@ static long getConfiguration(ConfigurationBuilder.ConfigOptions configOptions) */ static long getDefaultConfiguration() throws GandivaException { if (defaultConfiguration == 0L) { - synchronized (ConfigurationBuilder.class) { + synchronized (JniLoader.class) { if (defaultConfiguration == 0L) { JniLoader.getInstance(); // setup ConfigurationBuilder.ConfigOptions defaultConfigOptions = @@ -167,10 +167,9 @@ static long getDefaultConfiguration() throws GandivaException { /** Remove the configuration. */ static void removeConfiguration(ConfigurationBuilder.ConfigOptions configOptions) { if (configurationMap.containsKey(configOptions)) { - synchronized (ConfigurationBuilder.class) { + synchronized (JniLoader.class) { if (configurationMap.containsKey(configOptions)) { - (new ConfigurationBuilder()) - .releaseConfigInstance(configurationMap.remove(configOptions)); + new ConfigurationBuilder().releaseConfigInstance(configurationMap.remove(configOptions)); if (configOptions.equals(ConfigurationBuilder.ConfigOptions.getDefault())) { defaultConfiguration = 0; } diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java index 5c16c46e5ea46..f9bf570396420 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java @@ -45,7 +45,10 @@ public class Projector { private JniWrapper wrapper; private final long moduleId; + + @SuppressWarnings("UnusedVariable") private final Schema schema; + private final int numExprs; private boolean closed; @@ -108,7 +111,7 @@ public static Projector make(Schema schema, List exprs, boolean exprs, SelectionVectorType.SV_NONE, JniLoader.getConfiguration( - (new ConfigurationBuilder.ConfigOptions()).withOptimize(optimize))); + new ConfigurationBuilder.ConfigOptions().withOptimize(optimize))); } /** @@ -173,7 +176,7 @@ public static Projector make( exprs, selectionVectorType, JniLoader.getConfiguration( - (new ConfigurationBuilder.ConfigOptions()).withOptimize(optimize))); + new ConfigurationBuilder.ConfigOptions().withOptimize(optimize))); } /** diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVector.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVector.java index 4bb1f2be74a0e..7af23144af81a 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVector.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVector.java @@ -55,7 +55,7 @@ public final int getRecordCount() { * Set the number of records in the selection vector. */ final void setRecordCount(int recordCount) { - if (recordCount * getRecordSize() > buffer.capacity()) { + if ((long) recordCount * getRecordSize() > buffer.capacity()) { throw new IllegalArgumentException( "recordCount " + recordCount diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVectorInt16.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVectorInt16.java index 8ef35d0073cbb..9f32a505ff28d 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVectorInt16.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVectorInt16.java @@ -40,7 +40,6 @@ public SelectionVectorType getType() { public int getIndex(int index) { checkReadBounds(index); - char value = getBuffer().getChar(index * getRecordSize()); - return (int) value; + return getBuffer().getChar((long) index * getRecordSize()); } } diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVectorInt32.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVectorInt32.java index 324dd54be244b..81b2c8eb62043 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVectorInt32.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/SelectionVectorInt32.java @@ -40,6 +40,6 @@ public SelectionVectorType getType() { public int getIndex(int index) { checkReadBounds(index); - return getBuffer().getInt(index * getRecordSize()); + return getBuffer().getInt((long) index * getRecordSize()); } } diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java index 4f3f0f8fcf29d..9365bdecc000b 100644 --- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java +++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java @@ -29,6 +29,11 @@ public GandivaException(String msg, Exception cause) { @Override public String toString() { - return getMessage(); + return this.getMessage(); + } + + @Override + public String getMessage() { + return super.getMessage(); } } diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/BaseEvaluatorTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/BaseEvaluatorTest.java index 5f585e671f676..242500a10dd38 100644 --- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/BaseEvaluatorTest.java +++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/BaseEvaluatorTest.java @@ -16,7 +16,10 @@ */ package org.apache.arrow.gandiva.evaluator; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.HashSet; @@ -51,7 +54,7 @@ interface BaseEvaluator { long getElapsedMillis(); } - class ProjectEvaluator implements BaseEvaluator { + static class ProjectEvaluator implements BaseEvaluator { private Projector projector; private DataAndVectorGenerator generator; @@ -97,9 +100,9 @@ public long getElapsedMillis() { } } - class FilterEvaluator implements BaseEvaluator { + static class FilterEvaluator implements BaseEvaluator { - private Filter filter; + private final Filter filter; private long elapsedTime = 0; public FilterEvaluator(Filter filter) { @@ -109,7 +112,7 @@ public FilterEvaluator(Filter filter) { @Override public void evaluate(ArrowRecordBatch recordBatch, BufferAllocator allocator) throws GandivaException { - ArrowBuf selectionBuffer = allocator.buffer(recordBatch.getLength() * 2); + ArrowBuf selectionBuffer = allocator.buffer(recordBatch.getLength() * 2L); SelectionVectorInt16 selectionVector = new SelectionVectorInt16(selectionBuffer); try { @@ -135,7 +138,7 @@ interface DataAndVectorGenerator { ValueVector generateOutputVector(int numRowsInBatch); } - class Int32DataAndVectorGenerator implements DataAndVectorGenerator { + static class Int32DataAndVectorGenerator implements DataAndVectorGenerator { protected final BufferAllocator allocator; protected final Random rand; @@ -158,7 +161,7 @@ public ValueVector generateOutputVector(int numRowsInBatch) { } } - class BoundedInt32DataAndVectorGenerator extends Int32DataAndVectorGenerator { + static class BoundedInt32DataAndVectorGenerator extends Int32DataAndVectorGenerator { private final int upperBound; @@ -222,7 +225,7 @@ ArrowBuf arrowBufWithAllValid(int size) { } ArrowBuf intBuf(int[] ints) { - ArrowBuf buffer = allocator.buffer(ints.length * 4); + ArrowBuf buffer = allocator.buffer(ints.length * 4L); for (int i = 0; i < ints.length; i++) { buffer.writeInt(ints[i]); } @@ -255,7 +258,7 @@ VarCharVector varcharVector(String[] values) { VarCharVector vector = new VarCharVector("VarCharVector" + Math.random(), allocator); vector.allocateNew(); for (int i = 0; i < values.length; i++) { - vector.setSafe(i, values[i].getBytes(), 0, values[i].length()); + vector.setSafe(i, values[i].getBytes(StandardCharsets.UTF_8), 0, values[i].length()); } vector.setValueCount(values.length); @@ -263,26 +266,26 @@ VarCharVector varcharVector(String[] values) { } ArrowBuf longBuf(long[] longs) { - ArrowBuf buffer = allocator.buffer(longs.length * 8); - for (int i = 0; i < longs.length; i++) { - buffer.writeLong(longs[i]); + ArrowBuf buffer = allocator.buffer(longs.length * 8L); + for (long aLong : longs) { + buffer.writeLong(aLong); } return buffer; } ArrowBuf doubleBuf(double[] data) { - ArrowBuf buffer = allocator.buffer(data.length * 8); - for (int i = 0; i < data.length; i++) { - buffer.writeDouble(data[i]); + ArrowBuf buffer = allocator.buffer(data.length * 8L); + for (double datum : data) { + buffer.writeDouble(datum); } return buffer; } ArrowBuf stringToMillis(String[] dates) { - ArrowBuf buffer = allocator.buffer(dates.length * 8); - for (int i = 0; i < dates.length; i++) { - Instant instant = Instant.parse(dates[i]); + ArrowBuf buffer = allocator.buffer(dates.length * 8L); + for (String date : dates) { + Instant instant = Instant.parse(date); buffer.writeLong(instant.toEpochMilli()); } @@ -290,10 +293,10 @@ ArrowBuf stringToMillis(String[] dates) { } ArrowBuf stringToDayInterval(String[] values) { - ArrowBuf buffer = allocator.buffer(values.length * 8); - for (int i = 0; i < values.length; i++) { - buffer.writeInt(Integer.parseInt(values[i].split(" ")[0])); // days - buffer.writeInt(Integer.parseInt(values[i].split(" ")[1])); // millis + ArrowBuf buffer = allocator.buffer(values.length * 8L); + for (String value : values) { + buffer.writeInt(Integer.parseInt(Iterables.get(Splitter.on(' ').split(value), 0))); // days + buffer.writeInt(Integer.parseInt(Iterables.get(Splitter.on(' ').split(value), 1))); // millis } return buffer; } @@ -342,7 +345,7 @@ private void generateDataAndEvaluate( // generate data for (int i = 0; i < numFields; i++) { - ArrowBuf buf = allocator.buffer(numRowsInBatch * inputFieldSize); + ArrowBuf buf = allocator.buffer((long) numRowsInBatch * inputFieldSize); ArrowBuf validity = arrowBufWithAllValid(maxRowsInBatch); generateData(generator, numRowsInBatch, buf); diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterTest.java index a98a7cb6b5466..faf82a59cbf6b 100644 --- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterTest.java +++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/FilterTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; import java.util.stream.IntStream; @@ -47,12 +48,11 @@ private int[] selectionVectorToArray(SelectionVector vector) { return actual; } - private Charset utf8Charset = Charset.forName("UTF-8"); - private Charset utf16Charset = Charset.forName("UTF-16"); + private final Charset utf8Charset = StandardCharsets.UTF_8; List varBufs(String[] strings, Charset charset) { - ArrowBuf offsetsBuffer = allocator.buffer((strings.length + 1) * 4); - ArrowBuf dataBuffer = allocator.buffer(strings.length * 8); + ArrowBuf offsetsBuffer = allocator.buffer((strings.length + 1) * 4L); + ArrowBuf dataBuffer = allocator.buffer(strings.length * 8L); int startOffset = 0; for (int i = 0; i < strings.length; i++) { diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java index 74180c0f3598f..ca3f2c834f51f 100644 --- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java +++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorDecimalTest.java @@ -69,7 +69,6 @@ public void test_add() throws GandivaException { Projector eval = Projector.make(schema, exprs); int numRows = 4; - byte[] validity = new byte[] {(byte) 255}; String[] aValues = new String[] {"1.12345678", "2.12345678", "3.12345678", "4.12345678"}; String[] bValues = new String[] {"2.12345678", "3.12345678", "4.12345678", "5.12345678"}; @@ -199,7 +198,6 @@ public void test_multiply() throws GandivaException { Projector eval = Projector.make(schema, exprs); int numRows = 4; - byte[] validity = new byte[] {(byte) 255}; String[] aValues = new String[] {"1.12345678", "2.12345678", "3.12345678", "999999999999.99999999"}; String[] bValues = @@ -786,7 +784,6 @@ public void testCastToString() throws GandivaException { @Disabled("GH-43576 - Fix and enable this test") public void testCastStringToDecimal() throws GandivaException { Decimal decimalType = new Decimal(4, 2, 128); - Field dec = Field.nullable("dec", decimalType); Field str = Field.nullable("str", new ArrowType.Utf8()); TreeNode field = TreeBuilder.makeField(str); @@ -860,7 +857,7 @@ public void testInvalidDecimal() throws GandivaException { assertThrows( IllegalArgumentException.class, () -> { - Projector eval = + Projector unused = Projector.make( schema, Lists.newArrayList( @@ -884,7 +881,7 @@ public void testInvalidDecimalGt38() throws GandivaException { assertThrows( IllegalArgumentException.class, () -> { - Projector eval = + Projector unused = Projector.make( schema, Lists.newArrayList( diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java index 0d86bd9e72923..1e09ca09e4370 100644 --- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java +++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.Sets; import java.math.BigDecimal; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -33,6 +34,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; import org.apache.arrow.gandiva.exceptions.GandivaException; @@ -69,7 +71,7 @@ public class ProjectorTest extends BaseEvaluatorTest { private Charset utf16Charset = Charset.forName("UTF-16"); List varBufs(String[] strings, Charset charset) { - ArrowBuf offsetsBuffer = allocator.buffer((strings.length + 1) * 4); + ArrowBuf offsetsBuffer = allocator.buffer((strings.length + 1) * 4L); long dataBufferSize = 0L; for (String string : strings) { @@ -131,19 +133,22 @@ private void testMakeProjectorParallel(ConfigurationBuilder.ConfigOptions config IntStream.range(0, 1000) .forEach( i -> { - executors.submit( - () -> { - try { - Projector evaluator = - configOptions == null - ? Projector.make(schemas.get((int) (Math.random() * 100)), exprs) - : Projector.make( - schemas.get((int) (Math.random() * 100)), exprs, configOptions); - evaluator.close(); - } catch (GandivaException e) { - e.printStackTrace(); - } - }); + Future unused = + executors.submit( + () -> { + try { + Projector evaluator = + configOptions == null + ? Projector.make(schemas.get((int) (Math.random() * 100)), exprs) + : Projector.make( + schemas.get((int) (Math.random() * 100)), + exprs, + configOptions); + evaluator.close(); + } catch (GandivaException e) { + throw new RuntimeException(e); + } + }); }); executors.shutdown(); executors.awaitTermination(100, java.util.concurrent.TimeUnit.SECONDS); @@ -213,7 +218,7 @@ public void testMakeProjectorValidationError() throws InterruptedException { boolean exceptionThrown = false; try { - Projector evaluator1 = Projector.make(schema, exprs); + Projector unused = Projector.make(schema, exprs); } catch (GandivaException e) { exceptionThrown = true; } @@ -226,7 +231,7 @@ public void testMakeProjectorValidationError() throws InterruptedException { // try again to ensure no temporary resources. exceptionThrown = false; try { - Projector evaluator1 = Projector.make(schema, exprs); + Projector unused = Projector.make(schema, exprs); } catch (GandivaException e) { exceptionThrown = true; } @@ -358,49 +363,51 @@ public void testDivZeroParallel() throws GandivaException, InterruptedException IntStream.range(0, 1000) .forEach( i -> { - executors.submit( - () -> { - try { - Projector evaluator = Projector.make(s, exprs); - int numRows = 2; - byte[] validity = new byte[] {(byte) 255}; - int[] aValues = new int[] {2, 2}; - int[] bValues; - if (i % 2 == 0) { - errorCountExp.incrementAndGet(); - bValues = new int[] {1, 0}; - } else { - bValues = new int[] {1, 1}; - } - - ArrowBuf validitya = buf(validity); - ArrowBuf valuesa = intBuf(aValues); - ArrowBuf validityb = buf(validity); - ArrowBuf valuesb = intBuf(bValues); - ArrowRecordBatch batch = - new ArrowRecordBatch( - numRows, - Lists.newArrayList( - new ArrowFieldNode(numRows, 0), new ArrowFieldNode(numRows, 0)), - Lists.newArrayList(validitya, valuesa, validityb, valuesb)); - - IntVector intVector = new IntVector(EMPTY_SCHEMA_PATH, allocator); - intVector.allocateNew(numRows); - - List output = new ArrayList(); - output.add(intVector); - try { - evaluator.evaluate(batch, output); - } catch (GandivaException e) { - errorCount.incrementAndGet(); - } - // free buffers - releaseRecordBatch(batch); - releaseValueVectors(output); - evaluator.close(); - } catch (GandivaException ignore) { - } - }); + Future unused = + executors.submit( + () -> { + try { + Projector evaluator = Projector.make(s, exprs); + int numRows = 2; + byte[] validity = new byte[] {(byte) 255}; + int[] aValues = new int[] {2, 2}; + int[] bValues; + if (i % 2 == 0) { + errorCountExp.incrementAndGet(); + bValues = new int[] {1, 0}; + } else { + bValues = new int[] {1, 1}; + } + + ArrowBuf validitya = buf(validity); + ArrowBuf valuesa = intBuf(aValues); + ArrowBuf validityb = buf(validity); + ArrowBuf valuesb = intBuf(bValues); + ArrowRecordBatch batch = + new ArrowRecordBatch( + numRows, + Lists.newArrayList( + new ArrowFieldNode(numRows, 0), + new ArrowFieldNode(numRows, 0)), + Lists.newArrayList(validitya, valuesa, validityb, valuesb)); + + IntVector intVector = new IntVector(EMPTY_SCHEMA_PATH, allocator); + intVector.allocateNew(numRows); + + List output = new ArrayList(); + output.add(intVector); + try { + evaluator.evaluate(batch, output); + } catch (GandivaException e) { + errorCount.incrementAndGet(); + } + // free buffers + releaseRecordBatch(batch); + releaseValueVectors(output); + evaluator.close(); + } catch (GandivaException ignore) { + } + }); }); executors.shutdown(); executors.awaitTermination(100, java.util.concurrent.TimeUnit.SECONDS); @@ -414,8 +421,6 @@ public void testAdd3() throws GandivaException, Exception { Field n2x = Field.nullable("n2x", int32); Field n3x = Field.nullable("n3x", int32); - List args = new ArrayList(); - // x + n2x + n3x TreeNode add1 = TreeBuilder.makeFunction( @@ -595,7 +600,7 @@ public void testStringOutput() throws GandivaException { // match expected output. for (int i = 0; i < numRows; i++) { assertFalse(outVector.isNull(i)); - assertEquals(expected[i], new String(outVector.get(i))); + assertEquals(expected[i], new String(outVector.get(i), StandardCharsets.UTF_8)); } // test with insufficient data buffer. @@ -704,7 +709,7 @@ public void testRegexpReplace() throws GandivaException { // allocate data for output vector. VarCharVector outVector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator); - outVector.allocateNew(numRows * 15, numRows); + outVector.allocateNew(numRows * 15L, numRows); // evaluate expression List output = new ArrayList<>(); @@ -715,7 +720,7 @@ public void testRegexpReplace() throws GandivaException { // match expected output. for (int i = 0; i < numRows - 1; i++) { assertFalse(outVector.isNull(i), "Expect none value equals null"); - assertEquals(expected[i], new String(outVector.get(i))); + assertEquals(expected[i], new String(outVector.get(i), StandardCharsets.UTF_8)); } assertTrue(outVector.isNull(numRows - 1), "Last value must be null"); @@ -1892,7 +1897,7 @@ public void testUnknownFunction() { Schema schema = new Schema(Lists.newArrayList(c1, c2)); boolean caughtException = false; try { - Projector eval = Projector.make(schema, Lists.newArrayList(expr)); + Projector unused = Projector.make(schema, Lists.newArrayList(expr)); } catch (GandivaException ge) { caughtException = true; } @@ -1958,7 +1963,7 @@ public void testCastTimestampToString() throws Exception { for (int i = 0; i < exprs.size(); i++) { VarCharVector charVector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator); - charVector.allocateNew(numRows * 23, numRows); + charVector.allocateNew(numRows * 23L, numRows); output.add(charVector); } eval.evaluate(batch, output); @@ -1969,7 +1974,7 @@ public void testCastTimestampToString() throws Exception { for (int j = 0; j < numRows; j++) { assertFalse(charVector.isNull(j)); - assertEquals(expValues[j], new String(charVector.get(j))); + assertEquals(expValues[j], new String(charVector.get(j), StandardCharsets.UTF_8)); } } @@ -2416,7 +2421,7 @@ public void testCastVarcharFromInteger() throws Exception { for (int i = 0; i < exprs.size(); i++) { VarCharVector charVector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator); - charVector.allocateNew(numRows * 5, numRows); + charVector.allocateNew(numRows * 5L, numRows); output.add(charVector); } eval.evaluate(batch, output); @@ -2427,7 +2432,7 @@ public void testCastVarcharFromInteger() throws Exception { for (int j = 0; j < numRows; j++) { assertFalse(charVector.isNull(j)); - assertEquals(expValues[j], new String(charVector.get(j))); + assertEquals(expValues[j], new String(charVector.get(j), StandardCharsets.UTF_8)); } } @@ -2436,6 +2441,7 @@ public void testCastVarcharFromInteger() throws Exception { } @Test + @SuppressWarnings("FloatingPointLiteralPrecision") public void testCastVarcharFromFloat() throws Exception { Field inField = Field.nullable("input", float64); Field lenField = Field.nullable("outLength", int64); @@ -2523,7 +2529,7 @@ public void testCastVarcharFromFloat() throws Exception { for (int i = 0; i < exprs.size(); i++) { VarCharVector charVector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator); - charVector.allocateNew(numRows * 5, numRows); + charVector.allocateNew(numRows * 5L, numRows); output.add(charVector); } eval.evaluate(batch, output); @@ -2534,7 +2540,7 @@ public void testCastVarcharFromFloat() throws Exception { for (int j = 0; j < numRows; j++) { assertFalse(charVector.isNull(j)); - assertEquals(expValues[j], new String(charVector.get(j))); + assertEquals(expValues[j], new String(charVector.get(j), StandardCharsets.UTF_8)); } } @@ -2587,7 +2593,7 @@ public void testInitCap() throws Exception { // allocate data for output vector. VarCharVector outVector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator); - outVector.allocateNew(numRows * 100, numRows); + outVector.allocateNew(numRows * 100L, numRows); // evaluate expression List output = new ArrayList<>(); @@ -2598,7 +2604,7 @@ public void testInitCap() throws Exception { // match expected output. for (int i = 0; i < numRows - 1; i++) { assertFalse(outVector.isNull(i), "Expect none value equals null"); - assertEquals(expected[i], new String(outVector.get(i))); + assertEquals(expected[i], new String(outVector.get(i), StandardCharsets.UTF_8)); } assertTrue(outVector.isNull(numRows - 1), "Last value must be null"); diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/expression/TreeBuilderTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/expression/TreeBuilderTest.java index 63b31f1fb30d8..0a067f3eeb3ce 100644 --- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/expression/TreeBuilderTest.java +++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/expression/TreeBuilderTest.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -39,20 +40,20 @@ public void testMakeLiteral() throws GandivaException { assertEquals(true, node.getBooleanNode().getValue()); - n = TreeBuilder.makeLiteral(new Integer(10)); + n = TreeBuilder.makeLiteral(10); node = n.toProtobuf(); assertEquals(10, node.getIntNode().getValue()); - n = TreeBuilder.makeLiteral(new Long(50)); + n = TreeBuilder.makeLiteral(Long.valueOf(50)); node = n.toProtobuf(); assertEquals(50, node.getLongNode().getValue()); - Float f = new Float(2.5); + Float f = (float) 2.5; n = TreeBuilder.makeLiteral(f); node = n.toProtobuf(); assertEquals(f.floatValue(), node.getFloatNode().getValue(), 0.1); - Double d = new Double(3.3); + Double d = 3.3; n = TreeBuilder.makeLiteral(d); node = n.toProtobuf(); assertEquals(d.doubleValue(), node.getDoubleNode().getValue(), 0.1); @@ -60,9 +61,10 @@ public void testMakeLiteral() throws GandivaException { String s = new String("hello"); n = TreeBuilder.makeStringLiteral(s); node = n.toProtobuf(); - assertArrayEquals(s.getBytes(), node.getStringNode().getValue().toByteArray()); + assertArrayEquals( + s.getBytes(StandardCharsets.UTF_8), node.getStringNode().getValue().toByteArray()); - byte[] b = new String("hello").getBytes(); + byte[] b = new String("hello").getBytes(StandardCharsets.UTF_8); n = TreeBuilder.makeBinaryLiteral(b); node = n.toProtobuf(); assertArrayEquals(b, node.getBinaryNode().getValue().toByteArray()); diff --git a/java/performance/src/main/java/org/apache/arrow/adapter/jdbc/JdbcAdapterBenchmarks.java b/java/performance/src/main/java/org/apache/arrow/adapter/jdbc/JdbcAdapterBenchmarks.java index a2eb9674e4d37..ff2214ceb417a 100644 --- a/java/performance/src/main/java/org/apache/arrow/adapter/jdbc/JdbcAdapterBenchmarks.java +++ b/java/performance/src/main/java/org/apache/arrow/adapter/jdbc/JdbcAdapterBenchmarks.java @@ -154,13 +154,13 @@ public static class ConsumeState { private JdbcConsumer bitConsumer; - private JdbcToArrowConfig config; + // private JdbcToArrowConfig config; @Setup(Level.Trial) public void prepare() throws Exception { allocator = new RootAllocator(Integer.MAX_VALUE); - config = - new JdbcToArrowConfigBuilder().setAllocator(allocator).setTargetBatchSize(1024).build(); + // config = + // new JdbcToArrowConfigBuilder().setAllocator(allocator).setTargetBatchSize(1024).build(); Class.forName(DRIVER); conn = DriverManager.getConnection(URL); diff --git a/java/performance/src/main/java/org/apache/arrow/vector/BaseValueVectorBenchmarks.java b/java/performance/src/main/java/org/apache/arrow/vector/BaseValueVectorBenchmarks.java index bd53b13823622..040b320bec1a9 100644 --- a/java/performance/src/main/java/org/apache/arrow/vector/BaseValueVectorBenchmarks.java +++ b/java/performance/src/main/java/org/apache/arrow/vector/BaseValueVectorBenchmarks.java @@ -70,7 +70,7 @@ public void tearDown() { public int testComputeCombinedBufferSize() { int totalSize = 0; for (int i = 0; i < VECTOR_LENGTH; i++) { - totalSize += vector.computeCombinedBufferSize(i, 4); + totalSize = (int) (totalSize + vector.computeCombinedBufferSize(i, 4)); } return totalSize; } diff --git a/java/performance/src/main/java/org/apache/arrow/vector/VarCharBenchmarks.java b/java/performance/src/main/java/org/apache/arrow/vector/VarCharBenchmarks.java index b3924a7dd9461..5dd13178e701d 100644 --- a/java/performance/src/main/java/org/apache/arrow/vector/VarCharBenchmarks.java +++ b/java/performance/src/main/java/org/apache/arrow/vector/VarCharBenchmarks.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeUnit; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -61,7 +62,7 @@ public void prepare() { if (i % 3 == 0) { fromVector.setNull(i); } else { - fromVector.set(i, String.valueOf(i * 1000).getBytes()); + fromVector.set(i, String.valueOf(i * 1000).getBytes(StandardCharsets.UTF_8)); } } fromVector.setValueCount(VECTOR_LENGTH); diff --git a/java/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarks.java b/java/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarks.java index 0bce6569d268f..42e49b3cda127 100644 --- a/java/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarks.java +++ b/java/performance/src/main/java/org/apache/arrow/vector/VariableWidthVectorBenchmarks.java @@ -16,6 +16,7 @@ */ package org.apache.arrow.vector; +import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeUnit; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; @@ -45,7 +46,8 @@ public class VariableWidthVectorBenchmarks { private static final int ALLOCATOR_CAPACITY = 1024 * 1024; - private static byte[] bytes = VariableWidthVectorBenchmarks.class.getName().getBytes(); + private static byte[] bytes = + VariableWidthVectorBenchmarks.class.getName().getBytes(StandardCharsets.UTF_8); private ArrowBuf arrowBuff; private BufferAllocator allocator; diff --git a/java/performance/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoderBenchmarks.java b/java/performance/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoderBenchmarks.java index c24399dbca388..233604aa3cb4f 100644 --- a/java/performance/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoderBenchmarks.java +++ b/java/performance/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoderBenchmarks.java @@ -117,7 +117,7 @@ private int generateRandomIndex(int max) { private String generateUniqueKey(int length) { String str = "abcdefghijklmnopqrstuvwxyz"; Random random = new Random(); - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); for (int i = 0; i < length; i++) { int number = random.nextInt(26); sb.append(str.charAt(number)); From ceaf567c91d3427465ffdf21f68d9ae38cf00b85 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Sun, 15 Sep 2024 13:08:55 +0530 Subject: [PATCH 11/13] fix: adding splitter --- java/adapter/jdbc/pom.xml | 6 ++++++ .../apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java | 9 ++++----- java/pom.xml | 6 ++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 099798a95cd25..377dc00f73838 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -84,6 +84,12 @@ under the License. jackson-annotations + + org.apache.commons + commons-lang3 + test + + org.assertj assertj-core diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 74b1ca34d73c6..4abb99d4fcc23 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -58,6 +58,7 @@ import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.ObjectMapperFactory; import org.apache.arrow.vector.util.Text; +import org.apache.commons.lang3.StringUtils; /** * This is a Helper class which has functionalities to read and assert the values from the given @@ -447,16 +448,15 @@ public static byte[][] getBinaryValues(String[] values, String dataType) { return valueArr; } - @SuppressWarnings("StringSplitter") public static String[] getValues(String[] values, String dataType) { String value = ""; for (String val : values) { if (val.startsWith(dataType)) { - value = val.split("=")[1]; + value = StringUtils.split(val, "=")[1]; break; } } - return value.split(","); + return StringUtils.split(value, ","); } public static Integer[][] getListValues(String[] values, String dataType) { @@ -464,7 +464,6 @@ public static Integer[][] getListValues(String[] values, String dataType) { return getListValues(dataArr); } - @SuppressWarnings("StringSplitter") public static Integer[][] getListValues(String[] dataArr) { Integer[][] valueArr = new Integer[dataArr.length][]; int i = 0; @@ -474,7 +473,7 @@ public static Integer[][] getListValues(String[] dataArr) { } else if ("()".equals(data.trim())) { valueArr[i++] = new Integer[0]; } else { - String[] row = data.replace("(", "").replace(")", "").split(";"); + String[] row = StringUtils.split(data.replace("(", "").replace(")", ""), ";"); Integer[] arr = new Integer[row.length]; for (int j = 0; j < arr.length; j++) { arr[j] = "null".equals(row[j]) ? null : Integer.parseInt(row[j]); diff --git a/java/pom.xml b/java/pom.xml index 2051caf572a24..18b77929beee4 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -126,6 +126,7 @@ under the License. is addressed --> 3.2.2 + 3.12.0 @@ -232,6 +233,11 @@ under the License. logback-core ${logback.version} + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + From 5b857018bcac5b68e91be596d9b74c97cf80ebc2 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Sun, 15 Sep 2024 14:19:17 +0530 Subject: [PATCH 12/13] fix: adding lang3 for jdbc --- java/adapter/jdbc/pom.xml | 1 + java/pom.xml | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 377dc00f73838..1b93a0bf5fbc4 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -87,6 +87,7 @@ under the License. org.apache.commons commons-lang3 + ${commons.lang3.version} test diff --git a/java/pom.xml b/java/pom.xml index 18b77929beee4..84780fa8f338c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -233,11 +233,6 @@ under the License. logback-core ${logback.version} - - org.apache.commons - commons-lang3 - ${commons.lang3.version} - From 1354ad028f823e90ce763e48f01053c725b86e07 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Wed, 18 Sep 2024 08:29:21 +0530 Subject: [PATCH 13/13] fix:reverting a change temporarily --- .../arrow/flight/ServerSessionMiddleware.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java index d1f97118d02a3..b7de8fd65bc79 100644 --- a/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java +++ b/java/flight/flight-core/src/main/java/org/apache/arrow/flight/ServerSessionMiddleware.java @@ -16,10 +16,8 @@ */ package org.apache.arrow.flight; -import com.google.common.base.Splitter; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -77,6 +75,7 @@ private void closeSession(String id) { } @Override + @SuppressWarnings("StringSplitter") public ServerSessionMiddleware onCallStarted( CallInfo callInfo, CallHeaders incomingHeaders, RequestContext context) { String sessionId = null; @@ -85,15 +84,15 @@ public ServerSessionMiddleware onCallStarted( if (it != null) { findIdCookie: for (final String headerValue : it) { - for (final String cookie : Splitter.on(" ;").split(headerValue)) { - final List cookiePair = Splitter.on('=').splitToList(cookie); - if (cookiePair.size() != 2) { + for (final String cookie : headerValue.split(" ;")) { + final String[] cookiePair = cookie.split("="); + if (cookiePair.length != 2) { // Soft failure: Ignore invalid cookie list field break; } - if (sessionCookieName.equals(cookiePair.get(0)) && !cookiePair.get(1).isEmpty()) { - sessionId = cookiePair.get(1); + if (sessionCookieName.equals(cookiePair[0]) && !cookiePair[1].isEmpty()) { + sessionId = cookiePair[1]; break findIdCookie; } }