Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PARQUET-2464: Fix dictionaryPageOffset flag setting in toParquetMetadata method #1340

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

PARQUET-2464: Fix dictionaryPageOffset flag setting in toParquetMetadata method #1340

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
061ba3d
update test and fix
May 2, 2024
f71202f
fix styling error
May 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions ...et-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,8 +556,7 @@ private void addRowGroup(
columnMetaData.getTotalUncompressedSize(),
columnMetaData.getTotalSize(),
columnMetaData.getFirstDataPageOffset());
if (columnMetaData.getEncodingStats() != null
&& columnMetaData.getEncodingStats().hasDictionaryPages()) {
if (columnMetaData.hasDictionaryPage()) {
metaData.setDictionary_page_offset(columnMetaData.getDictionaryPageOffset());
}
long bloomFilterOffset = columnMetaData.getBloomFilterOffset();
Expand Down
45 changes: 40 additions & 5 deletions ...adoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,32 @@ public void testParquetMetadataConverterWithDictionary() throws IOException {
Assert.assertEquals(dicOffsetOriginal, dicOffsetConverted);
}

@Test
public void testParquetMetadataConverterWithDictionaryWithoutStats() throws IOException {
ParquetMetadata parquetMetaData = createParquetMetaData(Encoding.PLAIN_DICTIONARY, Encoding.PLAIN, false);

ParquetMetadataConverter converter = new ParquetMetadataConverter();
FileMetaData fmd1 = converter.toParquetMetadata(1, parquetMetaData);

// Flag should be true
fmd1.row_groups.forEach(rowGroup -> rowGroup.columns.forEach(column -> {
assertTrue(column.meta_data.isSetDictionary_page_offset());
}));

ByteArrayOutputStream metaDataOutputStream = new ByteArrayOutputStream();
Util.writeFileMetaData(fmd1, metaDataOutputStream);
ByteArrayInputStream metaDataInputStream = new ByteArrayInputStream(metaDataOutputStream.toByteArray());
FileMetaData fmd2 = Util.readFileMetaData(metaDataInputStream);
ParquetMetadata parquetMetaDataConverted = converter.fromParquetMetadata(fmd2);

long dicOffsetOriginal =
parquetMetaData.getBlocks().get(0).getColumns().get(0).getDictionaryPageOffset();
long dicOffsetConverted =
parquetMetaDataConverted.getBlocks().get(0).getColumns().get(0).getDictionaryPageOffset();

Assert.assertEquals(dicOffsetOriginal, dicOffsetConverted);
}

@Test
public void testParquetMetadataConverterWithoutDictionary() throws IOException {
ParquetMetadata parquetMetaData = createParquetMetaData(null, Encoding.PLAIN);
Expand Down Expand Up @@ -1248,17 +1274,26 @@ private static Statistics<?> createStatsTyped(PrimitiveType type, BigInteger min
}

private static ParquetMetadata createParquetMetaData(Encoding dicEncoding, Encoding dataEncoding) {
return createParquetMetaData(dicEncoding, dataEncoding, true);
}


private static ParquetMetadata createParquetMetaData(Encoding dicEncoding, Encoding dataEncoding,
boolean includeDicStats) {
MessageType schema = parseMessageType("message schema { optional int32 col (INT_32); }");
org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData =
new org.apache.parquet.hadoop.metadata.FileMetaData(schema, new HashMap<String, String>(), null);
List<BlockMetaData> blockMetaDataList = new ArrayList<BlockMetaData>();
BlockMetaData blockMetaData = new BlockMetaData();
EncodingStats.Builder builder = new EncodingStats.Builder();
if (dicEncoding != null) {
builder.addDictEncoding(dicEncoding).build();
EncodingStats es = null;
if (includeDicStats) {
EncodingStats.Builder builder = new EncodingStats.Builder();
if (dicEncoding != null) {
builder.addDictEncoding(dicEncoding).build();
}
builder.addDataEncoding(dataEncoding);
es = builder.build();
}
builder.addDataEncoding(dataEncoding);
EncodingStats es = builder.build();
Set<org.apache.parquet.column.Encoding> e = new HashSet<org.apache.parquet.column.Encoding>();
PrimitiveTypeName t = PrimitiveTypeName.INT32;
ColumnPath p = ColumnPath.get("col");
Expand Down
Loading