From 2c59e26211044e1adc7c2d50c498c81f303d1993 Mon Sep 17 00:00:00 2001 From: Colin Ho Date: Fri, 11 Oct 2024 10:14:43 -0700 Subject: [PATCH] cleanup --- src/daft-csv/src/read.rs | 29 ++++++++--------- src/daft-json/src/read.rs | 32 ++++++++----------- src/daft-micropartition/src/micropartition.rs | 1 - 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/daft-csv/src/read.rs b/src/daft-csv/src/read.rs index 5216d29f39..33a1c6cd8c 100644 --- a/src/daft-csv/src/read.rs +++ b/src/daft-csv/src/read.rs @@ -310,23 +310,20 @@ async fn read_csv_single_into_table( .into_iter() .collect::>>()?; // Handle empty table case. - let output_table = { - if collected_tables.is_empty() { - return Table::empty(Some(schema)); - } + if collected_tables.is_empty() { + return Table::empty(Some(schema)); + } - // // TODO(Clark): Don't concatenate all chunks from a file into a single table, since MicroPartition is natively chunked. - let concated_table = tables_concat(collected_tables)?; - if let Some(limit) = limit - && concated_table.len() > limit - { - // apply head in case that last chunk went over limit - concated_table.head(limit) - } else { - Ok(concated_table) - } - }?; - Ok(output_table) + // // TODO(Clark): Don't concatenate all chunks from a file into a single table, since MicroPartition is natively chunked. + let concated_table = tables_concat(collected_tables)?; + if let Some(limit) = limit + && concated_table.len() > limit + { + // apply head in case that last chunk went over limit + concated_table.head(limit) + } else { + Ok(concated_table) + } } async fn stream_csv_single( diff --git a/src/daft-json/src/read.rs b/src/daft-json/src/read.rs index de7b153b90..133c49c31f 100644 --- a/src/daft-json/src/read.rs +++ b/src/daft-json/src/read.rs @@ -271,24 +271,20 @@ async fn read_json_single_into_table( .into_iter() .collect::>>()?; // Handle empty table case. - let output_table = { - if collected_tables.is_empty() { - let daft_schema = Arc::new(Schema::try_from(&schema)?); - return Table::empty(Some(daft_schema)); - } - - // // TODO(Clark): Don't concatenate all chunks from a file into a single table, since MicroPartition is natively chunked. - let concated_table = tables_concat(collected_tables)?; - if let Some(limit) = limit - && concated_table.len() > limit - { - // apply head in case that last chunk went over limit - concated_table.head(limit) - } else { - Ok(concated_table) - } - }?; - Ok(output_table) + if collected_tables.is_empty() { + let daft_schema = Arc::new(Schema::try_from(&schema)?); + return Table::empty(Some(daft_schema)); + } + // // TODO(Clark): Don't concatenate all chunks from a file into a single table, since MicroPartition is natively chunked. + let concated_table = tables_concat(collected_tables)?; + if let Some(limit) = limit + && concated_table.len() > limit + { + // apply head in case that last chunk went over limit + concated_table.head(limit) + } else { + Ok(concated_table) + } } pub async fn stream_json( diff --git a/src/daft-micropartition/src/micropartition.rs b/src/daft-micropartition/src/micropartition.rs index 68224d85cf..2d1ae4f0e1 100644 --- a/src/daft-micropartition/src/micropartition.rs +++ b/src/daft-micropartition/src/micropartition.rs @@ -495,7 +495,6 @@ fn materialize_scan_task( // If there is a partition spec and partition values aren't duplicated in the data, inline the partition values // into the table when casting the schema. let fill_map = scan_task.partition_spec().map(|pspec| pspec.to_fill_map()); - println!("fill_map: {:?}", fill_map); table_values = table_values .iter() .map(|tbl| tbl.cast_to_schema_with_fill(cast_to_schema.as_ref(), fill_map.as_ref()))