@@ -463,7 +463,7 @@ impl Stream {
463
463
}
464
464
465
465
/// Converts arrow files in staging into parquet files, does so only for past minutes when run with `!shutdown_signal`
466
- pub async fn prepare_parquet (
466
+ pub fn prepare_parquet (
467
467
& self ,
468
468
init_signal : bool ,
469
469
shutdown_signal : bool ,
@@ -637,8 +637,6 @@ impl Stream {
637
637
}
638
638
639
639
self . update_staging_metrics ( & staging_files) ;
640
-
641
- let mut record_batches = Vec :: new ( ) ;
642
640
for ( parquet_path, arrow_files) in staging_files {
643
641
let record_reader = MergedReverseRecordReader :: try_new ( & arrow_files) ;
644
642
if record_reader. readers . is_empty ( ) {
@@ -656,7 +654,6 @@ impl Stream {
656
654
& schema,
657
655
& props,
658
656
time_partition,
659
- & mut record_batches,
660
657
) ? {
661
658
continue ;
662
659
}
@@ -682,7 +679,6 @@ impl Stream {
682
679
schema : & Arc < Schema > ,
683
680
props : & WriterProperties ,
684
681
time_partition : Option < & String > ,
685
- record_batches : & mut Vec < RecordBatch > ,
686
682
) -> Result < bool , StagingError > {
687
683
let mut part_file = OpenOptions :: new ( )
688
684
. create ( true )
@@ -692,8 +688,6 @@ impl Stream {
692
688
let mut writer = ArrowWriter :: try_new ( & mut part_file, schema. clone ( ) , Some ( props. clone ( ) ) ) ?;
693
689
for ref record in record_reader. merged_iter ( schema. clone ( ) , time_partition. cloned ( ) ) {
694
690
writer. write ( record) ?;
695
- // Collect record batches for finding statistics later
696
- record_batches. push ( record. clone ( ) ) ;
697
691
}
698
692
writer. close ( ) ?;
699
693
@@ -960,7 +954,7 @@ impl Stream {
960
954
}
961
955
962
956
/// First flushes arrows onto disk and then converts the arrow into parquet
963
- pub async fn flush_and_convert (
957
+ pub fn flush_and_convert (
964
958
& self ,
965
959
init_signal : bool ,
966
960
shutdown_signal : bool ,
@@ -975,7 +969,7 @@ impl Stream {
975
969
976
970
let start_convert = Instant :: now ( ) ;
977
971
978
- self . prepare_parquet ( init_signal, shutdown_signal) . await ?;
972
+ self . prepare_parquet ( init_signal, shutdown_signal) ?;
979
973
trace ! (
980
974
"Converting arrows to parquet on stream ({}) took: {}s" ,
981
975
self . stream_name,
@@ -1070,8 +1064,7 @@ impl Streams {
1070
1064
. map ( Arc :: clone)
1071
1065
. collect ( ) ;
1072
1066
for stream in streams {
1073
- joinset
1074
- . spawn ( async move { stream. flush_and_convert ( init_signal, shutdown_signal) . await } ) ;
1067
+ joinset. spawn ( async move { stream. flush_and_convert ( init_signal, shutdown_signal) } ) ;
1075
1068
}
1076
1069
}
1077
1070
}
0 commit comments