From cf309e6d054e59f2d613eb74bdcb83d4000a392d Mon Sep 17 00:00:00 2001 From: Shefeek Jinnah Date: Mon, 16 Nov 2026 11:01:41 +0530 Subject: [PATCH 1/2] perf(reader): read a TopK's best files first DataFusion's PushdownSort marks the parquet source under a TopK so its shared work queue hands out files in sort order (newest first for ORDER BY ts DESC LIMIT n), and the TopK's dynamic filter then skips the rest before they are opened. LocalModeOptimizer runs after that rule and rebuilds the scan with LiquidParquetSource::from_parquet_source, which cannot copy the mark (it is private to ParquetSource), and LiquidParquetSource implemented neither try_pushdown_sort nor reorder_files. Liquid scans therefore handed files out in listing order. On a 30-day span table, "newest timestamp in a day" opened 22 to 28 of the day's 62 files, about 18 s on a first read from S3. - LiquidParquetSource: try_pushdown_sort answers Inexact when the leading sort key is a plain file column and keeps the order; reorder_files sorts the queue by each file's min statistic in the key's direction (a copy of DataFusion's crate-private helper). The plan shows sort_order_for_reorder. - LocalModeOptimizer: after the rewrite, offer each TopK's sort to its input again and swap in the Inexact input with replace_children. Rerunning PushdownSort instead would rebuild the SortExec with a new dynamic filter while the scan keeps the old one, so nothing would be pruned; two datafusion-local snapshot tests caught that. Row groups inside a file are still read in file order. --- src/datafusion/src/optimizers/mod.rs | 325 ++++++++++++++++++- src/datafusion/src/reader/plantime/source.rs | 254 ++++++++++++++- 2 files changed, 575 insertions(+), 4 deletions(-) diff --git a/src/datafusion/src/optimizers/mod.rs b/src/datafusion/src/optimizers/mod.rs index 7daebe62..e6175901 100644 --- a/src/datafusion/src/optimizers/mod.rs +++ b/src/datafusion/src/optimizers/mod.rs @@ -22,7 +22,10 @@ use datafusion::{ }, physical_expr::{PhysicalExpr, projection::ProjectionExprs, utils::collect_columns}, physical_optimizer::{PhysicalOptimizerRule, pruning::PruningPredicateBuilder}, - physical_plan::ExecutionPlan, + physical_plan::{ + ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions, SortOrderPushdownResult, + sorts::sort::SortExec, + }, }; pub(crate) use lineage::HintAnalyzer; @@ -144,7 +147,7 @@ impl PhysicalOptimizerRule for LocalModeOptimizer { fn optimize( &self, plan: Arc, - _config: &ConfigOptions, + config: &ConfigOptions, ) -> Result, datafusion::error::DataFusionError> { let analysis = HintAnalyzer::analyze(&plan); let cache = self.cache.clone(); @@ -170,7 +173,12 @@ impl PhysicalOptimizerRule for LocalModeOptimizer { } convert_parquet_scan(node, &cache, hints, prefetch) }; - Ok(lineage::rewrite_with_hints(plan, &mut convert, &analysis)) + let plan = lineage::rewrite_with_hints(plan, &mut convert, &analysis); + if config.optimizer.enable_sort_pushdown { + push_topk_sorts_into_scans(plan) + } else { + Ok(plan) + } } fn name(&self) -> &str { @@ -182,6 +190,55 @@ impl PhysicalOptimizerRule for LocalModeOptimizer { } } +/// Offer each TopK's sort to its input again, now that the scans below it are +/// liquid, and take the input back only when it answers `Inexact`. +/// +/// [`LocalModeOptimizer`] runs after DataFusion's `PushdownSort`, which already +/// marked the *parquet* sources under a TopK to approximate its sort. +/// `ParquetSource` keeps that mark private, so the liquid source that replaced +/// it starts without one, and its files would be handed out in listing order: +/// for `ORDER BY ts DESC LIMIT n` that reaches the newest file last and opens +/// most of the others first. +/// +/// This is not a second run of `PushdownSort`, on purpose. That rule rebuilds +/// the `SortExec` on an `Inexact` answer, and a rebuilt TopK creates a new +/// dynamic filter, while the scan below still holds the old one, which +/// DataFusion's filter pushdown attached before this rule ran. Nothing would +/// update the scan's filter again, so the scan would prune nothing. Here the +/// `SortExec` is kept and only its input is replaced (`replace_children` keeps +/// the filter expression), so the scan's filter stays the TopK's. +/// +/// Only a sort with a fetch (a TopK) is offered: without a limit every row is +/// read anyway, so the order of the files changes nothing. `Exact` and +/// `Unsupported` answers leave the plan as it is; an `Exact` input was already +/// handled by DataFusion's own pass, which removed its `SortExec`. +fn push_topk_sorts_into_scans( + plan: Arc, +) -> Result, datafusion::error::DataFusionError> { + plan.transform_down(|node| { + let Some(sort) = node.downcast_ref::() else { + return Ok(Transformed::no(node)); + }; + if sort.fetch().is_none() { + return Ok(Transformed::no(node)); + } + match sort.input().try_pushdown_sort(sort.expr())? { + SortOrderPushdownResult::Inexact { inner } => { + // Recompute: an `Inexact` scan drops its declared ordering. + let node = Arc::clone(&node).replace_children( + vec![inner], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + )?; + Ok(Transformed::new(node, true, TreeNodeRecursion::Jump)) + } + SortOrderPushdownResult::Exact { .. } | SortOrderPushdownResult::Unsupported => { + Ok(Transformed::no(node)) + } + } + }) + .map(|t| t.data) +} + /// Rewrite the data source plan to use liquid cache, attaching `hints` (keyed by /// file-schema column name) to every parquet scan it rewrites. /// @@ -1186,6 +1243,268 @@ mod tests { "a scan reading a virtual column must stay on ParquetSource" ); } + + /// Four files of one column `ts`, each holding a disjoint range of 100 + /// values, named so that listing (path) order is oldest first: + /// `a` 0..99, `b` 100..199, `c` 200..299, `d` 300..399. That is how a + /// time-partitioned table lists its files, and it is the order in which a + /// newest-first TopK reaches the file it needs last. + fn write_time_sliced_files(dir: &Path) { + let schema = Arc::new(Schema::new(vec![Field::new("ts", DataType::Int32, false)])); + for (i, name) in ["a", "b", "c", "d"].iter().enumerate() { + let start = i as i32 * 100; + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(start..start + 100))], + ) + .unwrap(); + let file = File::create(dir.join(format!("{name}.parquet"))).unwrap(); + let mut writer = ArrowWriter::try_new(file, schema.clone(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + } + } + + /// Plan `sql` over the files of [`write_time_sliced_files`] as table `t`, + /// split across two file groups the way a multi-partition scan is, then run + /// [`LocalModeOptimizer`] on it as the last rule, as a session does. + async fn plan_time_sliced( + dir: &Path, + sql: &str, + enable_sort_pushdown: bool, + ) -> (Arc, SessionContext) { + let mut config = SessionConfig::new() + .with_target_partitions(2) + .with_collect_statistics(true); + config.options_mut().optimizer.repartition_file_min_size = 0; + config.options_mut().optimizer.enable_sort_pushdown = enable_sort_pushdown; + let ctx = SessionContext::new_with_config(config); + ctx.register_parquet("t", dir.to_str().unwrap(), Default::default()) + .await + .unwrap(); + let plan = ctx + .sql(sql) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + let cache = make_cache(dir).await; + let plan = LocalModeOptimizer::new(cache) + .with_prefetch(false) + .optimize(plan, ctx.state().config_options()) + .unwrap(); + (plan, ctx) + } + + /// The single liquid scan's config in `plan`. + fn liquid_scan(plan: &Arc) -> FileScanConfig { + let mut found = None; + plan.apply(|node| { + if let Some(exec) = node.downcast_ref::() + && let Some(cfg) = exec.data_source().downcast_ref::() + && cfg + .file_source() + .downcast_ref::() + .is_some() + { + found = Some(cfg.clone()); + return Ok(TreeNodeRecursion::Stop); + } + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(); + found.expect("plan has a liquid scan") + } + + /// File names in the order the scan's shared work queue hands them out: + /// the file groups concatenated, then the file source's `reorder_files`, + /// which is what `SharedWorkSource::from_config` does. + fn work_queue_order(cfg: &FileScanConfig) -> Vec { + let files: Vec = cfg + .file_groups + .iter() + .flat_map(|group| group.iter().cloned()) + .collect(); + cfg.file_source() + .reorder_files(files) + .iter() + .map(|f| { + f.object_meta + .location + .filename() + .unwrap() + .trim_end_matches(".parquet") + .to_string() + }) + .collect() + } + + /// Under `ORDER BY ts DESC LIMIT 1` the liquid scan must keep the sort that + /// DataFusion pushed into the parquet scan it replaced, and hand its files + /// out newest first across all file groups. Without that the queue is the + /// groups one after another (each newest first inside, so `b a d c` here), + /// and the file holding the answer comes after files the TopK's dynamic + /// filter could have skipped: on a 30-day span table that opened 22 to 28 + /// of a day's 62 files for one timestamp. + #[tokio::test] + async fn liquid_scan_hands_out_files_newest_first_under_a_topk() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, ctx) = plan_time_sliced( + dir.path(), + "SELECT ts FROM t ORDER BY ts DESC LIMIT 1", + true, + ) + .await; + + let cfg = liquid_scan(&plan); + assert!( + cfg.file_groups.len() >= 2, + "the case under test spans several file groups: {:?}", + cfg.file_groups + ); + let source = cfg + .file_source() + .downcast_ref::() + .unwrap(); + assert!( + source.sort_order_for_reorder().is_some(), + "the liquid scan lost the pushed sort" + ); + assert_eq!(work_queue_order(&cfg), ["d", "c", "b", "a"]); + let displayed = DisplayableExecutionPlan::new(plan.as_ref()) + .indent(true) + .to_string(); + assert!( + displayed.contains("sort_order_for_reorder=[ts@0 DESC"), + "{displayed}" + ); + + // The reorder only changes which file is read first: the answer is the same. + let batches = collect(plan, ctx.task_ctx()).await.unwrap(); + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, 1); + let ts = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ts.value(0), 399); + } + + /// The point of the order: once the newest files are read, the TopK's + /// dynamic filter rules out the rest before they are opened. With two + /// workers taking `d` and `c` first, `b` and `a` must be skipped, so at + /// most two of the four files are opened. Read in listing order (`b a d c` + /// across the two groups) three or four are; and a TopK whose filter is not + /// the one the scan holds (what rebuilding the `SortExec` would cause) + /// opens all four. + #[tokio::test] + async fn liquid_topk_skips_the_older_files() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, ctx) = plan_time_sliced( + dir.path(), + "SELECT ts FROM t ORDER BY ts DESC LIMIT 1", + true, + ) + .await; + let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await.unwrap(); + let ts = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ts.value(0), 399); + + let metrics = DisplayableExecutionPlan::with_metrics(plan.as_ref()) + .indent(true) + .to_string(); + let pruned = metrics + .split("files_ranges_pruned_statistics=") + .nth(1) + .unwrap_or_else(|| panic!("no file pruning metric in:\n{metrics}")); + let numbers: Vec = pruned + .split(|c: char| !c.is_ascii_digit()) + .filter(|s| !s.is_empty()) + .take(2) + .map(|s| s.parse().unwrap()) + .collect(); + let (total, opened) = (numbers[0], numbers[1]); + assert_eq!(total, 4, "{metrics}"); + assert!(opened <= 2, "opened {opened} of {total} files:\n{metrics}"); + } + + /// Ascending is the mirror image: oldest file first. + #[tokio::test] + async fn liquid_scan_hands_out_files_oldest_first_for_an_ascending_topk() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, _ctx) = + plan_time_sliced(dir.path(), "SELECT ts FROM t ORDER BY ts ASC LIMIT 1", true).await; + assert_eq!(work_queue_order(&liquid_scan(&plan)), ["a", "b", "c", "d"]); + } + + /// With sort pushdown turned off the rule pushes nothing either: the liquid + /// scan keeps no sort and the queue keeps listing order. + #[tokio::test] + async fn liquid_scan_keeps_listing_order_when_sort_pushdown_is_off() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, _ctx) = plan_time_sliced( + dir.path(), + "SELECT ts FROM t ORDER BY ts DESC LIMIT 1", + false, + ) + .await; + let cfg = liquid_scan(&plan); + let source = cfg + .file_source() + .downcast_ref::() + .unwrap(); + assert!(source.sort_order_for_reorder().is_none()); + assert_eq!(work_queue_order(&cfg), ["a", "b", "c", "d"]); + } + + /// A scan with no sort above it is left as it was. + #[tokio::test] + async fn liquid_scan_without_a_sort_keeps_no_sort_order() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, _ctx) = + plan_time_sliced(dir.path(), "SELECT ts FROM t WHERE ts > 150", true).await; + let cfg = liquid_scan(&plan); + let source = cfg + .file_source() + .downcast_ref::() + .unwrap(); + assert!(source.sort_order_for_reorder().is_none()); + assert_eq!(work_queue_order(&cfg), ["a", "b", "c", "d"]); + } + + /// A sort on an expression, not a plain column, cannot be ordered by file + /// statistics, so the liquid scan declines it. + #[tokio::test] + async fn liquid_scan_declines_a_sort_on_an_expression() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, ctx) = plan_time_sliced( + dir.path(), + "SELECT ts FROM t ORDER BY ts % 7 DESC LIMIT 1", + true, + ) + .await; + let source = liquid_scan(&plan) + .file_source() + .downcast_ref::() + .unwrap() + .clone(); + assert!(source.sort_order_for_reorder().is_none()); + let batches = collect(plan, ctx.task_ctx()).await.unwrap(); + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, 1); + } } /// Pure unit tests for the footprint byte-math (no cache / no t4 mount, so they diff --git a/src/datafusion/src/reader/plantime/source.rs b/src/datafusion/src/reader/plantime/source.rs index 85291e1f..18604e8c 100644 --- a/src/datafusion/src/reader/plantime/source.rs +++ b/src/datafusion/src/reader/plantime/source.rs @@ -1,6 +1,7 @@ use super::LiquidMorselizer; use crate::cache::{ColumnLineages, LiquidCacheParquetRef}; use datafusion::{ + common::ScalarValue, common::{internal_err, tree_node::TreeNodeRecursion}, config::{ConfigOptions, TableParquetOptions}, datasource::{ @@ -13,13 +14,15 @@ use datafusion::{ error::Result, physical_expr::projection::ProjectionExprs, physical_expr::utils::conjunction, + physical_expr::{EquivalenceProperties, LexOrdering, PhysicalSortExpr, expressions::Column}, physical_expr_adapter::DefaultPhysicalExprAdapterFactory, physical_plan::{ - DisplayFormatType, PhysicalExpr, apply_expression_roots, + DisplayFormatType, PhysicalExpr, SortOrderPushdownResult, apply_expression_roots, filter_pushdown::{FilterPushdownPropagation, PushedDown, PushedDownPredicate}, metrics::ExecutionPlanMetricsSet, }, }; +use datafusion_datasource::PartitionedFile; use datafusion_datasource::morsel::Morselizer; use object_store::ObjectStore; use std::{ @@ -41,6 +44,10 @@ pub struct LiquidParquetSource { span: Option>, lineages: Arc, prefetch: bool, + /// The sort a parent `SortExec` asked this scan to approximate (set by + /// [`FileSource::try_pushdown_sort`]). Drives [`FileSource::reorder_files`], + /// which orders the shared work queue so a TopK sees its best files first. + sort_order_for_reorder: Option, } impl LiquidParquetSource { @@ -119,6 +126,10 @@ impl LiquidParquetSource { span: None, lineages: Arc::default(), prefetch: true, + // A `ParquetSource` keeps its own pushed sort private, so it cannot + // be carried over here. `LocalModeOptimizer` pushes the sorts down + // again once the scans are liquid. + sort_order_for_reorder: None, }; if let Some(predicate) = predicate { @@ -132,6 +143,76 @@ impl LiquidParquetSource { pub fn predicate(&self) -> Option> { self.predicate.clone() } + + /// The sort this scan was asked to approximate, if any. + pub fn sort_order_for_reorder(&self) -> Option<&LexOrdering> { + self.sort_order_for_reorder.as_ref() + } +} + +/// Order `files` so the files most likely to hold a TopK's rows come first: +/// by the per-file `min` of each leading plain-column sort key, in the key's +/// direction (largest `min` first for a descending key). Files without that +/// statistic go last. +/// +/// This mirrors `reorder_files_by_min_statistics` in +/// `datafusion-datasource-parquet`, which is crate-private. Sharing the same +/// order with DataFusion's own reader keeps a liquid scan's TopK behaviour the +/// same as a plain parquet scan's: the files that tighten the dynamic filter +/// the most are opened first, and every file the filter then rules out is +/// skipped without being opened. +fn reorder_files_by_min_statistics( + mut files: Vec, + sort_order: Option<&LexOrdering>, + table_schema: &arrow_schema::Schema, +) -> Vec { + let Some(sort_order) = sort_order else { + return files; + }; + // The longest prefix of plain columns that resolve in the table schema; + // the leading key is required. + let mut keys: Vec<(usize, bool)> = Vec::new(); + for sort_expr in sort_order.iter() { + let Some(col) = sort_expr.expr.downcast_ref::() else { + break; + }; + match table_schema.index_of(col.name()) { + Ok(idx) => keys.push((idx, sort_expr.options.descending)), + Err(_) => break, + } + } + if keys.is_empty() { + return files; + } + + let file_min = |file: &PartitionedFile, idx: usize| -> Option { + file.statistics + .as_ref()? + .column_statistics + .get(idx)? + .min_value + .get_value() + .cloned() + }; + files.sort_by(|a, b| { + for &(idx, descending) in &keys { + let ord = match (file_min(a, idx), file_min(b, idx)) { + (Some(va), Some(vb)) => { + let cmp = va.partial_cmp(&vb).unwrap_or(std::cmp::Ordering::Equal); + if descending { cmp.reverse() } else { cmp } + } + // Files with statistics first, whatever the direction. + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + }; + if ord != std::cmp::Ordering::Equal { + return ord; + } + } + std::cmp::Ordering::Equal + }); + files } impl FileSource for LiquidParquetSource { @@ -228,6 +309,9 @@ impl FileSource for LiquidParquetSource { if let Some(predicate) = self.filter() { write!(f, ", predicate={predicate}")?; } + if let Some(sort_order) = &self.sort_order_for_reorder { + write!(f, ", sort_order_for_reorder=[{sort_order}]")?; + } Ok(()) } DisplayFormatType::TreeRender => Ok(()), @@ -274,6 +358,69 @@ impl FileSource for LiquidParquetSource { .with_updated_node(source)) } + /// Approximate a sort for a parent TopK by ordering the files it reads. + /// + /// Mirrors `ParquetSource::try_pushdown_sort`, minus the row-group work + /// this reader does not do: when the request's leading key is a plain + /// column of the file schema, answer `Inexact` and remember the request, + /// so [`Self::reorder_files`] puts the files with the best statistics at + /// the front of the shared work queue. The parent `SortExec` stays, so the + /// result is the same; the TopK's dynamic filter just tightens after the + /// first files instead of the last. + /// + /// Everything else stays `Unsupported`, which leaves today's behaviour: + /// - a request the source's own ordering already satisfies, where + /// `FileScanConfig` sorts the file groups by statistics and may drop the + /// `SortExec` (an `Inexact` answer here would forbid that); + /// - a request whose proper prefix the ordering satisfies, so the + /// `SortExec`'s prefix-aware early stop still applies (the same reason + /// `ParquetSource` declines it); + /// - a leading key that is not a plain file column, which statistics + /// cannot order. + fn try_pushdown_sort( + &self, + order: &[PhysicalSortExpr], + eq_properties: &EquivalenceProperties, + ) -> Result>> { + let Some(sort_order) = LexOrdering::new(order.iter().cloned()) else { + return Ok(SortOrderPushdownResult::Unsupported); + }; + if eq_properties.ordering_satisfy(order.iter().cloned())? { + return Ok(SortOrderPushdownResult::Unsupported); + } + for prefix_len in 1..order.len() { + if eq_properties.ordering_satisfy(order[..prefix_len].iter().cloned())? { + return Ok(SortOrderPushdownResult::Unsupported); + } + } + let leading_is_file_column = sort_order + .first() + .expr + .downcast_ref::() + .is_some_and(|col| { + self.table_schema + .file_schema() + .field_with_name(col.name()) + .is_ok() + }); + if !leading_is_file_column { + return Ok(SortOrderPushdownResult::Unsupported); + } + let mut source = self.clone(); + source.sort_order_for_reorder = Some(sort_order); + Ok(SortOrderPushdownResult::Inexact { + inner: Arc::new(source) as Arc, + }) + } + + fn reorder_files(&self, files: Vec) -> Vec { + reorder_files_by_min_statistics( + files, + self.sort_order_for_reorder.as_ref(), + self.table_schema.table_schema(), + ) + } + fn apply_expressions( &self, f: &mut dyn FnMut(&Arc) -> Result, @@ -286,3 +433,108 @@ impl FileSource for LiquidParquetSource { ) } } + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::SortOptions; + use arrow_schema::{DataType, Field, Schema}; + use datafusion::common::{ColumnStatistics, Statistics, stats::Precision}; + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{BinaryExpr, Literal}; + + fn schema() -> Schema { + Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("ts", DataType::Int64, true), + ]) + } + + /// A file whose `ts` column has `min` (`None` = no statistic). + fn file(name: &str, ts_min: Option) -> PartitionedFile { + let file = PartitionedFile::new(name, 100); + let Some(min) = ts_min else { + return file; + }; + let ts = ColumnStatistics { + min_value: Precision::Exact(ScalarValue::Int64(Some(min))), + ..ColumnStatistics::new_unknown() + }; + file.with_statistics(Arc::new(Statistics { + num_rows: Precision::Absent, + total_byte_size: Precision::Absent, + column_statistics: vec![ColumnStatistics::new_unknown(), ts], + })) + } + + fn names(files: &[PartitionedFile]) -> Vec<&str> { + files + .iter() + .map(|f| f.object_meta.location.as_ref()) + .collect() + } + + fn ts_order(descending: bool) -> LexOrdering { + LexOrdering::new(vec![PhysicalSortExpr::new( + Arc::new(Column::new("ts", 1)), + SortOptions { + descending, + nulls_first: false, + }, + )]) + .unwrap() + } + + fn files() -> Vec { + vec![ + file("mid", Some(50)), + file("none", None), + file("old", Some(10)), + file("new", Some(90)), + ] + } + + #[test] + fn descending_puts_the_largest_min_first_and_missing_stats_last() { + let order = ts_order(true); + let sorted = reorder_files_by_min_statistics(files(), Some(&order), &schema()); + assert_eq!(names(&sorted), ["new", "mid", "old", "none"]); + } + + #[test] + fn ascending_puts_the_smallest_min_first_and_missing_stats_last() { + let order = ts_order(false); + let sorted = reorder_files_by_min_statistics(files(), Some(&order), &schema()); + assert_eq!(names(&sorted), ["old", "mid", "new", "none"]); + } + + #[test] + fn no_sort_order_keeps_the_given_order() { + let sorted = reorder_files_by_min_statistics(files(), None, &schema()); + assert_eq!(names(&sorted), ["mid", "none", "old", "new"]); + } + + #[test] + fn a_leading_expression_keeps_the_given_order() { + let expr: Arc = Arc::new(BinaryExpr::new( + Arc::new(Column::new("ts", 1)), + Operator::Plus, + Arc::new(Literal::new(ScalarValue::Int64(Some(1)))), + )); + let order = + LexOrdering::new(vec![PhysicalSortExpr::new(expr, SortOptions::default())]).unwrap(); + let sorted = reorder_files_by_min_statistics(files(), Some(&order), &schema()); + assert_eq!(names(&sorted), ["mid", "none", "old", "new"]); + } + + #[test] + fn a_column_missing_from_the_schema_keeps_the_given_order() { + let order = LexOrdering::new(vec![PhysicalSortExpr::new( + Arc::new(Column::new("gone", 5)), + SortOptions::default(), + )]) + .unwrap(); + let sorted = reorder_files_by_min_statistics(files(), Some(&order), &schema()); + assert_eq!(names(&sorted), ["mid", "none", "old", "new"]); + } +} From eee39f04d13df48d102d6c3e7aaa01e7f4b77186 Mon Sep 17 00:00:00 2001 From: Shefeek Jinnah Date: Mon, 16 Nov 2026 11:13:42 +0530 Subject: [PATCH 2/2] fix(bench): run tpcds_gen on Python 3.14 argparse on Python 3.14 refuses a type that is not callable, and `Optional[Path]` is not, so the TPC-DS job failed while generating its data (main too, since the runners moved to 3.14). `type=Path` with the existing `default=None` keeps the option optional. --- benchmark/tpcds/tpcds_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/tpcds/tpcds_gen.py b/benchmark/tpcds/tpcds_gen.py index 8b745719..d25d4f31 100644 --- a/benchmark/tpcds/tpcds_gen.py +++ b/benchmark/tpcds/tpcds_gen.py @@ -27,7 +27,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--data-dir", type=Path, default=Path("benchmark/tpcds/data"), help="Output root for tables") parser.add_argument("--answers-dir", type=Path, help="Output root for answer Parquet files (optional)") parser.add_argument("--queries-dir", type=Path, default=Path("benchmark/tpcds/queries"), help="Directory containing qNN.sql files") - parser.add_argument("--manifest", type=Optional[Path], default=None, help="Path to write manifest.json") + parser.add_argument("--manifest", type=Path, default=None, help="Path to write manifest.json") parser.add_argument("--duckdb", type=Path, help="Optional path to DuckDB executable") parser.add_argument("--seed", type=int, help="Optional seed if DuckDB supports deterministic dsdgen") parser.add_argument("--overwrite", action="store_true", help="Allow overwriting existing Parquet/manifest files")