diff --git a/benchmark/tpcds/tpcds_gen.py b/benchmark/tpcds/tpcds_gen.py index 8b745719..d25d4f31 100644 --- a/benchmark/tpcds/tpcds_gen.py +++ b/benchmark/tpcds/tpcds_gen.py @@ -27,7 +27,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--data-dir", type=Path, default=Path("benchmark/tpcds/data"), help="Output root for tables") parser.add_argument("--answers-dir", type=Path, help="Output root for answer Parquet files (optional)") parser.add_argument("--queries-dir", type=Path, default=Path("benchmark/tpcds/queries"), help="Directory containing qNN.sql files") - parser.add_argument("--manifest", type=Optional[Path], default=None, help="Path to write manifest.json") + parser.add_argument("--manifest", type=Path, default=None, help="Path to write manifest.json") parser.add_argument("--duckdb", type=Path, help="Optional path to DuckDB executable") parser.add_argument("--seed", type=int, help="Optional seed if DuckDB supports deterministic dsdgen") parser.add_argument("--overwrite", action="store_true", help="Allow overwriting existing Parquet/manifest files") diff --git a/src/datafusion/src/optimizers/mod.rs b/src/datafusion/src/optimizers/mod.rs index 7daebe62..e6175901 100644 --- a/src/datafusion/src/optimizers/mod.rs +++ b/src/datafusion/src/optimizers/mod.rs @@ -22,7 +22,10 @@ use datafusion::{ }, physical_expr::{PhysicalExpr, projection::ProjectionExprs, utils::collect_columns}, physical_optimizer::{PhysicalOptimizerRule, pruning::PruningPredicateBuilder}, - physical_plan::ExecutionPlan, + physical_plan::{ + ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions, SortOrderPushdownResult, + sorts::sort::SortExec, + }, }; pub(crate) use lineage::HintAnalyzer; @@ -144,7 +147,7 @@ impl PhysicalOptimizerRule for LocalModeOptimizer { fn optimize( &self, plan: Arc, - _config: &ConfigOptions, + config: &ConfigOptions, ) -> Result, datafusion::error::DataFusionError> { let analysis = HintAnalyzer::analyze(&plan); let cache = self.cache.clone(); @@ -170,7 +173,12 @@ impl PhysicalOptimizerRule for LocalModeOptimizer { } convert_parquet_scan(node, &cache, hints, prefetch) }; - Ok(lineage::rewrite_with_hints(plan, &mut convert, &analysis)) + let plan = lineage::rewrite_with_hints(plan, &mut convert, &analysis); + if config.optimizer.enable_sort_pushdown { + push_topk_sorts_into_scans(plan) + } else { + Ok(plan) + } } fn name(&self) -> &str { @@ -182,6 +190,55 @@ impl PhysicalOptimizerRule for LocalModeOptimizer { } } +/// Offer each TopK's sort to its input again, now that the scans below it are +/// liquid, and take the input back only when it answers `Inexact`. +/// +/// [`LocalModeOptimizer`] runs after DataFusion's `PushdownSort`, which already +/// marked the *parquet* sources under a TopK to approximate its sort. +/// `ParquetSource` keeps that mark private, so the liquid source that replaced +/// it starts without one, and its files would be handed out in listing order: +/// for `ORDER BY ts DESC LIMIT n` that reaches the newest file last and opens +/// most of the others first. +/// +/// This is not a second run of `PushdownSort`, on purpose. That rule rebuilds +/// the `SortExec` on an `Inexact` answer, and a rebuilt TopK creates a new +/// dynamic filter, while the scan below still holds the old one, which +/// DataFusion's filter pushdown attached before this rule ran. Nothing would +/// update the scan's filter again, so the scan would prune nothing. Here the +/// `SortExec` is kept and only its input is replaced (`replace_children` keeps +/// the filter expression), so the scan's filter stays the TopK's. +/// +/// Only a sort with a fetch (a TopK) is offered: without a limit every row is +/// read anyway, so the order of the files changes nothing. `Exact` and +/// `Unsupported` answers leave the plan as it is; an `Exact` input was already +/// handled by DataFusion's own pass, which removed its `SortExec`. +fn push_topk_sorts_into_scans( + plan: Arc, +) -> Result, datafusion::error::DataFusionError> { + plan.transform_down(|node| { + let Some(sort) = node.downcast_ref::() else { + return Ok(Transformed::no(node)); + }; + if sort.fetch().is_none() { + return Ok(Transformed::no(node)); + } + match sort.input().try_pushdown_sort(sort.expr())? { + SortOrderPushdownResult::Inexact { inner } => { + // Recompute: an `Inexact` scan drops its declared ordering. + let node = Arc::clone(&node).replace_children( + vec![inner], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + )?; + Ok(Transformed::new(node, true, TreeNodeRecursion::Jump)) + } + SortOrderPushdownResult::Exact { .. } | SortOrderPushdownResult::Unsupported => { + Ok(Transformed::no(node)) + } + } + }) + .map(|t| t.data) +} + /// Rewrite the data source plan to use liquid cache, attaching `hints` (keyed by /// file-schema column name) to every parquet scan it rewrites. /// @@ -1186,6 +1243,268 @@ mod tests { "a scan reading a virtual column must stay on ParquetSource" ); } + + /// Four files of one column `ts`, each holding a disjoint range of 100 + /// values, named so that listing (path) order is oldest first: + /// `a` 0..99, `b` 100..199, `c` 200..299, `d` 300..399. That is how a + /// time-partitioned table lists its files, and it is the order in which a + /// newest-first TopK reaches the file it needs last. + fn write_time_sliced_files(dir: &Path) { + let schema = Arc::new(Schema::new(vec![Field::new("ts", DataType::Int32, false)])); + for (i, name) in ["a", "b", "c", "d"].iter().enumerate() { + let start = i as i32 * 100; + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(start..start + 100))], + ) + .unwrap(); + let file = File::create(dir.join(format!("{name}.parquet"))).unwrap(); + let mut writer = ArrowWriter::try_new(file, schema.clone(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + } + } + + /// Plan `sql` over the files of [`write_time_sliced_files`] as table `t`, + /// split across two file groups the way a multi-partition scan is, then run + /// [`LocalModeOptimizer`] on it as the last rule, as a session does. + async fn plan_time_sliced( + dir: &Path, + sql: &str, + enable_sort_pushdown: bool, + ) -> (Arc, SessionContext) { + let mut config = SessionConfig::new() + .with_target_partitions(2) + .with_collect_statistics(true); + config.options_mut().optimizer.repartition_file_min_size = 0; + config.options_mut().optimizer.enable_sort_pushdown = enable_sort_pushdown; + let ctx = SessionContext::new_with_config(config); + ctx.register_parquet("t", dir.to_str().unwrap(), Default::default()) + .await + .unwrap(); + let plan = ctx + .sql(sql) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + let cache = make_cache(dir).await; + let plan = LocalModeOptimizer::new(cache) + .with_prefetch(false) + .optimize(plan, ctx.state().config_options()) + .unwrap(); + (plan, ctx) + } + + /// The single liquid scan's config in `plan`. + fn liquid_scan(plan: &Arc) -> FileScanConfig { + let mut found = None; + plan.apply(|node| { + if let Some(exec) = node.downcast_ref::() + && let Some(cfg) = exec.data_source().downcast_ref::() + && cfg + .file_source() + .downcast_ref::() + .is_some() + { + found = Some(cfg.clone()); + return Ok(TreeNodeRecursion::Stop); + } + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(); + found.expect("plan has a liquid scan") + } + + /// File names in the order the scan's shared work queue hands them out: + /// the file groups concatenated, then the file source's `reorder_files`, + /// which is what `SharedWorkSource::from_config` does. + fn work_queue_order(cfg: &FileScanConfig) -> Vec { + let files: Vec = cfg + .file_groups + .iter() + .flat_map(|group| group.iter().cloned()) + .collect(); + cfg.file_source() + .reorder_files(files) + .iter() + .map(|f| { + f.object_meta + .location + .filename() + .unwrap() + .trim_end_matches(".parquet") + .to_string() + }) + .collect() + } + + /// Under `ORDER BY ts DESC LIMIT 1` the liquid scan must keep the sort that + /// DataFusion pushed into the parquet scan it replaced, and hand its files + /// out newest first across all file groups. Without that the queue is the + /// groups one after another (each newest first inside, so `b a d c` here), + /// and the file holding the answer comes after files the TopK's dynamic + /// filter could have skipped: on a 30-day span table that opened 22 to 28 + /// of a day's 62 files for one timestamp. + #[tokio::test] + async fn liquid_scan_hands_out_files_newest_first_under_a_topk() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, ctx) = plan_time_sliced( + dir.path(), + "SELECT ts FROM t ORDER BY ts DESC LIMIT 1", + true, + ) + .await; + + let cfg = liquid_scan(&plan); + assert!( + cfg.file_groups.len() >= 2, + "the case under test spans several file groups: {:?}", + cfg.file_groups + ); + let source = cfg + .file_source() + .downcast_ref::() + .unwrap(); + assert!( + source.sort_order_for_reorder().is_some(), + "the liquid scan lost the pushed sort" + ); + assert_eq!(work_queue_order(&cfg), ["d", "c", "b", "a"]); + let displayed = DisplayableExecutionPlan::new(plan.as_ref()) + .indent(true) + .to_string(); + assert!( + displayed.contains("sort_order_for_reorder=[ts@0 DESC"), + "{displayed}" + ); + + // The reorder only changes which file is read first: the answer is the same. + let batches = collect(plan, ctx.task_ctx()).await.unwrap(); + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, 1); + let ts = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ts.value(0), 399); + } + + /// The point of the order: once the newest files are read, the TopK's + /// dynamic filter rules out the rest before they are opened. With two + /// workers taking `d` and `c` first, `b` and `a` must be skipped, so at + /// most two of the four files are opened. Read in listing order (`b a d c` + /// across the two groups) three or four are; and a TopK whose filter is not + /// the one the scan holds (what rebuilding the `SortExec` would cause) + /// opens all four. + #[tokio::test] + async fn liquid_topk_skips_the_older_files() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, ctx) = plan_time_sliced( + dir.path(), + "SELECT ts FROM t ORDER BY ts DESC LIMIT 1", + true, + ) + .await; + let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await.unwrap(); + let ts = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ts.value(0), 399); + + let metrics = DisplayableExecutionPlan::with_metrics(plan.as_ref()) + .indent(true) + .to_string(); + let pruned = metrics + .split("files_ranges_pruned_statistics=") + .nth(1) + .unwrap_or_else(|| panic!("no file pruning metric in:\n{metrics}")); + let numbers: Vec = pruned + .split(|c: char| !c.is_ascii_digit()) + .filter(|s| !s.is_empty()) + .take(2) + .map(|s| s.parse().unwrap()) + .collect(); + let (total, opened) = (numbers[0], numbers[1]); + assert_eq!(total, 4, "{metrics}"); + assert!(opened <= 2, "opened {opened} of {total} files:\n{metrics}"); + } + + /// Ascending is the mirror image: oldest file first. + #[tokio::test] + async fn liquid_scan_hands_out_files_oldest_first_for_an_ascending_topk() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, _ctx) = + plan_time_sliced(dir.path(), "SELECT ts FROM t ORDER BY ts ASC LIMIT 1", true).await; + assert_eq!(work_queue_order(&liquid_scan(&plan)), ["a", "b", "c", "d"]); + } + + /// With sort pushdown turned off the rule pushes nothing either: the liquid + /// scan keeps no sort and the queue keeps listing order. + #[tokio::test] + async fn liquid_scan_keeps_listing_order_when_sort_pushdown_is_off() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, _ctx) = plan_time_sliced( + dir.path(), + "SELECT ts FROM t ORDER BY ts DESC LIMIT 1", + false, + ) + .await; + let cfg = liquid_scan(&plan); + let source = cfg + .file_source() + .downcast_ref::() + .unwrap(); + assert!(source.sort_order_for_reorder().is_none()); + assert_eq!(work_queue_order(&cfg), ["a", "b", "c", "d"]); + } + + /// A scan with no sort above it is left as it was. + #[tokio::test] + async fn liquid_scan_without_a_sort_keeps_no_sort_order() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, _ctx) = + plan_time_sliced(dir.path(), "SELECT ts FROM t WHERE ts > 150", true).await; + let cfg = liquid_scan(&plan); + let source = cfg + .file_source() + .downcast_ref::() + .unwrap(); + assert!(source.sort_order_for_reorder().is_none()); + assert_eq!(work_queue_order(&cfg), ["a", "b", "c", "d"]); + } + + /// A sort on an expression, not a plain column, cannot be ordered by file + /// statistics, so the liquid scan declines it. + #[tokio::test] + async fn liquid_scan_declines_a_sort_on_an_expression() { + let dir = tempfile::tempdir().unwrap(); + write_time_sliced_files(dir.path()); + let (plan, ctx) = plan_time_sliced( + dir.path(), + "SELECT ts FROM t ORDER BY ts % 7 DESC LIMIT 1", + true, + ) + .await; + let source = liquid_scan(&plan) + .file_source() + .downcast_ref::() + .unwrap() + .clone(); + assert!(source.sort_order_for_reorder().is_none()); + let batches = collect(plan, ctx.task_ctx()).await.unwrap(); + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, 1); + } } /// Pure unit tests for the footprint byte-math (no cache / no t4 mount, so they diff --git a/src/datafusion/src/reader/plantime/source.rs b/src/datafusion/src/reader/plantime/source.rs index 85291e1f..18604e8c 100644 --- a/src/datafusion/src/reader/plantime/source.rs +++ b/src/datafusion/src/reader/plantime/source.rs @@ -1,6 +1,7 @@ use super::LiquidMorselizer; use crate::cache::{ColumnLineages, LiquidCacheParquetRef}; use datafusion::{ + common::ScalarValue, common::{internal_err, tree_node::TreeNodeRecursion}, config::{ConfigOptions, TableParquetOptions}, datasource::{ @@ -13,13 +14,15 @@ use datafusion::{ error::Result, physical_expr::projection::ProjectionExprs, physical_expr::utils::conjunction, + physical_expr::{EquivalenceProperties, LexOrdering, PhysicalSortExpr, expressions::Column}, physical_expr_adapter::DefaultPhysicalExprAdapterFactory, physical_plan::{ - DisplayFormatType, PhysicalExpr, apply_expression_roots, + DisplayFormatType, PhysicalExpr, SortOrderPushdownResult, apply_expression_roots, filter_pushdown::{FilterPushdownPropagation, PushedDown, PushedDownPredicate}, metrics::ExecutionPlanMetricsSet, }, }; +use datafusion_datasource::PartitionedFile; use datafusion_datasource::morsel::Morselizer; use object_store::ObjectStore; use std::{ @@ -41,6 +44,10 @@ pub struct LiquidParquetSource { span: Option>, lineages: Arc, prefetch: bool, + /// The sort a parent `SortExec` asked this scan to approximate (set by + /// [`FileSource::try_pushdown_sort`]). Drives [`FileSource::reorder_files`], + /// which orders the shared work queue so a TopK sees its best files first. + sort_order_for_reorder: Option, } impl LiquidParquetSource { @@ -119,6 +126,10 @@ impl LiquidParquetSource { span: None, lineages: Arc::default(), prefetch: true, + // A `ParquetSource` keeps its own pushed sort private, so it cannot + // be carried over here. `LocalModeOptimizer` pushes the sorts down + // again once the scans are liquid. + sort_order_for_reorder: None, }; if let Some(predicate) = predicate { @@ -132,6 +143,76 @@ impl LiquidParquetSource { pub fn predicate(&self) -> Option> { self.predicate.clone() } + + /// The sort this scan was asked to approximate, if any. + pub fn sort_order_for_reorder(&self) -> Option<&LexOrdering> { + self.sort_order_for_reorder.as_ref() + } +} + +/// Order `files` so the files most likely to hold a TopK's rows come first: +/// by the per-file `min` of each leading plain-column sort key, in the key's +/// direction (largest `min` first for a descending key). Files without that +/// statistic go last. +/// +/// This mirrors `reorder_files_by_min_statistics` in +/// `datafusion-datasource-parquet`, which is crate-private. Sharing the same +/// order with DataFusion's own reader keeps a liquid scan's TopK behaviour the +/// same as a plain parquet scan's: the files that tighten the dynamic filter +/// the most are opened first, and every file the filter then rules out is +/// skipped without being opened. +fn reorder_files_by_min_statistics( + mut files: Vec, + sort_order: Option<&LexOrdering>, + table_schema: &arrow_schema::Schema, +) -> Vec { + let Some(sort_order) = sort_order else { + return files; + }; + // The longest prefix of plain columns that resolve in the table schema; + // the leading key is required. + let mut keys: Vec<(usize, bool)> = Vec::new(); + for sort_expr in sort_order.iter() { + let Some(col) = sort_expr.expr.downcast_ref::() else { + break; + }; + match table_schema.index_of(col.name()) { + Ok(idx) => keys.push((idx, sort_expr.options.descending)), + Err(_) => break, + } + } + if keys.is_empty() { + return files; + } + + let file_min = |file: &PartitionedFile, idx: usize| -> Option { + file.statistics + .as_ref()? + .column_statistics + .get(idx)? + .min_value + .get_value() + .cloned() + }; + files.sort_by(|a, b| { + for &(idx, descending) in &keys { + let ord = match (file_min(a, idx), file_min(b, idx)) { + (Some(va), Some(vb)) => { + let cmp = va.partial_cmp(&vb).unwrap_or(std::cmp::Ordering::Equal); + if descending { cmp.reverse() } else { cmp } + } + // Files with statistics first, whatever the direction. + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + }; + if ord != std::cmp::Ordering::Equal { + return ord; + } + } + std::cmp::Ordering::Equal + }); + files } impl FileSource for LiquidParquetSource { @@ -228,6 +309,9 @@ impl FileSource for LiquidParquetSource { if let Some(predicate) = self.filter() { write!(f, ", predicate={predicate}")?; } + if let Some(sort_order) = &self.sort_order_for_reorder { + write!(f, ", sort_order_for_reorder=[{sort_order}]")?; + } Ok(()) } DisplayFormatType::TreeRender => Ok(()), @@ -274,6 +358,69 @@ impl FileSource for LiquidParquetSource { .with_updated_node(source)) } + /// Approximate a sort for a parent TopK by ordering the files it reads. + /// + /// Mirrors `ParquetSource::try_pushdown_sort`, minus the row-group work + /// this reader does not do: when the request's leading key is a plain + /// column of the file schema, answer `Inexact` and remember the request, + /// so [`Self::reorder_files`] puts the files with the best statistics at + /// the front of the shared work queue. The parent `SortExec` stays, so the + /// result is the same; the TopK's dynamic filter just tightens after the + /// first files instead of the last. + /// + /// Everything else stays `Unsupported`, which leaves today's behaviour: + /// - a request the source's own ordering already satisfies, where + /// `FileScanConfig` sorts the file groups by statistics and may drop the + /// `SortExec` (an `Inexact` answer here would forbid that); + /// - a request whose proper prefix the ordering satisfies, so the + /// `SortExec`'s prefix-aware early stop still applies (the same reason + /// `ParquetSource` declines it); + /// - a leading key that is not a plain file column, which statistics + /// cannot order. + fn try_pushdown_sort( + &self, + order: &[PhysicalSortExpr], + eq_properties: &EquivalenceProperties, + ) -> Result>> { + let Some(sort_order) = LexOrdering::new(order.iter().cloned()) else { + return Ok(SortOrderPushdownResult::Unsupported); + }; + if eq_properties.ordering_satisfy(order.iter().cloned())? { + return Ok(SortOrderPushdownResult::Unsupported); + } + for prefix_len in 1..order.len() { + if eq_properties.ordering_satisfy(order[..prefix_len].iter().cloned())? { + return Ok(SortOrderPushdownResult::Unsupported); + } + } + let leading_is_file_column = sort_order + .first() + .expr + .downcast_ref::() + .is_some_and(|col| { + self.table_schema + .file_schema() + .field_with_name(col.name()) + .is_ok() + }); + if !leading_is_file_column { + return Ok(SortOrderPushdownResult::Unsupported); + } + let mut source = self.clone(); + source.sort_order_for_reorder = Some(sort_order); + Ok(SortOrderPushdownResult::Inexact { + inner: Arc::new(source) as Arc, + }) + } + + fn reorder_files(&self, files: Vec) -> Vec { + reorder_files_by_min_statistics( + files, + self.sort_order_for_reorder.as_ref(), + self.table_schema.table_schema(), + ) + } + fn apply_expressions( &self, f: &mut dyn FnMut(&Arc) -> Result, @@ -286,3 +433,108 @@ impl FileSource for LiquidParquetSource { ) } } + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::SortOptions; + use arrow_schema::{DataType, Field, Schema}; + use datafusion::common::{ColumnStatistics, Statistics, stats::Precision}; + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{BinaryExpr, Literal}; + + fn schema() -> Schema { + Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("ts", DataType::Int64, true), + ]) + } + + /// A file whose `ts` column has `min` (`None` = no statistic). + fn file(name: &str, ts_min: Option) -> PartitionedFile { + let file = PartitionedFile::new(name, 100); + let Some(min) = ts_min else { + return file; + }; + let ts = ColumnStatistics { + min_value: Precision::Exact(ScalarValue::Int64(Some(min))), + ..ColumnStatistics::new_unknown() + }; + file.with_statistics(Arc::new(Statistics { + num_rows: Precision::Absent, + total_byte_size: Precision::Absent, + column_statistics: vec![ColumnStatistics::new_unknown(), ts], + })) + } + + fn names(files: &[PartitionedFile]) -> Vec<&str> { + files + .iter() + .map(|f| f.object_meta.location.as_ref()) + .collect() + } + + fn ts_order(descending: bool) -> LexOrdering { + LexOrdering::new(vec![PhysicalSortExpr::new( + Arc::new(Column::new("ts", 1)), + SortOptions { + descending, + nulls_first: false, + }, + )]) + .unwrap() + } + + fn files() -> Vec { + vec![ + file("mid", Some(50)), + file("none", None), + file("old", Some(10)), + file("new", Some(90)), + ] + } + + #[test] + fn descending_puts_the_largest_min_first_and_missing_stats_last() { + let order = ts_order(true); + let sorted = reorder_files_by_min_statistics(files(), Some(&order), &schema()); + assert_eq!(names(&sorted), ["new", "mid", "old", "none"]); + } + + #[test] + fn ascending_puts_the_smallest_min_first_and_missing_stats_last() { + let order = ts_order(false); + let sorted = reorder_files_by_min_statistics(files(), Some(&order), &schema()); + assert_eq!(names(&sorted), ["old", "mid", "new", "none"]); + } + + #[test] + fn no_sort_order_keeps_the_given_order() { + let sorted = reorder_files_by_min_statistics(files(), None, &schema()); + assert_eq!(names(&sorted), ["mid", "none", "old", "new"]); + } + + #[test] + fn a_leading_expression_keeps_the_given_order() { + let expr: Arc = Arc::new(BinaryExpr::new( + Arc::new(Column::new("ts", 1)), + Operator::Plus, + Arc::new(Literal::new(ScalarValue::Int64(Some(1)))), + )); + let order = + LexOrdering::new(vec![PhysicalSortExpr::new(expr, SortOptions::default())]).unwrap(); + let sorted = reorder_files_by_min_statistics(files(), Some(&order), &schema()); + assert_eq!(names(&sorted), ["mid", "none", "old", "new"]); + } + + #[test] + fn a_column_missing_from_the_schema_keeps_the_given_order() { + let order = LexOrdering::new(vec![PhysicalSortExpr::new( + Arc::new(Column::new("gone", 5)), + SortOptions::default(), + )]) + .unwrap(); + let sorted = reorder_files_by_min_statistics(files(), Some(&order), &schema()); + assert_eq!(names(&sorted), ["mid", "none", "old", "new"]); + } +}