From e8804530865ad9cc3f9666cf1463f7a8d0b7bd5a Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 19 Jun 2025 14:54:40 +0800 Subject: [PATCH 01/80] add MaterializedCTE plan --- .../builders/builder_materialized_cte.rs | 26 ++++++++++++++ .../service/src/pipelines/builders/mod.rs | 1 + .../service/src/pipelines/pipeline_builder.rs | 2 ++ .../transform_recursive_cte_source.rs | 4 +++ src/query/sql/src/executor/format.rs | 10 ++++++ src/query/sql/src/executor/physical_plan.rs | 21 ++++++++++- .../sql/src/executor/physical_plan_visitor.rs | 17 +++++++++ .../sql/src/executor/physical_plans/mod.rs | 2 ++ .../physical_materialized_cte.rs | 36 +++++++++++++++++++ 9 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 src/query/service/src/pipelines/builders/builder_materialized_cte.rs create mode 100644 src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs new file mode 100644 index 0000000000000..321eccd3e0ccd --- /dev/null +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -0,0 +1,26 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::Result; +use databend_common_sql::executor::physical_plans::MaterializedCTE; + +use crate::pipelines::PipelineBuilder; + +impl PipelineBuilder { + pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { + self.build_pipeline(&cte.left)?; + self.build_pipeline(&cte.right)?; + Ok(()) + } +} \ No newline at end of file diff --git a/src/query/service/src/pipelines/builders/mod.rs b/src/query/service/src/pipelines/builders/mod.rs index 552580bc7c467..052416707a126 100644 --- a/src/query/service/src/pipelines/builders/mod.rs +++ b/src/query/service/src/pipelines/builders/mod.rs @@ -49,6 +49,7 @@ mod builder_union_all; mod builder_window; mod merge_into_join_optimizations; mod transform_builder; +mod builder_materialized_cte; pub use builder_replace_into::RawValueSource; pub use builder_replace_into::ValueSource; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index dd9f9d3af7a17..c7f9d773d5e55 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -292,6 +292,8 @@ impl PipelineBuilder { PhysicalPlan::Exchange(_) => Err(ErrorCode::Internal( "Invalid physical plan with PhysicalPlan::Exchange", )), + + PhysicalPlan::MaterializedCTE(cte) => self.build_materialized_cte(cte), }?; self.is_exchange_stack.pop(); diff --git a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs index d8596ddfc9e5f..14c353fd8c331 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs @@ -320,6 +320,10 @@ async fn create_memory_table_for_cte_scan( PhysicalPlan::AsyncFunction(plan) => { create_memory_table_for_cte_scan(ctx, plan.input.as_ref()).await?; } + PhysicalPlan::MaterializedCTE(plan) => { + create_memory_table_for_cte_scan(ctx, plan.left.as_ref()).await?; + create_memory_table_for_cte_scan(ctx, plan.right.as_ref()).await?; + } PhysicalPlan::TableScan(_) | PhysicalPlan::ConstantTableScan(_) | PhysicalPlan::ExpressionScan(_) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 1e3f8879339f3..95eaf484d59b7 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -527,6 +527,16 @@ fn to_format_tree( PhysicalPlan::BroadcastSink(_plan) => { Ok(FormatTreeNode::new("RuntimeFilterSink".to_string())) } + PhysicalPlan::MaterializedCTE(plan) => { + let mut children = Vec::new(); + append_profile_info(&mut children, profs, plan.plan_id); + children.push(to_format_tree(&plan.left, metadata, profs, context)?); + children.push(to_format_tree(&plan.right, metadata, profs, context)?); + Ok(FormatTreeNode::with_children( + "MaterializedCTE".to_string(), + children, + )) + } } } diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 33f29fabf5cd1..0c1c7f1c60d2a 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -71,6 +71,7 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; +use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; @@ -161,6 +162,8 @@ pub enum PhysicalPlan { // broadcast BroadcastSource(BroadcastSource), BroadcastSink(BroadcastSink), + + MaterializedCTE(Box), } impl PhysicalPlan { @@ -422,6 +425,12 @@ impl PhysicalPlan { *next_id += 1; plan.input.adjust_plan_id(next_id); } + PhysicalPlan::MaterializedCTE(plan) => { + plan.plan_id = *next_id; + *next_id += 1; + plan.left.adjust_plan_id(next_id); + plan.right.adjust_plan_id(next_id); + } } } @@ -480,6 +489,7 @@ impl PhysicalPlan { PhysicalPlan::RecursiveCteScan(v) => v.plan_id, PhysicalPlan::BroadcastSource(v) => v.plan_id, PhysicalPlan::BroadcastSink(v) => v.plan_id, + PhysicalPlan::MaterializedCTE(v) => v.plan_id, } } @@ -537,6 +547,7 @@ impl PhysicalPlan { PhysicalPlan::ChunkAppendData(_) => todo!(), PhysicalPlan::ChunkMerge(_) => todo!(), PhysicalPlan::ChunkCommitInsert(_) => todo!(), + PhysicalPlan::MaterializedCTE(plan) => plan.output_schema(), } } @@ -600,6 +611,7 @@ impl PhysicalPlan { PhysicalPlan::ChunkCommitInsert(_) => "Commit".to_string(), PhysicalPlan::BroadcastSource(_) => "RuntimeFilterSource".to_string(), PhysicalPlan::BroadcastSink(_) => "RuntimeFilterSink".to_string(), + PhysicalPlan::MaterializedCTE(_) => "MaterializedCTE".to_string(), } } @@ -674,6 +686,9 @@ impl PhysicalPlan { CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v.as_ref())), CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v.as_ref())), }, + PhysicalPlan::MaterializedCTE(plan) => Box::new( + std::iter::once(plan.left.as_ref()).chain(std::iter::once(plan.right.as_ref())), + ), } } @@ -747,6 +762,9 @@ impl PhysicalPlan { CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v.as_mut())), CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v.as_mut())), }, + PhysicalPlan::MaterializedCTE(plan) => Box::new( + std::iter::once(plan.left.as_mut()).chain(std::iter::once(plan.right.as_mut())), + ), PhysicalPlan::BroadcastSink(plan) => Box::new(std::iter::once(plan.input.as_mut())), } } @@ -805,7 +823,8 @@ impl PhysicalPlan { | PhysicalPlan::ChunkMerge(_) | PhysicalPlan::ChunkCommitInsert(_) | PhysicalPlan::BroadcastSource(_) - | PhysicalPlan::BroadcastSink(_) => None, + | PhysicalPlan::BroadcastSink(_) + | PhysicalPlan::MaterializedCTE(_) => None, } } diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index c26807609bf81..926ffdb938827 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -61,6 +61,7 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; +use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; @@ -124,6 +125,7 @@ pub trait PhysicalPlanReplacer { PhysicalPlan::ChunkCommitInsert(plan) => self.replace_chunk_commit_insert(plan), PhysicalPlan::BroadcastSource(plan) => self.replace_runtime_filter_source(plan), PhysicalPlan::BroadcastSink(plan) => self.replace_runtime_filter_sink(plan), + PhysicalPlan::MaterializedCTE(plan) => self.replace_materialized_cte(plan), } } @@ -642,6 +644,17 @@ pub trait PhysicalPlanReplacer { }, ))) } + + fn replace_materialized_cte(&mut self, plan: &MaterializedCTE) -> Result { + let left = self.replace(&plan.left)?; + let right = self.replace(&plan.right)?; + Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { + plan_id: plan.plan_id, + left: Box::new(left), + right: Box::new(right), + stat_info: plan.stat_info.clone(), + }))) + } } impl PhysicalPlan { @@ -794,6 +807,10 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSink(plan) => { Self::traverse(&plan.input, pre_visit, visit, post_visit); } + PhysicalPlan::MaterializedCTE(plan) => { + Self::traverse(&plan.left, pre_visit, visit, post_visit); + Self::traverse(&plan.right, pre_visit, visit, post_visit); + } } post_visit(plan); } diff --git a/src/query/sql/src/executor/physical_plans/mod.rs b/src/query/sql/src/executor/physical_plans/mod.rs index 2e04928a2a55c..9f9ee83dce48c 100644 --- a/src/query/sql/src/executor/physical_plans/mod.rs +++ b/src/query/sql/src/executor/physical_plans/mod.rs @@ -53,6 +53,7 @@ mod physical_replace_async_source; mod physical_replace_deduplicate; mod physical_replace_into; mod physical_row_fetch; +mod physical_materialized_cte; mod physical_sort; mod physical_table_scan; mod physical_udf; @@ -106,6 +107,7 @@ pub use physical_replace_async_source::ReplaceAsyncSourcer; pub use physical_replace_deduplicate::*; pub use physical_replace_into::ReplaceInto; pub use physical_row_fetch::RowFetch; +pub use physical_materialized_cte::MaterializedCTE; pub use physical_sort::Sort; pub use physical_table_scan::TableScan; pub use physical_udf::Udf; diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs new file mode 100644 index 0000000000000..901d1eed99764 --- /dev/null +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -0,0 +1,36 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::Result; +use databend_common_expression::DataSchemaRef; + +use crate::executor::explain::PlanStatsInfo; +use crate::executor::PhysicalPlan; + +/// This is a binary operator that executes its children in order (left to right), and returns the results of the right child +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct MaterializedCTE { + // A unique id of operator in a `PhysicalPlan` tree, only used for display. + pub plan_id: u32, + // Only used for explain + pub stat_info: Option, + pub left: Box, + pub right: Box, +} + +impl MaterializedCTE { + pub fn output_schema(&self) -> Result { + self.right.output_schema() + } +} From 466e1635c240527cda5b78fd9aea14a0604e45f5 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 19 Jun 2025 21:09:00 +0800 Subject: [PATCH 02/80] build pipeline --- .../builders/builder_materialized_cte.rs | 29 +++++++- .../service/src/pipelines/builders/mod.rs | 2 +- .../service/src/pipelines/pipeline_builder.rs | 4 ++ .../processors/transforms/materialized_cte.rs | 69 +++++++++++++++++++ .../pipelines/processors/transforms/mod.rs | 3 + src/query/sql/src/executor/physical_plan.rs | 2 +- .../sql/src/executor/physical_plan_visitor.rs | 3 +- .../sql/src/executor/physical_plans/mod.rs | 4 +- .../physical_materialized_cte.rs | 1 + 9 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 src/query/service/src/pipelines/processors/transforms/materialized_cte.rs diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 321eccd3e0ccd..385a07ccc8e44 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -15,12 +15,35 @@ use databend_common_exception::Result; use databend_common_sql::executor::physical_plans::MaterializedCTE; +use crate::pipelines::processors::transforms::MaterializedCteSink; use crate::pipelines::PipelineBuilder; - +use crate::sessions::QueryContext; impl PipelineBuilder { pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { - self.build_pipeline(&cte.left)?; + // init builder for cte pipeline + let sub_context = QueryContext::create_from(self.ctx.as_ref()); + let sub_builder = PipelineBuilder::create( + self.func_ctx.clone(), + self.settings.clone(), + sub_context, + self.main_pipeline.get_scopes(), + ); + + // build cte pipeline + let mut build_res = sub_builder.finalize(&cte.left)?; + build_res.main_pipeline.try_resize(1)?; + let (tx, rx) = tokio::sync::watch::channel(Default::default()); + self.cte_receivers.insert(cte.cte_name.clone(), rx); + build_res + .main_pipeline + .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; + + // add cte pipeline to pipelines + self.pipelines.push(build_res.main_pipeline); + self.pipelines.extend(build_res.sources_pipelines); + + // build main pipeline self.build_pipeline(&cte.right)?; Ok(()) } -} \ No newline at end of file +} diff --git a/src/query/service/src/pipelines/builders/mod.rs b/src/query/service/src/pipelines/builders/mod.rs index 052416707a126..a229fa18d008d 100644 --- a/src/query/service/src/pipelines/builders/mod.rs +++ b/src/query/service/src/pipelines/builders/mod.rs @@ -30,6 +30,7 @@ mod builder_hilbert_partition; mod builder_insert_multi_table; mod builder_join; mod builder_limit; +mod builder_materialized_cte; mod builder_mutation; mod builder_mutation_manipulate; mod builder_mutation_organize; @@ -49,7 +50,6 @@ mod builder_union_all; mod builder_window; mod merge_into_join_optimizations; mod transform_builder; -mod builder_materialized_cte; pub use builder_replace_into::RawValueSource; pub use builder_replace_into::ValueSource; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index c7f9d773d5e55..c44fefd55cfbb 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -27,9 +27,11 @@ use databend_common_pipeline_core::ExecutionInfo; use databend_common_pipeline_core::Pipeline; use databend_common_settings::Settings; use databend_common_sql::executor::PhysicalPlan; +use tokio::sync::watch::Receiver; use super::PipelineBuilderData; use crate::interpreters::CreateTableInterpreter; +use crate::pipelines::processors::transforms::MaterializedCteData; use crate::pipelines::processors::HashJoinBuildState; use crate::pipelines::processors::HashJoinState; use crate::pipelines::PipelineBuildResult; @@ -57,6 +59,7 @@ pub struct PipelineBuilder { pub(crate) is_exchange_stack: Vec, pub contain_sink_processor: bool, + pub cte_receivers: HashMap>>, } impl PipelineBuilder { @@ -79,6 +82,7 @@ impl PipelineBuilder { r_cte_scan_interpreters: vec![], contain_sink_processor: false, is_exchange_stack: vec![], + cte_receivers: HashMap::new(), } } diff --git a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs new file mode 100644 index 0000000000000..7d0d8e28c1018 --- /dev/null +++ b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_pipeline_core::processors::InputPort; +use databend_common_pipeline_core::processors::ProcessorPtr; +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +use databend_common_pipeline_sinks::Sink; +use databend_common_pipeline_sinks::Sinker; +use tokio::sync::watch::Sender; + +pub struct MaterializedCteSink { + sender: Sender>, + blocks: Vec, +} + +#[derive(Default)] +pub struct MaterializedCteData { + blocks: Vec, +} + +impl MaterializedCteData { + pub fn new(blocks: Vec) -> Self { + Self { blocks } + } +} + +impl MaterializedCteSink { + pub fn create( + input: Arc, + sender: Sender>, + ) -> Result { + Ok(ProcessorPtr::create(Sinker::create(input, Self { + blocks: vec![], + sender, + }))) + } +} + +impl Sink for MaterializedCteSink { + const NAME: &'static str = "MaterializedCteSink"; + + fn consume(&mut self, data_block: DataBlock) -> Result<()> { + self.blocks.push(data_block); + Ok(()) + } + + fn on_finish(&mut self) -> Result<()> { + self.sender + .send(Arc::new(MaterializedCteData::new(self.blocks.clone()))) + .map_err(|_| { + ErrorCode::Internal("Failed to send blocks to materialized cte consumer") + })?; + Ok(()) + } +} diff --git a/src/query/service/src/pipelines/processors/transforms/mod.rs b/src/query/service/src/pipelines/processors/transforms/mod.rs index 80966daa5fa8d..25f8dea06f2c9 100644 --- a/src/query/service/src/pipelines/processors/transforms/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/mod.rs @@ -16,6 +16,7 @@ pub mod aggregator; #[allow(dead_code)] mod broadcast; mod hash_join; +mod materialized_cte; pub(crate) mod range_join; mod runtime_pool; mod transform_add_computed_columns; @@ -46,6 +47,8 @@ mod window; pub use broadcast::BroadcastSinkProcessor; pub use broadcast::BroadcastSourceProcessor; pub use hash_join::*; +pub use materialized_cte::MaterializedCteData; +pub use materialized_cte::MaterializedCteSink; pub use transform_add_computed_columns::TransformAddComputedColumns; pub use transform_add_const_columns::TransformAddConstColumns; pub use transform_add_internal_columns::TransformAddInternalColumns; diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 0c1c7f1c60d2a..2bbf03bb7ef70 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -62,6 +62,7 @@ use crate::executor::physical_plans::ExpressionScan; use crate::executor::physical_plans::Filter; use crate::executor::physical_plans::HashJoin; use crate::executor::physical_plans::Limit; +use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Mutation; use crate::executor::physical_plans::ProjectSet; use crate::executor::physical_plans::RangeJoin; @@ -71,7 +72,6 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; -use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index 926ffdb938827..3973665cc64b1 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -52,6 +52,7 @@ use crate::executor::physical_plans::ExchangeSource; use crate::executor::physical_plans::Filter; use crate::executor::physical_plans::HashJoin; use crate::executor::physical_plans::Limit; +use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Mutation; use crate::executor::physical_plans::MutationSource; use crate::executor::physical_plans::ProjectSet; @@ -61,7 +62,6 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; -use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; @@ -653,6 +653,7 @@ pub trait PhysicalPlanReplacer { left: Box::new(left), right: Box::new(right), stat_info: plan.stat_info.clone(), + cte_name: plan.cte_name.clone(), }))) } } diff --git a/src/query/sql/src/executor/physical_plans/mod.rs b/src/query/sql/src/executor/physical_plans/mod.rs index 9f9ee83dce48c..6275ef8b46771 100644 --- a/src/query/sql/src/executor/physical_plans/mod.rs +++ b/src/query/sql/src/executor/physical_plans/mod.rs @@ -38,6 +38,7 @@ mod physical_hash_join; mod physical_join; mod physical_join_filter; mod physical_limit; +mod physical_materialized_cte; mod physical_multi_table_insert; mod physical_mutation; mod physical_mutation_into_organize; @@ -53,7 +54,6 @@ mod physical_replace_async_source; mod physical_replace_deduplicate; mod physical_replace_into; mod physical_row_fetch; -mod physical_materialized_cte; mod physical_sort; mod physical_table_scan; mod physical_udf; @@ -91,6 +91,7 @@ pub use physical_join_filter::JoinRuntimeFilter; pub use physical_join_filter::PhysicalRuntimeFilter; pub use physical_join_filter::PhysicalRuntimeFilters; pub use physical_limit::Limit; +pub use physical_materialized_cte::MaterializedCTE; pub use physical_multi_table_insert::*; pub use physical_mutation::*; pub use physical_mutation_into_organize::MutationOrganize; @@ -107,7 +108,6 @@ pub use physical_replace_async_source::ReplaceAsyncSourcer; pub use physical_replace_deduplicate::*; pub use physical_replace_into::ReplaceInto; pub use physical_row_fetch::RowFetch; -pub use physical_materialized_cte::MaterializedCTE; pub use physical_sort::Sort; pub use physical_table_scan::TableScan; pub use physical_udf::Udf; diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index 901d1eed99764..b252b80d0e269 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -27,6 +27,7 @@ pub struct MaterializedCTE { pub stat_info: Option, pub left: Box, pub right: Box, + pub cte_name: String, } impl MaterializedCTE { From db16044ed4b484d0340bb92b72584900a30a2c55 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sat, 21 Jun 2025 06:27:58 +0800 Subject: [PATCH 03/80] build pipeline --- .../builders/builder_cte_consumer.rs | 60 ++++++++++++++++ .../builders/builder_materialized_cte.rs | 6 +- .../service/src/pipelines/builders/mod.rs | 1 + .../service/src/pipelines/pipeline_builder.rs | 3 + .../processors/transforms/materialized_cte.rs | 69 ++++++++++++++++++- .../pipelines/processors/transforms/mod.rs | 1 + .../transform_recursive_cte_source.rs | 1 + src/query/sql/src/executor/format.rs | 16 +++++ src/query/sql/src/executor/physical_plan.rs | 14 +++- .../sql/src/executor/physical_plan_visitor.rs | 7 ++ .../sql/src/executor/physical_plans/mod.rs | 2 + .../physical_plans/physical_cte_consumer.rs | 35 ++++++++++ 12 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 src/query/service/src/pipelines/builders/builder_cte_consumer.rs create mode 100644 src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs diff --git a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs new file mode 100644 index 0000000000000..adfe50a2b0fad --- /dev/null +++ b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs @@ -0,0 +1,60 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_sql::executor::physical_plans::CTEConsumer; +use databend_common_storages_fuse::TableContext; + +use crate::pipelines::processors::transforms::CTESource; +use crate::pipelines::PipelineBuilder; + +impl PipelineBuilder { + pub(crate) fn build_cte_consumer(&mut self, cte: &CTEConsumer) -> Result<()> { + let receiver = self + .cte_receivers + .get(&cte.cte_name) + .ok_or_else(|| { + ErrorCode::Internal(format!("CTE receiver not found for name: {}", cte.cte_name)) + })? + .clone(); + + let current_consumer_id = + *self + .next_cte_consumer_id + .get(&cte.cte_name) + .ok_or_else(|| { + ErrorCode::Internal(format!( + "CTE consumer id not found for name: {}", + cte.cte_name + )) + })?; + + self.next_cte_consumer_id + .insert(cte.cte_name.clone(), current_consumer_id + 1); + + self.main_pipeline.add_source( + |output_port| { + CTESource::create( + self.ctx.clone(), + output_port.clone(), + receiver.clone(), + current_consumer_id, + ) + }, + self.ctx.get_settings().get_max_threads()? as usize, + )?; + Ok(()) + } +} diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 385a07ccc8e44..02e42f3ee885f 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -12,9 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + use databend_common_exception::Result; use databend_common_sql::executor::physical_plans::MaterializedCTE; +use crate::pipelines::processors::transforms::MaterializedCteData; use crate::pipelines::processors::transforms::MaterializedCteSink; use crate::pipelines::PipelineBuilder; use crate::sessions::QueryContext; @@ -32,8 +35,9 @@ impl PipelineBuilder { // build cte pipeline let mut build_res = sub_builder.finalize(&cte.left)?; build_res.main_pipeline.try_resize(1)?; - let (tx, rx) = tokio::sync::watch::channel(Default::default()); + let (tx, rx) = tokio::sync::watch::channel(Arc::new(MaterializedCteData::default())); self.cte_receivers.insert(cte.cte_name.clone(), rx); + self.next_cte_consumer_id.insert(cte.cte_name.clone(), 0); build_res .main_pipeline .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; diff --git a/src/query/service/src/pipelines/builders/mod.rs b/src/query/service/src/pipelines/builders/mod.rs index a229fa18d008d..ba86e21fd6dc0 100644 --- a/src/query/service/src/pipelines/builders/mod.rs +++ b/src/query/service/src/pipelines/builders/mod.rs @@ -22,6 +22,7 @@ mod builder_commit; mod builder_compact; mod builder_copy_into_location; mod builder_copy_into_table; +mod builder_cte_consumer; mod builder_distributed_insert_select; mod builder_exchange; mod builder_fill_missing_columns; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index c44fefd55cfbb..d7a593fe7c2cc 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -60,6 +60,7 @@ pub struct PipelineBuilder { pub contain_sink_processor: bool, pub cte_receivers: HashMap>>, + pub next_cte_consumer_id: HashMap, } impl PipelineBuilder { @@ -83,6 +84,7 @@ impl PipelineBuilder { contain_sink_processor: false, is_exchange_stack: vec![], cte_receivers: HashMap::new(), + next_cte_consumer_id: HashMap::new(), } } @@ -298,6 +300,7 @@ impl PipelineBuilder { )), PhysicalPlan::MaterializedCTE(cte) => self.build_materialized_cte(cte), + PhysicalPlan::CTEConsumer(cte_consumer) => self.build_cte_consumer(cte_consumer), }?; self.is_exchange_stack.pop(); diff --git a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs index 7d0d8e28c1018..092ae0ac94366 100644 --- a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs +++ b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs @@ -1,9 +1,12 @@ +use std::collections::HashMap; use std::sync::Arc; +use std::sync::Mutex; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_pipeline_core::processors::InputPort; +use databend_common_pipeline_core::processors::OutputPort; use databend_common_pipeline_core::processors::ProcessorPtr; // Copyright 2021 Datafuse Labs // @@ -20,6 +23,9 @@ use databend_common_pipeline_core::processors::ProcessorPtr; // limitations under the License. use databend_common_pipeline_sinks::Sink; use databend_common_pipeline_sinks::Sinker; +use databend_common_pipeline_sources::AsyncSource; +use databend_common_pipeline_sources::AsyncSourcer; +use tokio::sync::watch::Receiver; use tokio::sync::watch::Sender; pub struct MaterializedCteSink { @@ -30,11 +36,29 @@ pub struct MaterializedCteSink { #[derive(Default)] pub struct MaterializedCteData { blocks: Vec, + // consumer_id -> current_index + consumer_states: Arc>>, } impl MaterializedCteData { pub fn new(blocks: Vec) -> Self { - Self { blocks } + Self { + blocks, + consumer_states: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub fn get_next_block(&self, consumer_id: usize) -> Option { + let mut states = self.consumer_states.lock().unwrap(); + let current_index = states.get(&consumer_id).copied().unwrap_or(0); + + if current_index < self.blocks.len() { + let block = self.blocks[current_index].clone(); + states.insert(consumer_id, current_index + 1); + Some(block) + } else { + None + } } } @@ -67,3 +91,46 @@ impl Sink for MaterializedCteSink { Ok(()) } } + +pub struct CTESource { + receiver: Receiver>, + data: Option>, + consumer_id: usize, +} + +impl CTESource { + pub fn create( + ctx: Arc, + output_port: Arc, + receiver: Receiver>, + consumer_id: usize, + ) -> Result { + AsyncSourcer::create(ctx, output_port, Self { + receiver, + data: None, + consumer_id, + }) + } +} + +#[async_trait::async_trait] +impl AsyncSource for CTESource { + const NAME: &'static str = "CTEConsumerSource"; + + #[async_backtrace::framed] + async fn generate(&mut self) -> Result> { + if self.data.is_none() { + self.receiver.changed().await.map_err(|_| { + ErrorCode::Internal("Failed to get data from receiver in CTEConsumerSource") + })?; + self.data = Some(self.receiver.borrow().clone()); + } + + if let Some(data) = &self.data { + if let Some(block) = data.get_next_block(self.consumer_id) { + return Ok(Some(block)); + } + } + Ok(None) + } +} diff --git a/src/query/service/src/pipelines/processors/transforms/mod.rs b/src/query/service/src/pipelines/processors/transforms/mod.rs index 25f8dea06f2c9..90aaaf365f3cf 100644 --- a/src/query/service/src/pipelines/processors/transforms/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/mod.rs @@ -47,6 +47,7 @@ mod window; pub use broadcast::BroadcastSinkProcessor; pub use broadcast::BroadcastSourceProcessor; pub use hash_join::*; +pub use materialized_cte::CTESource; pub use materialized_cte::MaterializedCteData; pub use materialized_cte::MaterializedCteSink; pub use transform_add_computed_columns::TransformAddComputedColumns; diff --git a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs index 14c353fd8c331..ee9b098b5ee57 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs @@ -356,6 +356,7 @@ async fn create_memory_table_for_cte_scan( | PhysicalPlan::ChunkMerge(_) | PhysicalPlan::ChunkCommitInsert(_) | PhysicalPlan::BroadcastSource(_) + | PhysicalPlan::CTEConsumer(_) | PhysicalPlan::BroadcastSink(_) => {} } Ok(()) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 95eaf484d59b7..c364da2bb00fb 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -537,6 +537,22 @@ fn to_format_tree( children, )) } + PhysicalPlan::CTEConsumer(plan) => { + let mut children = Vec::new(); + children.push(FormatTreeNode::new(format!( + "cte_name: {}", + plan.cte_name.clone() + ))); + children.push(FormatTreeNode::new(format!( + "cte_schema: [{}]", + format_output_columns(plan.cte_schema.clone(), metadata, false) + ))); + append_profile_info(&mut children, profs, plan.plan_id); + Ok(FormatTreeNode::with_children( + "CTEConsumer".to_string(), + children, + )) + } } } diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 2bbf03bb7ef70..7899b6aca9633 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -37,6 +37,7 @@ use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFinal; use crate::executor::physical_plans::AggregatePartial; use crate::executor::physical_plans::AsyncFunction; +use crate::executor::physical_plans::CTEConsumer; use crate::executor::physical_plans::CacheScan; use crate::executor::physical_plans::ChunkAppendData; use crate::executor::physical_plans::ChunkCastSchema; @@ -164,6 +165,7 @@ pub enum PhysicalPlan { BroadcastSink(BroadcastSink), MaterializedCTE(Box), + CTEConsumer(Box), } impl PhysicalPlan { @@ -431,6 +433,10 @@ impl PhysicalPlan { plan.left.adjust_plan_id(next_id); plan.right.adjust_plan_id(next_id); } + PhysicalPlan::CTEConsumer(plan) => { + plan.plan_id = *next_id; + *next_id += 1; + } } } @@ -490,6 +496,7 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSource(v) => v.plan_id, PhysicalPlan::BroadcastSink(v) => v.plan_id, PhysicalPlan::MaterializedCTE(v) => v.plan_id, + PhysicalPlan::CTEConsumer(v) => v.plan_id, } } @@ -548,6 +555,7 @@ impl PhysicalPlan { PhysicalPlan::ChunkMerge(_) => todo!(), PhysicalPlan::ChunkCommitInsert(_) => todo!(), PhysicalPlan::MaterializedCTE(plan) => plan.output_schema(), + PhysicalPlan::CTEConsumer(plan) => plan.output_schema(), } } @@ -612,6 +620,7 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSource(_) => "RuntimeFilterSource".to_string(), PhysicalPlan::BroadcastSink(_) => "RuntimeFilterSink".to_string(), PhysicalPlan::MaterializedCTE(_) => "MaterializedCTE".to_string(), + PhysicalPlan::CTEConsumer(_) => "CTEConsumer".to_string(), } } @@ -625,6 +634,7 @@ impl PhysicalPlan { | PhysicalPlan::ReplaceAsyncSourcer(_) | PhysicalPlan::Recluster(_) | PhysicalPlan::RecursiveCteScan(_) + | PhysicalPlan::CTEConsumer(_) | PhysicalPlan::BroadcastSource(_) => Box::new(std::iter::empty()), PhysicalPlan::HilbertPartition(plan) => Box::new(std::iter::once(plan.input.as_ref())), PhysicalPlan::Filter(plan) => Box::new(std::iter::once(plan.input.as_ref())), @@ -702,6 +712,7 @@ impl PhysicalPlan { | PhysicalPlan::ReplaceAsyncSourcer(_) | PhysicalPlan::Recluster(_) | PhysicalPlan::BroadcastSource(_) + | PhysicalPlan::CTEConsumer(_) | PhysicalPlan::RecursiveCteScan(_) => Box::new(std::iter::empty()), PhysicalPlan::HilbertPartition(plan) => Box::new(std::iter::once(plan.input.as_mut())), PhysicalPlan::Filter(plan) => Box::new(std::iter::once(plan.input.as_mut())), @@ -824,7 +835,8 @@ impl PhysicalPlan { | PhysicalPlan::ChunkCommitInsert(_) | PhysicalPlan::BroadcastSource(_) | PhysicalPlan::BroadcastSink(_) - | PhysicalPlan::MaterializedCTE(_) => None, + | PhysicalPlan::MaterializedCTE(_) + | PhysicalPlan::CTEConsumer(_) => None, } } diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index 3973665cc64b1..fa3ae41012517 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -29,6 +29,7 @@ use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFinal; use crate::executor::physical_plans::AggregatePartial; use crate::executor::physical_plans::AsyncFunction; +use crate::executor::physical_plans::CTEConsumer; use crate::executor::physical_plans::ChunkAppendData; use crate::executor::physical_plans::ChunkCastSchema; use crate::executor::physical_plans::ChunkCommitInsert; @@ -126,6 +127,7 @@ pub trait PhysicalPlanReplacer { PhysicalPlan::BroadcastSource(plan) => self.replace_runtime_filter_source(plan), PhysicalPlan::BroadcastSink(plan) => self.replace_runtime_filter_sink(plan), PhysicalPlan::MaterializedCTE(plan) => self.replace_materialized_cte(plan), + PhysicalPlan::CTEConsumer(plan) => self.replace_cte_consumer(plan), } } @@ -656,6 +658,10 @@ pub trait PhysicalPlanReplacer { cte_name: plan.cte_name.clone(), }))) } + + fn replace_cte_consumer(&mut self, plan: &CTEConsumer) -> Result { + Ok(PhysicalPlan::CTEConsumer(Box::new(plan.clone()))) + } } impl PhysicalPlan { @@ -679,6 +685,7 @@ impl PhysicalPlan { | PhysicalPlan::ExchangeSource(_) | PhysicalPlan::CompactSource(_) | PhysicalPlan::MutationSource(_) + | PhysicalPlan::CTEConsumer(_) | PhysicalPlan::BroadcastSource(_) => {} PhysicalPlan::Filter(plan) => { Self::traverse(&plan.input, pre_visit, visit, post_visit); diff --git a/src/query/sql/src/executor/physical_plans/mod.rs b/src/query/sql/src/executor/physical_plans/mod.rs index 6275ef8b46771..75b70be749b3b 100644 --- a/src/query/sql/src/executor/physical_plans/mod.rs +++ b/src/query/sql/src/executor/physical_plans/mod.rs @@ -61,6 +61,7 @@ mod physical_union_all; mod physical_window; mod physical_window_partition; +mod physical_cte_consumer; pub use common::*; pub use physical_add_stream_column::AddStreamColumn; pub use physical_aggregate_expand::AggregateExpand; @@ -78,6 +79,7 @@ pub use physical_compact_source::CompactSource; pub use physical_constant_table_scan::ConstantTableScan; pub use physical_copy_into_location::CopyIntoLocation; pub use physical_copy_into_table::*; +pub use physical_cte_consumer::*; pub use physical_distributed_insert_select::DistributedInsertSelect; pub use physical_eval_scalar::EvalScalar; pub use physical_exchange::Exchange; diff --git a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs new file mode 100644 index 0000000000000..e1e002f435d30 --- /dev/null +++ b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs @@ -0,0 +1,35 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::Result; +use databend_common_expression::DataSchemaRef; + +use crate::executor::explain::PlanStatsInfo; + +/// This is a leaf operator that consumes the result of a materialized CTE. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct CTEConsumer { + // A unique id of operator in a `PhysicalPlan` tree, only used for display. + pub plan_id: u32, + // Only used for explain + pub stat_info: Option, + pub cte_name: String, + pub cte_schema: DataSchemaRef, +} + +impl CTEConsumer { + pub fn output_schema(&self) -> Result { + Ok(self.cte_schema.clone()) + } +} From 26bb0ab33ca6d19c79581b802cfef007df987773 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 23 Jun 2025 12:41:42 +0800 Subject: [PATCH 04/80] add operator --- .../sql/src/executor/physical_plan_builder.rs | 7 ++ .../physical_plans/physical_cte_consumer.rs | 17 ++++ .../physical_materialized_cte.rs | 21 +++++ src/query/sql/src/planner/binder/util.rs | 6 ++ .../src/planner/optimizer/ir/expr/s_expr.rs | 2 + .../sql/src/planner/optimizer/ir/format.rs | 2 + .../optimizer/optimizers/hyper_dp/dphyp.rs | 15 ++++ .../hyper_dp/dynamic_sample/dynamic_sample.rs | 2 + .../decorrelate/subquery_decorrelator.rs | 7 ++ .../join_rules/rule_semi_to_inner_join.rs | 2 + .../sql/src/planner/plans/cte_consumer.rs | 88 +++++++++++++++++++ .../sql/src/planner/plans/materialized_cte.rs | 42 +++++++++ src/query/sql/src/planner/plans/mod.rs | 4 + src/query/sql/src/planner/plans/operator.rs | 32 +++++++ 14 files changed, 247 insertions(+) create mode 100644 src/query/sql/src/planner/plans/cte_consumer.rs create mode 100644 src/query/sql/src/planner/plans/materialized_cte.rs diff --git a/src/query/sql/src/executor/physical_plan_builder.rs b/src/query/sql/src/executor/physical_plan_builder.rs index c50adc9f30db4..cc866a3f44b36 100644 --- a/src/query/sql/src/executor/physical_plan_builder.rs +++ b/src/query/sql/src/executor/physical_plan_builder.rs @@ -129,6 +129,13 @@ impl PhysicalPlanBuilder { self.build_mutation_source(mutation_source).await } RelOperator::CompactBlock(compact) => self.build_compact_block(compact).await, + RelOperator::MaterializedCTE(materialized_cte) => { + self.build_materialized_cte(s_expr, materialized_cte, stat_info) + .await + } + RelOperator::CTEConsumer(cte_consumer) => { + self.build_cte_consumer(cte_consumer, stat_info).await + } } } diff --git a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs index e1e002f435d30..819c5437ac5cb 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs @@ -16,6 +16,8 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::executor::explain::PlanStatsInfo; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanBuilder; /// This is a leaf operator that consumes the result of a materialized CTE. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -33,3 +35,18 @@ impl CTEConsumer { Ok(self.cte_schema.clone()) } } + +impl PhysicalPlanBuilder { + pub(crate) async fn build_cte_consumer( + &mut self, + cte_consumer: &crate::plans::CTEConsumer, + stat_info: PlanStatsInfo, + ) -> Result { + Ok(PhysicalPlan::CTEConsumer(Box::new(CTEConsumer { + plan_id: 0, + stat_info: Some(stat_info), + cte_name: cte_consumer.cte_name.clone(), + cte_schema: Default::default(), + }))) + } +} diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index b252b80d0e269..6b42cba900749 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -17,6 +17,8 @@ use databend_common_expression::DataSchemaRef; use crate::executor::explain::PlanStatsInfo; use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanBuilder; +use crate::optimizer::ir::SExpr; /// This is a binary operator that executes its children in order (left to right), and returns the results of the right child #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -35,3 +37,22 @@ impl MaterializedCTE { self.right.output_schema() } } + +impl PhysicalPlanBuilder { + pub(crate) async fn build_materialized_cte( + &mut self, + s_expr: &SExpr, + materialized_cte: &crate::plans::MaterializedCTE, + stat_info: PlanStatsInfo, + ) -> Result { + let left_side = Box::new(self.build(s_expr.child(0)?, Default::default()).await?); + let right_side = Box::new(self.build(s_expr.child(1)?, Default::default()).await?); + Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { + plan_id: 0, + stat_info: Some(stat_info), + left: left_side, + right: right_side, + cte_name: materialized_cte.cte_name.clone(), + }))) + } +} diff --git a/src/query/sql/src/planner/binder/util.rs b/src/query/sql/src/planner/binder/util.rs index 039a42c2f806d..be6568d310337 100644 --- a/src/query/sql/src/planner/binder/util.rs +++ b/src/query/sql/src/planner/binder/util.rs @@ -53,6 +53,11 @@ impl Binder { self.count_r_cte_scan(expr.child(1)?, cte_scan_names, cte_types)?; } + RelOperator::MaterializedCTE(_) => { + self.count_r_cte_scan(expr.child(0)?, cte_scan_names, cte_types)?; + self.count_r_cte_scan(expr.child(1)?, cte_scan_names, cte_types)?; + } + RelOperator::ProjectSet(_) | RelOperator::AsyncFunction(_) | RelOperator::Udf(_) @@ -72,6 +77,7 @@ impl Binder { | RelOperator::DummyTableScan(_) | RelOperator::ConstantTableScan(_) | RelOperator::ExpressionScan(_) + | RelOperator::CTEConsumer(_) | RelOperator::CacheScan(_) => {} // Each recursive step in a recursive query generates new rows, and these rows are used for the next recursion. // Each step depends on the results of the previous step, so it's essential to ensure that the result set is built incrementally. diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index ac784174ad191..f67a3416ea94c 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -374,6 +374,8 @@ impl SExpr { | RelOperator::CacheScan(_) | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::CompactBlock(_) => {} }; for child in &self.children { diff --git a/src/query/sql/src/planner/optimizer/ir/format.rs b/src/query/sql/src/planner/optimizer/ir/format.rs index f9613af6b35ef..5d6909f2cf374 100644 --- a/src/query/sql/src/planner/optimizer/ir/format.rs +++ b/src/query/sql/src/planner/optimizer/ir/format.rs @@ -80,6 +80,8 @@ fn display_rel_op(rel_op: &RelOperator) -> String { RelOperator::Mutation(_) => "MergeInto".to_string(), RelOperator::MutationSource(_) => "MutationSource".to_string(), RelOperator::CompactBlock(_) => "CompactBlock".to_string(), + RelOperator::MaterializedCTE(_) => "MaterializedCTE".to_string(), + RelOperator::CTEConsumer(_) => "CTEConsumer".to_string(), } } diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 868b55f05e935..4d8c1d4daac5a 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -270,6 +270,18 @@ impl DPhpyOptimizer { Ok((new_s_expr, left_res.1 && right_res.1)) } + async fn process_materialized_cte_node( + &mut self, + s_expr: &SExpr, + ) -> Result<(Arc, bool)> { + let new_s_expr = self.new_children(s_expr).await?; + Ok((Arc::new(new_s_expr), true)) + } + + async fn process_cte_consumer_node(&mut self, s_expr: &SExpr) -> Result<(Arc, bool)> { + Ok((Arc::new(s_expr.clone()), true)) + } + /// Process a unary operator node async fn process_unary_node( &mut self, @@ -332,6 +344,9 @@ impl DPhpyOptimizer { RelOperator::Join(_) => self.process_join_node(s_expr, join_conditions).await, + RelOperator::MaterializedCTE(_) => self.process_materialized_cte_node(s_expr).await, + RelOperator::CTEConsumer(_) => self.process_cte_consumer_node(s_expr).await, + RelOperator::ProjectSet(_) | RelOperator::Aggregate(_) | RelOperator::Sort(_) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs index eb645151c4230..cdc9071d13c84 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs @@ -92,6 +92,8 @@ pub async fn dynamic_sample( | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::CompactBlock(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::MutationSource(_) => { s_expr.plan().derive_stats(&RelExpr::with_s_expr(s_expr)) } diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs index 079f80afcb9fe..a3f091cda6831 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs @@ -325,6 +325,12 @@ impl SubqueryDecorrelatorOptimizer { )) } + RelOperator::MaterializedCTE(_) => Ok(SExpr::create_binary( + s_expr.plan.clone(), + Arc::new(self.optimize_sync(s_expr.left_child())?), + Arc::new(self.optimize_sync(s_expr.right_child())?), + )), + RelOperator::DummyTableScan(_) | RelOperator::Scan(_) | RelOperator::ConstantTableScan(_) @@ -334,6 +340,7 @@ impl SubqueryDecorrelatorOptimizer { | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::MutationSource(_) + | RelOperator::CTEConsumer(_) | RelOperator::CompactBlock(_) => Ok(s_expr.clone()), } } diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs index 46ebf4648860d..1cfa72a2ed63c 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs @@ -152,6 +152,8 @@ fn find_group_by_keys( | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::MutationSource(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::CompactBlock(_) => {} } Ok(()) diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs new file mode 100644 index 0000000000000..af3450e008e78 --- /dev/null +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -0,0 +1,88 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::hash::Hash; +use std::hash::Hasher; + +use databend_common_expression::DataField; + +use crate::plans::Operator; +use crate::plans::RelOp; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CTEConsumer { + pub cte_name: String, + pub fields: Vec, +} + +impl CTEConsumer { + pub fn new(cte_name: String, fields: Vec) -> Self { + Self { cte_name, fields } + } + + // pub fn used_columns(&self) -> Result { + // let mut used_columns = ColumnSet::new(); + // for field in self.fields.iter() { + // used_columns.insert(field.name().parse()?); + // } + // Ok(used_columns) + // } +} + +impl Hash for CTEConsumer { + fn hash(&self, state: &mut H) { + self.cte_name.hash(state); + for field in self.fields.iter() { + field.name().hash(state); + } + } +} + +impl Operator for CTEConsumer { + fn rel_op(&self) -> RelOp { + RelOp::CTEConsumer + } + + // fn arity(&self) -> usize { + // 0 + // } + + // fn derive_relational_prop(&self, _rel_expr: &RelExpr) -> Result> { + // Ok(Arc::new(RelationalProperty { + // output_columns: self.used_columns()?, + // outer_columns: ColumnSet::new(), + // used_columns: self.used_columns()?, + // orderings: vec![], + // partition_orderings: None, + // })) + // } + + // fn derive_physical_prop(&self, _rel_expr: &RelExpr) -> Result { + // Ok(PhysicalProperty { + // distribution: Distribution::Serial, + // }) + // } + + // fn compute_required_prop_child( + // &self, + // _ctx: Arc, + // _rel_expr: &RelExpr, + // _child_index: usize, + // _required: &RequiredProperty, + // ) -> Result { + // Err(ErrorCode::Internal( + // "Cannot compute required property for CTEConsumer".to_string(), + // )) + // } +} diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs new file mode 100644 index 0000000000000..b7793e8bd3546 --- /dev/null +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -0,0 +1,42 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::hash::Hash; +use std::hash::Hasher; + +use crate::plans::Operator; +use crate::plans::RelOp; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaterializedCTE { + pub cte_name: String, +} + +impl MaterializedCTE { + pub fn new(cte_name: String) -> Self { + Self { cte_name } + } +} + +impl Hash for MaterializedCTE { + fn hash(&self, state: &mut H) { + self.cte_name.hash(state); + } +} + +impl Operator for MaterializedCTE { + fn rel_op(&self) -> RelOp { + RelOp::MaterializedCTE + } +} diff --git a/src/query/sql/src/planner/plans/mod.rs b/src/query/sql/src/planner/plans/mod.rs index 2f324de8199d8..bcd237abc68a6 100644 --- a/src/query/sql/src/planner/plans/mod.rs +++ b/src/query/sql/src/planner/plans/mod.rs @@ -19,6 +19,7 @@ mod call; mod constant_table_scan; mod copy_into_location; mod copy_into_table; +mod cte_consumer; mod data_mask; mod ddl; mod dummy_table_scan; @@ -31,6 +32,7 @@ mod insert_multi_table; mod join; mod kill; mod limit; +mod materialized_cte; mod mutation; mod mutation_source; mod operator; @@ -61,6 +63,7 @@ pub use call::CallPlan; pub use constant_table_scan::ConstantTableScan; pub use copy_into_location::*; pub use copy_into_table::*; +pub use cte_consumer::*; pub use data_mask::*; pub use ddl::*; pub use dummy_table_scan::DummyTableScan; @@ -73,6 +76,7 @@ pub use insert_multi_table::*; pub use join::*; pub use kill::KillPlan; pub use limit::*; +pub use materialized_cte::*; pub use mutation::MatchedEvaluator; pub use mutation::Mutation; pub use mutation::UnmatchedEvaluator; diff --git a/src/query/sql/src/planner/plans/operator.rs b/src/query/sql/src/planner/plans/operator.rs index 3d3a67518fed0..d31852361cfd4 100644 --- a/src/query/sql/src/planner/plans/operator.rs +++ b/src/query/sql/src/planner/plans/operator.rs @@ -29,6 +29,7 @@ use crate::optimizer::ir::StatInfo; use crate::plans::r_cte_scan::RecursiveCteScan; use crate::plans::Aggregate; use crate::plans::AsyncFunction; +use crate::plans::CTEConsumer; use crate::plans::CacheScan; use crate::plans::ConstantTableScan; use crate::plans::DummyTableScan; @@ -38,6 +39,7 @@ use crate::plans::ExpressionScan; use crate::plans::Filter; use crate::plans::Join; use crate::plans::Limit; +use crate::plans::MaterializedCTE; use crate::plans::Mutation; use crate::plans::OptimizeCompactBlock; use crate::plans::ProjectSet; @@ -119,6 +121,8 @@ pub enum RelOp { MergeInto, CompactBlock, MutationSource, + MaterializedCTE, + CTEConsumer, // Pattern Pattern, @@ -155,6 +159,8 @@ pub enum RelOperator { Mutation(Mutation), CompactBlock(OptimizeCompactBlock), MutationSource(MutationSource), + MaterializedCTE(MaterializedCTE), + CTEConsumer(CTEConsumer), } impl RelOperator { @@ -172,6 +178,8 @@ impl RelOperator { | RelOperator::AsyncFunction(_) | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::CompactBlock(_) => false, RelOperator::Join(op) => op.has_subquery(), RelOperator::EvalScalar(op) => op.items.iter().any(|expr| expr.scalar.has_subquery()), @@ -219,6 +227,8 @@ impl RelOperator { | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::CompactBlock(_) + | RelOperator::MaterializedCTE(_) + | RelOperator::CTEConsumer(_) | RelOperator::MutationSource(_) => (), RelOperator::Join(op) => { for condition in &op.equi_conditions { @@ -299,6 +309,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.rel_op(), RelOperator::CompactBlock(rel_op) => rel_op.rel_op(), RelOperator::MutationSource(rel_op) => rel_op.rel_op(), + RelOperator::MaterializedCTE(rel_op) => rel_op.rel_op(), + RelOperator::CTEConsumer(rel_op) => rel_op.rel_op(), } } @@ -325,6 +337,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.arity(), RelOperator::CompactBlock(rel_op) => rel_op.arity(), RelOperator::MutationSource(rel_op) => rel_op.arity(), + RelOperator::MaterializedCTE(rel_op) => rel_op.arity(), + RelOperator::CTEConsumer(rel_op) => rel_op.arity(), } } @@ -351,6 +365,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.derive_relational_prop(rel_expr), RelOperator::CompactBlock(rel_op) => rel_op.derive_relational_prop(rel_expr), RelOperator::MutationSource(rel_op) => rel_op.derive_relational_prop(rel_expr), + RelOperator::MaterializedCTE(rel_op) => rel_op.derive_relational_prop(rel_expr), + RelOperator::CTEConsumer(rel_op) => rel_op.derive_relational_prop(rel_expr), } } @@ -377,6 +393,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.derive_physical_prop(rel_expr), RelOperator::CompactBlock(rel_op) => rel_op.derive_physical_prop(rel_expr), RelOperator::MutationSource(rel_op) => rel_op.derive_physical_prop(rel_expr), + RelOperator::MaterializedCTE(rel_op) => rel_op.derive_physical_prop(rel_expr), + RelOperator::CTEConsumer(rel_op) => rel_op.derive_physical_prop(rel_expr), } } @@ -403,6 +421,8 @@ impl Operator for RelOperator { RelOperator::Mutation(rel_op) => rel_op.derive_stats(rel_expr), RelOperator::CompactBlock(rel_op) => rel_op.derive_stats(rel_expr), RelOperator::MutationSource(rel_op) => rel_op.derive_stats(rel_expr), + RelOperator::MaterializedCTE(rel_op) => rel_op.derive_stats(rel_expr), + RelOperator::CTEConsumer(rel_op) => rel_op.derive_stats(rel_expr), } } @@ -477,6 +497,12 @@ impl Operator for RelOperator { RelOperator::MutationSource(rel_op) => { rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) } + RelOperator::MaterializedCTE(rel_op) => { + rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) + } + RelOperator::CTEConsumer(rel_op) => { + rel_op.compute_required_prop_child(ctx, rel_expr, child_index, required) + } } } @@ -550,6 +576,12 @@ impl Operator for RelOperator { RelOperator::MutationSource(rel_op) => { rel_op.compute_required_prop_children(ctx, rel_expr, required) } + RelOperator::MaterializedCTE(rel_op) => { + rel_op.compute_required_prop_children(ctx, rel_expr, required) + } + RelOperator::CTEConsumer(rel_op) => { + rel_op.compute_required_prop_children(ctx, rel_expr, required) + } } } } From 7e986a974c459c428c14eec042452f31b360530c Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 23 Jun 2025 15:39:41 +0800 Subject: [PATCH 05/80] remove m cte temp table --- src/query/catalog/src/table_context.rs | 4 - .../src/interpreters/hook/compact_hook.rs | 2 - .../src/interpreters/hook/refresh_hook.rs | 3 - .../src/interpreters/hook/vacuum_hook.rs | 8 - .../service/src/interpreters/interpreter.rs | 2 - .../interpreters/interpreter_table_create.rs | 2 - .../interpreter_table_recluster.rs | 2 - src/query/service/src/interpreters/mod.rs | 1 - .../src/servers/http/v1/query/http_query.rs | 3 - src/query/service/src/sessions/query_ctx.rs | 45 ----- .../tests/it/sql/exec/get_table_bind_test.rs | 8 - .../it/storages/fuse/operations/commit.rs | 9 +- .../sql/src/planner/binder/bind_query/bind.rs | 170 ------------------ 13 files changed, 1 insertion(+), 258 deletions(-) diff --git a/src/query/catalog/src/table_context.rs b/src/query/catalog/src/table_context.rs index d4dc75c7d0215..42b9141b5e469 100644 --- a/src/query/catalog/src/table_context.rs +++ b/src/query/catalog/src/table_context.rs @@ -403,10 +403,6 @@ pub trait TableContext: Send + Sync { fn is_temp_table(&self, catalog_name: &str, database_name: &str, table_name: &str) -> bool; fn get_shared_settings(&self) -> Arc; - fn add_m_cte_temp_table(&self, database_name: &str, table_name: &str); - - async fn drop_m_cte_temp_table(&self) -> Result<()>; - fn add_streams_ref(&self, _catalog: &str, _database: &str, _stream: &str, _consume: bool) { unimplemented!() } diff --git a/src/query/service/src/interpreters/hook/compact_hook.rs b/src/query/service/src/interpreters/hook/compact_hook.rs index 0029a2b0ff0b1..e1da58ed2147d 100644 --- a/src/query/service/src/interpreters/hook/compact_hook.rs +++ b/src/query/service/src/interpreters/hook/compact_hook.rs @@ -33,7 +33,6 @@ use log::info; use crate::interpreters::common::metrics_inc_compact_hook_compact_time_ms; use crate::interpreters::common::metrics_inc_compact_hook_main_operation_time_ms; -use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::Interpreter; @@ -183,7 +182,6 @@ async fn compact_table( let query_ctx = ctx.clone(); build_res.main_pipeline.set_on_finished(always_callback( move |_info: &ExecutionInfo| { - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; Ok(()) diff --git a/src/query/service/src/interpreters/hook/refresh_hook.rs b/src/query/service/src/interpreters/hook/refresh_hook.rs index 3589c79459a59..64a3bc3920f13 100644 --- a/src/query/service/src/interpreters/hook/refresh_hook.rs +++ b/src/query/service/src/interpreters/hook/refresh_hook.rs @@ -38,7 +38,6 @@ use databend_storages_common_table_meta::meta::Location; use log::info; use parking_lot::RwLock; -use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::Interpreter; @@ -125,7 +124,6 @@ async fn do_refresh(ctx: Arc, desc: RefreshDesc) -> Result<()> { let query_ctx = ctx_cloned.clone(); build_res.main_pipeline.set_on_finished(always_callback( move |_: &ExecutionInfo| { - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; Ok(()) @@ -162,7 +160,6 @@ async fn do_refresh(ctx: Arc, desc: RefreshDesc) -> Result<()> { let query_ctx = ctx_cloned.clone(); build_res.main_pipeline.set_on_finished(always_callback( move |_info: &ExecutionInfo| { - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; Ok(()) diff --git a/src/query/service/src/interpreters/hook/vacuum_hook.rs b/src/query/service/src/interpreters/hook/vacuum_hook.rs index b1f0aaa977ecb..93b886094e4c1 100644 --- a/src/query/service/src/interpreters/hook/vacuum_hook.rs +++ b/src/query/service/src/interpreters/hook/vacuum_hook.rs @@ -102,11 +102,3 @@ pub fn hook_disk_temp_dir(query_ctx: &Arc) -> Result<()> { Ok(()) } - -pub fn hook_clear_m_cte_temp_table(query_ctx: &Arc) -> Result<()> { - let _ = GlobalIORuntime::instance().block_on(async move { - query_ctx.drop_m_cte_temp_table().await?; - Ok(()) - }); - Ok(()) -} diff --git a/src/query/service/src/interpreters/interpreter.rs b/src/query/service/src/interpreters/interpreter.rs index 1d747ba85ad4b..c3c0a98c79113 100644 --- a/src/query/service/src/interpreters/interpreter.rs +++ b/src/query/service/src/interpreters/interpreter.rs @@ -49,7 +49,6 @@ use log::info; use md5::Digest; use md5::Md5; -use super::hook::vacuum_hook::hook_clear_m_cte_temp_table; use super::hook::vacuum_hook::hook_disk_temp_dir; use super::hook::vacuum_hook::hook_vacuum_temp_files; use super::InterpreterMetrics; @@ -363,7 +362,6 @@ pub fn on_execution_finished(info: &ExecutionInfo, query_ctx: Arc) ); } - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; diff --git a/src/query/service/src/interpreters/interpreter_table_create.rs b/src/query/service/src/interpreters/interpreter_table_create.rs index 0886f95104fef..920ae8e6d9290 100644 --- a/src/query/service/src/interpreters/interpreter_table_create.rs +++ b/src/query/service/src/interpreters/interpreter_table_create.rs @@ -74,7 +74,6 @@ use crate::interpreters::common::table_option_validation::is_valid_data_retentio use crate::interpreters::common::table_option_validation::is_valid_option_of_type; use crate::interpreters::common::table_option_validation::is_valid_random_seed; use crate::interpreters::common::table_option_validation::is_valid_row_per_block; -use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::InsertInterpreter; @@ -280,7 +279,6 @@ impl CreateTableInterpreter { pipeline .main_pipeline .set_on_finished(always_callback(move |_: &ExecutionInfo| { - hook_clear_m_cte_temp_table(&query_ctx)?; hook_vacuum_temp_files(&query_ctx)?; hook_disk_temp_dir(&query_ctx)?; Ok(()) diff --git a/src/query/service/src/interpreters/interpreter_table_recluster.rs b/src/query/service/src/interpreters/interpreter_table_recluster.rs index f3c53597b06d7..3c27d2ba621f9 100644 --- a/src/query/service/src/interpreters/interpreter_table_recluster.rs +++ b/src/query/service/src/interpreters/interpreter_table_recluster.rs @@ -68,7 +68,6 @@ use derive_visitor::DriveMut; use log::error; use log::warn; -use crate::interpreters::hook::vacuum_hook::hook_clear_m_cte_temp_table; use crate::interpreters::hook::vacuum_hook::hook_disk_temp_dir; use crate::interpreters::hook::vacuum_hook::hook_vacuum_temp_files; use crate::interpreters::interpreter_insert_multi_table::scalar_expr_to_remote_expr; @@ -247,7 +246,6 @@ impl ReclusterTableInterpreter { ctx.evict_table_from_cache(&catalog, &database, &table)?; ctx.unload_spill_meta(); - hook_clear_m_cte_temp_table(&ctx)?; hook_vacuum_temp_files(&ctx)?; hook_disk_temp_dir(&ctx)?; match &info.res { diff --git a/src/query/service/src/interpreters/mod.rs b/src/query/service/src/interpreters/mod.rs index 22b7783f332a1..1587886a868b7 100644 --- a/src/query/service/src/interpreters/mod.rs +++ b/src/query/service/src/interpreters/mod.rs @@ -173,7 +173,6 @@ mod util; pub use access::ManagementModeAccess; pub use common::InterpreterQueryLog; -pub use hook::vacuum_hook::hook_clear_m_cte_temp_table; pub use hook::HookOperator; pub use interpreter::interpreter_plan_sql; pub use interpreter::Interpreter; diff --git a/src/query/service/src/servers/http/v1/query/http_query.rs b/src/query/service/src/servers/http/v1/query/http_query.rs index 147f38520d77a..e6f1888c5f2ad 100644 --- a/src/query/service/src/servers/http/v1/query/http_query.rs +++ b/src/query/service/src/servers/http/v1/query/http_query.rs @@ -791,9 +791,6 @@ impl HttpQuery { .with_context(|| "failed to start query") .flatten() { - crate::interpreters::hook_clear_m_cte_temp_table(&query_context) - .inspect_err(|e| warn!("clear_m_cte_temp_table fail: {e}")) - .ok(); let state = ExecuteStopped { stats: Progresses::default(), schema: vec![], diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index ef41cfd4048d9..cbd2898980ba6 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -89,7 +89,6 @@ use databend_common_meta_app::principal::UserPrivilegeType; use databend_common_meta_app::principal::COPY_MAX_FILES_COMMIT_MSG; use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use databend_common_meta_app::schema::CatalogType; -use databend_common_meta_app::schema::DropTableByIdReq; use databend_common_meta_app::schema::GetTableCopiedFileReq; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::storage::StorageParams; @@ -122,13 +121,11 @@ use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::UserApiProvider; use databend_common_version::DATABEND_COMMIT_VERSION; use databend_common_version::DATABEND_ENTERPRISE_LICENSE_EMBEDDED; -use databend_storages_common_session::drop_table_by_id; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::TableMetaTimestamps; use databend_storages_common_table_meta::meta::TableSnapshot; -use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX; use jiff::tz::TimeZone; use jiff::Zoned; use log::debug; @@ -168,9 +165,6 @@ pub struct QueryContext { fragment_id: Arc, // Used by synchronized generate aggregating indexes when new data written. written_segment_locs: Arc>>, - // Temp table for materialized CTE, first string is the database_name, second string is the table_name - // All temp tables' catalog is `CATALOG_DEFAULT`, so we don't need to store it. - m_cte_temp_table: Arc>>, } impl QueryContext { @@ -196,7 +190,6 @@ impl QueryContext { fragment_id: Arc::new(AtomicUsize::new(0)), written_segment_locs: Default::default(), block_threshold: Default::default(), - m_cte_temp_table: Default::default(), }) } @@ -1827,44 +1820,6 @@ impl TableContext for QueryContext { .is_temp_table(database_name, table_name) } - fn add_m_cte_temp_table(&self, database_name: &str, table_name: &str) { - self.m_cte_temp_table - .write() - .push((database_name.to_string(), table_name.to_string())); - } - - async fn drop_m_cte_temp_table(&self) -> Result<()> { - let temp_tbl_mgr = self.shared.session.session_ctx.temp_tbl_mgr(); - let m_cte_temp_table = self.m_cte_temp_table.read().clone(); - let tenant = self.get_tenant(); - for (db_name, table_name) in m_cte_temp_table.iter() { - let table = self.get_table(CATALOG_DEFAULT, db_name, table_name).await?; - let db = self - .get_catalog(CATALOG_DEFAULT) - .await? - .get_database(&tenant, db_name) - .await?; - let drop_table_req = DropTableByIdReq { - if_exists: true, - tenant: tenant.clone(), - tb_id: table.get_table_info().ident.table_id, - table_name: table_name.to_string(), - db_id: db.get_db_info().database_id.db_id, - db_name: db.name().to_string(), - engine: table.engine().to_string(), - temp_prefix: table - .options() - .get(OPT_KEY_TEMP_PREFIX) - .cloned() - .unwrap_or_default(), - }; - drop_table_by_id(temp_tbl_mgr.clone(), drop_table_req).await?; - } - let mut m_cte_temp_table = self.m_cte_temp_table.write(); - m_cte_temp_table.clear(); - Ok(()) - } - fn add_streams_ref(&self, catalog: &str, database: &str, stream: &str, consume: bool) { let mut streams = self.shared.streams_refs.write(); let stream_key = ( diff --git a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs index 978f0fc79b8d8..79cbad0ff12ee 100644 --- a/src/query/service/tests/it/sql/exec/get_table_bind_test.rs +++ b/src/query/service/tests/it/sql/exec/get_table_bind_test.rs @@ -995,14 +995,6 @@ impl TableContext for CtxDelegation { false } - fn add_m_cte_temp_table(&self, _database_name: &str, _table_name: &str) { - todo!() - } - - async fn drop_m_cte_temp_table(&self) -> Result<()> { - todo!() - } - fn set_cluster(&self, _: Arc) { todo!() } diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 1fd9bdcfa23b9..7db1e682e7d3c 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -869,18 +869,11 @@ impl TableContext for CtxDelegation { fn is_temp_table(&self, _catalog_name: &str, _database_name: &str, _table_name: &str) -> bool { false } - fn add_m_cte_temp_table(&self, _database_name: &str, _table_name: &str) { - todo!() - } - - async fn drop_m_cte_temp_table(&self) -> Result<()> { - todo!() - } fn set_cluster(&self, _: Arc) { todo!() } - + async fn get_warehouse_cluster(&self) -> Result> { todo!() } diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 46867955cb746..72ee1d755812f 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -15,28 +15,15 @@ use std::collections::HashMap; use std::sync::Arc; -use databend_common_ast::ast::ColumnDefinition; -use databend_common_ast::ast::CreateOption; -use databend_common_ast::ast::CreateTableSource; -use databend_common_ast::ast::CreateTableStmt; -use databend_common_ast::ast::Engine; use databend_common_ast::ast::Expr; -use databend_common_ast::ast::Identifier; use databend_common_ast::ast::Query; use databend_common_ast::ast::SetExpr; use databend_common_ast::ast::TableReference; -use databend_common_ast::ast::TableType; use databend_common_ast::ast::With; -use databend_common_ast::ast::CTE; -use databend_common_ast::Span; -use databend_common_catalog::catalog::CATALOG_DEFAULT; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::types::convert_to_type_name; use derive_visitor::Drive; -use derive_visitor::DriveMut; use derive_visitor::Visitor; -use derive_visitor::VisitorMut; use crate::binder::CteInfo; use crate::normalize_identifier; @@ -163,10 +150,6 @@ impl Binder { columns: vec![], materialized: cte.materialized, }; - // If the CTE is materialized, we'll construct a temp table for it. - if cte.materialized { - self.m_cte_to_temp_table(cte, idx, with.clone())?; - } bind_context .cte_context .cte_map @@ -243,157 +226,4 @@ impl Binder { Arc::new(child), )) } - - fn m_cte_to_temp_table(&mut self, cte: &CTE, cte_index: usize, mut with: With) -> Result<()> { - let engine = if self.ctx.get_settings().get_persist_materialized_cte()? { - Engine::Fuse - } else { - Engine::Memory - }; - let query_id = self.ctx.get_id(); - let database = self.ctx.get_current_database(); - let mut table_identifier = cte.alias.name.clone(); - table_identifier.name = format!("{}${}", table_identifier.name, query_id.replace("-", "")); - let table_name = normalize_identifier(&table_identifier, &self.name_resolution_ctx).name; - self.m_cte_table_name.insert( - normalize_identifier(&cte.alias.name, &self.name_resolution_ctx).name, - table_name.clone(), - ); - if self - .ctx - .is_temp_table(CATALOG_DEFAULT, &database, &table_name) - { - return Err(ErrorCode::Internal(format!( - "Temporary table {:?} already exists in current session, please change the materialized CTE name", - table_name - ))); - } - - let mut expr_replacer = TableNameReplacer::new( - database.clone(), - self.m_cte_table_name.clone(), - self.name_resolution_ctx.clone(), - ); - let mut as_query = cte.query.clone(); - with.ctes.truncate(cte_index); - with.ctes.retain(|cte| !cte.materialized); - as_query.with = if !with.ctes.is_empty() { - Some(with) - } else { - None - }; - as_query.drive_mut(&mut expr_replacer); - - let source = if cte.alias.columns.is_empty() { - None - } else { - let mut bind_context = BindContext::new(); - let (_, bind_context) = self.bind_query(&mut bind_context, &as_query)?; - let columns = &bind_context.columns; - if columns.len() != cte.alias.columns.len() { - return Err(ErrorCode::Internal("Number of columns does not match")); - } - Some(CreateTableSource::Columns( - columns - .iter() - .zip(cte.alias.columns.iter()) - .map(|(column, ident)| { - let data_type = convert_to_type_name(&column.data_type); - ColumnDefinition { - name: ident.clone(), - data_type, - expr: None, - comment: None, - } - }) - .collect(), - None, - )) - }; - - let catalog = self.ctx.get_current_catalog(); - let create_table_stmt = CreateTableStmt { - create_option: CreateOption::Create, - catalog: Some(Identifier::from_name(Span::None, catalog.clone())), - database: Some(Identifier::from_name(Span::None, database.clone())), - table: table_identifier, - source, - engine: Some(engine), - uri_location: None, - cluster_by: None, - table_options: Default::default(), - iceberg_table_partition: None, - table_properties: Default::default(), - as_query: Some(as_query), - table_type: TableType::Temporary, - }; - - let create_table_sql = create_table_stmt.to_string(); - log::info!("[CTE]create_table_sql: {create_table_sql}"); - if let Some(subquery_executor) = &self.subquery_executor { - let _ = databend_common_base::runtime::block_on(async move { - subquery_executor - .execute_query_with_sql_string(&create_table_sql) - .await - })?; - } else { - return Err(ErrorCode::Internal("Binder's Subquery executor is not set")); - }; - - self.ctx.add_m_cte_temp_table(&database, &table_name); - - self.ctx - .evict_table_from_cache(&catalog, &database, &table_name) - } -} - -#[derive(VisitorMut)] -#[visitor(TableReference(enter), Expr(enter))] -pub struct TableNameReplacer { - database: String, - new_name: HashMap, - name_resolution_ctx: NameResolutionContext, -} - -impl TableNameReplacer { - pub fn new( - database: String, - new_name: HashMap, - name_resolution_ctx: NameResolutionContext, - ) -> Self { - Self { - database, - new_name, - name_resolution_ctx, - } - } - - fn replace_identifier(&mut self, identifier: &mut Identifier) { - let name = normalize_identifier(identifier, &self.name_resolution_ctx).name; - if let Some(new_name) = self.new_name.get(&name) { - identifier.name = new_name.clone(); - } - } - - fn enter_table_reference(&mut self, table_reference: &mut TableReference) { - if let TableReference::Table { - database, table, .. - } = table_reference - { - if database.is_none() || database.as_ref().unwrap().name == self.database { - self.replace_identifier(table); - } - } - } - - fn enter_expr(&mut self, expr: &mut Expr) { - if let Expr::ColumnRef { column, .. } = expr { - if column.database.is_none() || column.database.as_ref().unwrap().name == self.database - { - if let Some(table_identifier) = &mut column.table { - self.replace_identifier(table_identifier); - } - } - } - } } From e4ee842503209327e7a6ec369223518b74e3de5c Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 23 Jun 2025 20:49:53 +0800 Subject: [PATCH 06/80] bind --- .../it/storages/fuse/operations/commit.rs | 2 +- .../sql/src/planner/binder/bind_context.rs | 1 + .../sql/src/planner/binder/bind_query/bind.rs | 42 +++++++++++++++++++ .../binder/bind_table_reference/bind_table.rs | 19 ++++----- .../bind_table_function.rs | 2 - src/query/sql/src/planner/binder/table.rs | 1 - src/query/sql/src/planner/dataframe.rs | 1 - .../planner/expression/expression_parser.rs | 1 - .../sql/src/planner/metadata/metadata.rs | 11 ----- .../sql/src/planner/plans/cte_consumer.rs | 37 +++++----------- .../sql/src/planner/plans/materialized_cte.rs | 9 +--- src/query/sql/src/planner/plans/operator.rs | 20 +++++++++ 12 files changed, 83 insertions(+), 63 deletions(-) diff --git a/src/query/service/tests/it/storages/fuse/operations/commit.rs b/src/query/service/tests/it/storages/fuse/operations/commit.rs index 7db1e682e7d3c..4bbb7fdf2a47b 100644 --- a/src/query/service/tests/it/storages/fuse/operations/commit.rs +++ b/src/query/service/tests/it/storages/fuse/operations/commit.rs @@ -873,7 +873,7 @@ impl TableContext for CtxDelegation { fn set_cluster(&self, _: Arc) { todo!() } - + async fn get_warehouse_cluster(&self) -> Result> { todo!() } diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 2cf026e08983e..3f74fe384b8a6 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -160,6 +160,7 @@ pub struct CteContext { /// If the `BindContext` is created from a CTE, record the cte name pub cte_name: Option, pub cte_map: Box>, + pub is_binding_materialized_cte: bool, } impl CteContext { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 72ee1d755812f..83d3b0a6e16b2 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -31,7 +31,9 @@ use crate::optimizer::ir::SExpr; use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; use crate::planner::binder::Binder; +use crate::planner::binder::CteContext; use crate::plans::BoundColumnRef; +use crate::plans::MaterializedCTE; use crate::plans::ScalarExpr; use crate::plans::Sort; use crate::plans::SortItem; @@ -85,6 +87,10 @@ impl Binder { // Bind limit. s_expr = self.bind_query_limit(query, s_expr, limit, offset); + if let Some(with) = &with { + s_expr = self.bind_materialized_cte(with, s_expr, bind_context.cte_context.clone())?; + } + Ok((s_expr, bind_context)) } @@ -226,4 +232,40 @@ impl Binder { Arc::new(child), )) } + + fn bind_materialized_cte( + &mut self, + with: &With, + main_query_expr: SExpr, + mut cte_context: CteContext, + ) -> Result { + let mut current_expr = main_query_expr; + cte_context.is_binding_materialized_cte = true; + + for cte in with.ctes.iter().rev() { + if cte.materialized { + let cte_name = self.normalize_identifier(&cte.alias.name).name; + + // Create a new bind context for the CTE definition + let mut cte_bind_context = BindContext { + cte_context: cte_context.clone(), + ..Default::default() + }; + + // Bind the CTE definition + let (cte_definition_expr, _) = + self.bind_query(&mut cte_bind_context, &cte.query)?; + + // Create the MaterializedCTE operator + let materialized_cte = MaterializedCTE::new(cte_name); + current_expr = SExpr::create_binary( + Arc::new(materialized_cte.into()), + Arc::new(cte_definition_expr), + Arc::new(current_expr), + ); + } + } + + Ok(current_expr) + } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 7848170697095..685cbf6f6eaf5 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; + use databend_common_ast::ast::Identifier; use databend_common_ast::ast::SampleConfig; use databend_common_ast::ast::Statement; @@ -33,6 +35,8 @@ use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; use crate::optimizer::ir::SExpr; +use crate::plans::CTEConsumer; +use crate::plans::RelOperator; use crate::BindContext; impl Binder { @@ -69,12 +73,13 @@ impl Binder { (false, None, String::new()) }; - // Check and bind common table expression - let mut cte_suffix_name = None; let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - cte_suffix_name = Some(self.ctx.get_id().replace("-", "")); + let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { + cte_name: table_name, + }))); + return Ok((s_expr, bind_context.clone())); } else { if self .metadata @@ -104,11 +109,6 @@ impl Binder { // Resolve table with catalog let table_meta = { - let table_name = if let Some(cte_suffix_name) = cte_suffix_name.as_ref() { - format!("{}${}", &table_name, cte_suffix_name) - } else { - table_name.clone() - }; match self.resolve_data_source( catalog.as_str(), database.as_str(), @@ -161,7 +161,6 @@ impl Binder { bind_context.view_info.is_some(), bind_context.planning_agg_index, false, - None, false, ); let (s_expr, mut bind_context) = self.bind_base_table( @@ -247,7 +246,6 @@ impl Binder { false, false, false, - None, bind_context.allow_virtual_column, ); let (s_expr, mut new_bind_context) = @@ -280,7 +278,6 @@ impl Binder { bind_context.view_info.is_some(), bind_context.planning_agg_index, false, - cte_suffix_name, bind_context.allow_virtual_column, ); diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs index 3cfe23223548f..31c9b8bcbd0df 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table_function.rs @@ -151,7 +151,6 @@ impl Binder { false, false, false, - None, false, ); @@ -214,7 +213,6 @@ impl Binder { false, false, false, - None, false, ); diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index 23ff11114564a..e9785f40f5922 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -142,7 +142,6 @@ impl Binder { false, false, true, - None, bind_context.allow_virtual_column, ); diff --git a/src/query/sql/src/planner/dataframe.rs b/src/query/sql/src/planner/dataframe.rs index 277a010eace81..98ac53efc657d 100644 --- a/src/query/sql/src/planner/dataframe.rs +++ b/src/query/sql/src/planner/dataframe.rs @@ -101,7 +101,6 @@ impl Dataframe { false, false, false, - None, false, ); diff --git a/src/query/sql/src/planner/expression/expression_parser.rs b/src/query/sql/src/planner/expression/expression_parser.rs index 52ed0cc99a6fb..8937e880ec55f 100644 --- a/src/query/sql/src/planner/expression/expression_parser.rs +++ b/src/query/sql/src/planner/expression/expression_parser.rs @@ -63,7 +63,6 @@ pub fn bind_table(table_meta: Arc) -> Result<(BindContext, MetadataRe false, false, false, - None, false, ); diff --git a/src/query/sql/src/planner/metadata/metadata.rs b/src/query/sql/src/planner/metadata/metadata.rs index 11e16b1541f7f..f7889a42c8447 100644 --- a/src/query/sql/src/planner/metadata/metadata.rs +++ b/src/query/sql/src/planner/metadata/metadata.rs @@ -341,11 +341,9 @@ impl Metadata { source_of_view: bool, source_of_index: bool, source_of_stage: bool, - cte_suffix_name: Option, allow_virtual_column: bool, ) -> IndexType { let table_name = table_meta.name().to_string(); - let table_name = Self::remove_cte_suffix(table_name, cte_suffix_name); let table_index = self.tables.len(); // If exists table alias name, use it instead of origin name @@ -539,15 +537,6 @@ impl Metadata { self.base_column_scan_id.get(&column_index).cloned() } - fn remove_cte_suffix(mut table_name: String, cte_suffix_name: Option) -> String { - if let Some(suffix) = cte_suffix_name { - if table_name.ends_with(&suffix) { - table_name.truncate(table_name.len() - suffix.len() - 1); - } - } - table_name - } - pub fn replace_all_tables(&mut self, table: Arc) { for entry in self.tables.iter_mut() { entry.table = table.clone(); diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index af3450e008e78..3cedd1cb6cda5 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -13,41 +13,24 @@ // limitations under the License. use std::hash::Hash; -use std::hash::Hasher; - -use databend_common_expression::DataField; use crate::plans::Operator; use crate::plans::RelOp; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct CTEConsumer { pub cte_name: String, - pub fields: Vec, -} - -impl CTEConsumer { - pub fn new(cte_name: String, fields: Vec) -> Self { - Self { cte_name, fields } - } - - // pub fn used_columns(&self) -> Result { - // let mut used_columns = ColumnSet::new(); - // for field in self.fields.iter() { - // used_columns.insert(field.name().parse()?); - // } - // Ok(used_columns) - // } } -impl Hash for CTEConsumer { - fn hash(&self, state: &mut H) { - self.cte_name.hash(state); - for field in self.fields.iter() { - field.name().hash(state); - } - } -} +// impl CTEConsumer { +// // pub fn used_columns(&self) -> Result { +// // let mut used_columns = ColumnSet::new(); +// // for field in self.fields.iter() { +// // used_columns.insert(field.name().parse()?); +// // } +// // Ok(used_columns) +// // } +// } impl Operator for CTEConsumer { fn rel_op(&self) -> RelOp { diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index b7793e8bd3546..3a4193c43926a 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -13,12 +13,11 @@ // limitations under the License. use std::hash::Hash; -use std::hash::Hasher; use crate::plans::Operator; use crate::plans::RelOp; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, } @@ -29,12 +28,6 @@ impl MaterializedCTE { } } -impl Hash for MaterializedCTE { - fn hash(&self, state: &mut H) { - self.cte_name.hash(state); - } -} - impl Operator for MaterializedCTE { fn rel_op(&self) -> RelOp { RelOp::MaterializedCTE diff --git a/src/query/sql/src/planner/plans/operator.rs b/src/query/sql/src/planner/plans/operator.rs index d31852361cfd4..72d23ace98c73 100644 --- a/src/query/sql/src/planner/plans/operator.rs +++ b/src/query/sql/src/planner/plans/operator.rs @@ -970,3 +970,23 @@ impl TryFrom for MutationSource { } } } + +impl From for RelOperator { + fn from(v: MaterializedCTE) -> Self { + Self::MaterializedCTE(v) + } +} + +impl TryFrom for MaterializedCTE { + type Error = ErrorCode; + fn try_from(value: RelOperator) -> Result { + if let RelOperator::MaterializedCTE(value) = value { + Ok(value) + } else { + Err(ErrorCode::Internal(format!( + "Cannot downcast {:?} to MaterializedCTE", + value.rel_op() + ))) + } + } +} From e5d7472b467e8eced8bcc9b291b5929a330d5813 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 23 Jun 2025 22:48:06 +0800 Subject: [PATCH 07/80] fix --- .../src/pipelines/builders/builder_materialized_cte.rs | 8 ++------ src/query/service/src/sessions/query_ctx.rs | 2 -- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 02e42f3ee885f..01db9afb7cf61 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -25,12 +25,8 @@ impl PipelineBuilder { pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { // init builder for cte pipeline let sub_context = QueryContext::create_from(self.ctx.as_ref()); - let sub_builder = PipelineBuilder::create( - self.func_ctx.clone(), - self.settings.clone(), - sub_context, - self.main_pipeline.get_scopes(), - ); + let sub_builder = + PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); // build cte pipeline let mut build_res = sub_builder.finalize(&cte.left)?; diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 10652f5e93ad9..1d5731a56f6d9 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -120,8 +120,6 @@ use databend_common_storages_stream::stream_table::StreamTable; use databend_common_users::GrantObjectVisibilityChecker; use databend_common_users::UserApiProvider; use databend_common_version::DATABEND_COMMIT_VERSION; -use databend_common_version::DATABEND_ENTERPRISE_LICENSE_EMBEDDED; -use databend_storages_common_session::drop_table_by_id; use databend_storages_common_session::SessionState; use databend_storages_common_session::TxnManagerRef; use databend_storages_common_table_meta::meta::Location; From 9a53ba2d80413f5322c9dbb3a9821b18c5b5b3b3 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 29 Jun 2025 17:10:02 +0800 Subject: [PATCH 08/80] remove unused field --- src/query/sql/src/planner/binder/bind_context.rs | 1 - src/query/sql/src/planner/binder/bind_query/bind.rs | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 3f74fe384b8a6..d1ac8e3430a99 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -197,7 +197,6 @@ pub struct CteInfo { pub query: Query, pub materialized: bool, pub recursive: bool, - pub cte_idx: IndexType, // If cte is materialized, save its columns pub columns: Vec, } diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 83d3b0a6e16b2..9ef31759beab9 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -135,7 +135,7 @@ impl Binder { return Ok(()); }; - for (idx, cte) in with.ctes.iter().enumerate() { + for cte in with.ctes.iter() { let table_name = self.normalize_identifier(&cte.alias.name).name; if bind_context.cte_context.cte_map.contains_key(&table_name) { return Err(ErrorCode::SemanticError(format!( @@ -152,7 +152,6 @@ impl Binder { columns_alias: column_name, query: *cte.query.clone(), recursive: with.recursive, - cte_idx: idx, columns: vec![], materialized: cte.materialized, }; From ff73950677075a6635e6e3de9d3a811a877f94d1 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 08:59:54 +0800 Subject: [PATCH 09/80] fix bind --- .../binder/bind_table_reference/bind_table.rs | 81 ++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 685cbf6f6eaf5..c8fd57af473a5 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeMap; use std::sync::Arc; +use dashmap::DashMap; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::SampleConfig; use databend_common_ast::ast::Statement; @@ -34,10 +36,12 @@ use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; +use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::CTEConsumer; use crate::plans::RelOperator; use crate::BindContext; +use crate::binder::ExprContext; impl Binder { /// Bind a base table. @@ -76,10 +80,85 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { + // For materialized CTE, we need to add column bindings to the bind_context + // so that column references can be resolved correctly + let mut new_bind_context = bind_context.clone(); + + // We need to bind the CTE definition to get the columns + // This is similar to how non-materialized CTEs are handled + let mut cte_bind_context = BindContext { + parent: Some(Box::new(bind_context.clone())), + bound_internal_columns: BTreeMap::new(), + columns: vec![], + aggregate_info: Default::default(), + windows: Default::default(), + srf_info: Default::default(), + cte_context: bind_context.cte_context.clone(), + in_grouping: false, + view_info: None, + have_async_func: false, + have_udf_script: false, + have_udf_server: false, + inverted_index_map: Box::default(), + allow_virtual_column: false, + expr_context: ExprContext::default(), + planning_agg_index: false, + window_definitions: DashMap::new(), + }; + + cte_bind_context.cte_context.cte_name = Some(table_name.to_string()); + + // Bind the CTE definition to get the columns + let (_, mut res_bind_context) = + self.bind_query(&mut cte_bind_context, &cte_info.query)?; + + // Apply column aliases + let mut cols_alias = cte_info.columns_alias.clone(); + if let Some(alias) = alias { + for (idx, col_alias) in alias.columns.iter().enumerate() { + if idx < cte_info.columns_alias.len() { + cols_alias[idx] = col_alias.name.clone(); + } else { + cols_alias.push(col_alias.name.clone()); + } + } + } + + let alias_table_name = alias + .as_ref() + .map(|alias| normalize_identifier(&alias.name, &self.name_resolution_ctx).name) + .unwrap_or_else(|| table_name.to_string()); + + for column in res_bind_context.columns.iter_mut() { + column.database_name = None; + column.table_name = Some(alias_table_name.clone()); + } + + if cols_alias.len() > res_bind_context.columns.len() { + return Err(ErrorCode::SemanticError(format!( + "The CTE '{}' has {} columns, but {} aliases were provided. Ensure the number of aliases matches the number of columns in the CTE.", + table_name, + res_bind_context.columns.len(), + cols_alias.len() + )) + .set_span(*span)); + } + + for (index, column_name) in cols_alias.iter().enumerate() { + res_bind_context.columns[index].column_name = column_name.clone(); + } + + log::info!("[CTE] columns: {:?}", res_bind_context.columns); + + // Add the columns to the new bind context + for column in res_bind_context.columns { + new_bind_context.add_column_binding(column); + } + let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, }))); - return Ok((s_expr, bind_context.clone())); + return Ok((s_expr, new_bind_context)); } else { if self .metadata From 519713429e9379142d8ce04e2b808727188d5d6e Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 09:50:56 +0800 Subject: [PATCH 10/80] fix schema --- .../physical_plans/physical_cte_consumer.rs | 2 +- .../binder/bind_table_reference/bind_table.rs | 18 +++++++++++++++-- .../sql/src/planner/plans/cte_consumer.rs | 20 +++++++++---------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs index 819c5437ac5cb..cfb418c43fae2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs @@ -46,7 +46,7 @@ impl PhysicalPlanBuilder { plan_id: 0, stat_info: Some(stat_info), cte_name: cte_consumer.cte_name.clone(), - cte_schema: Default::default(), + cte_schema: cte_consumer.cte_schema.clone(), }))) } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index c8fd57af473a5..66489f324b5fa 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -31,18 +31,19 @@ use databend_common_catalog::table_with_options::get_with_opt_consume; use databend_common_catalog::table_with_options::get_with_opt_max_batch_size; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_expression::DataField; +use databend_common_expression::DataSchemaRefExt; use databend_common_storages_view::view_table::QUERY; use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; +use crate::binder::ExprContext; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::CTEConsumer; use crate::plans::RelOperator; use crate::BindContext; -use crate::binder::ExprContext; - impl Binder { /// Bind a base table. /// A base table is a table that is not a view or CTE. @@ -148,6 +149,18 @@ impl Binder { res_bind_context.columns[index].column_name = column_name.clone(); } + let fields = res_bind_context + .columns + .iter() + .map(|column_binding| { + DataField::new( + &column_binding.index.to_string(), + *column_binding.data_type.clone(), + ) + }) + .collect(); + let cte_schema = DataSchemaRefExt::create(fields); + log::info!("[CTE] columns: {:?}", res_bind_context.columns); // Add the columns to the new bind context @@ -157,6 +170,7 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, + cte_schema: cte_schema, }))); return Ok((s_expr, new_bind_context)); } else { diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index 3cedd1cb6cda5..bb161ac105829 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -14,23 +14,23 @@ use std::hash::Hash; +use databend_common_expression::DataSchemaRef; + use crate::plans::Operator; use crate::plans::RelOp; +use std::hash::Hasher; -#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct CTEConsumer { pub cte_name: String, + pub cte_schema: DataSchemaRef, } -// impl CTEConsumer { -// // pub fn used_columns(&self) -> Result { -// // let mut used_columns = ColumnSet::new(); -// // for field in self.fields.iter() { -// // used_columns.insert(field.name().parse()?); -// // } -// // Ok(used_columns) -// // } -// } +impl Hash for CTEConsumer { + fn hash(&self, state: &mut H) { + self.cte_name.hash(state); + } +} impl Operator for CTEConsumer { fn rel_op(&self) -> RelOp { From ba5be420881941de370d41c878911e832e0aa05f Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 16:03:59 +0800 Subject: [PATCH 11/80] fix --- src/query/sql/src/executor/physical_plan_builder.rs | 2 +- .../executor/physical_plans/physical_materialized_cte.rs | 6 ++++-- src/query/sql/src/planner/binder/bind_query/bind.rs | 4 ++-- src/query/sql/src/planner/plans/materialized_cte.rs | 6 ++++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/query/sql/src/executor/physical_plan_builder.rs b/src/query/sql/src/executor/physical_plan_builder.rs index cc866a3f44b36..aa948994a22b5 100644 --- a/src/query/sql/src/executor/physical_plan_builder.rs +++ b/src/query/sql/src/executor/physical_plan_builder.rs @@ -130,7 +130,7 @@ impl PhysicalPlanBuilder { } RelOperator::CompactBlock(compact) => self.build_compact_block(compact).await, RelOperator::MaterializedCTE(materialized_cte) => { - self.build_materialized_cte(s_expr, materialized_cte, stat_info) + self.build_materialized_cte(s_expr, materialized_cte, stat_info, required) .await } RelOperator::CTEConsumer(cte_consumer) => { diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index 6b42cba900749..f34abf156aabd 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -19,6 +19,7 @@ use crate::executor::explain::PlanStatsInfo; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; +use crate::ColumnSet; /// This is a binary operator that executes its children in order (left to right), and returns the results of the right child #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] @@ -44,9 +45,10 @@ impl PhysicalPlanBuilder { s_expr: &SExpr, materialized_cte: &crate::plans::MaterializedCTE, stat_info: PlanStatsInfo, + required: ColumnSet, ) -> Result { - let left_side = Box::new(self.build(s_expr.child(0)?, Default::default()).await?); - let right_side = Box::new(self.build(s_expr.child(1)?, Default::default()).await?); + let left_side = Box::new(self.build(s_expr.child(0)?, materialized_cte.required.clone()).await?); + let right_side = Box::new(self.build(s_expr.child(1)?, required).await?); Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { plan_id: 0, stat_info: Some(stat_info), diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 9ef31759beab9..988e98a76fc02 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -252,11 +252,11 @@ impl Binder { }; // Bind the CTE definition - let (cte_definition_expr, _) = + let (cte_definition_expr, bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; // Create the MaterializedCTE operator - let materialized_cte = MaterializedCTE::new(cte_name); + let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); current_expr = SExpr::create_binary( Arc::new(materialized_cte.into()), Arc::new(cte_definition_expr), diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index 3a4193c43926a..c37b286ef9436 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -16,15 +16,17 @@ use std::hash::Hash; use crate::plans::Operator; use crate::plans::RelOp; +use crate::ColumnSet; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, + pub required: ColumnSet, } impl MaterializedCTE { - pub fn new(cte_name: String) -> Self { - Self { cte_name } + pub fn new(cte_name: String, required: ColumnSet) -> Self { + Self { cte_name, required } } } From 7690dbf8657c0ff7e1031d6e40e4713a18a0c764 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 16:14:20 +0800 Subject: [PATCH 12/80] make lint --- .../src/executor/physical_plans/physical_materialized_cte.rs | 5 ++++- .../src/planner/binder/bind_table_reference/bind_table.rs | 2 +- src/query/sql/src/planner/plans/cte_consumer.rs | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index f34abf156aabd..401278e26280d 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -47,7 +47,10 @@ impl PhysicalPlanBuilder { stat_info: PlanStatsInfo, required: ColumnSet, ) -> Result { - let left_side = Box::new(self.build(s_expr.child(0)?, materialized_cte.required.clone()).await?); + let left_side = Box::new( + self.build(s_expr.child(0)?, materialized_cte.required.clone()) + .await?, + ); let right_side = Box::new(self.build(s_expr.child(1)?, required).await?); Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { plan_id: 0, diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 66489f324b5fa..8b7ae6d7cb160 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -170,7 +170,7 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, - cte_schema: cte_schema, + cte_schema, }))); return Ok((s_expr, new_bind_context)); } else { diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index bb161ac105829..d0aa0c0ec6020 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -13,12 +13,12 @@ // limitations under the License. use std::hash::Hash; +use std::hash::Hasher; use databend_common_expression::DataSchemaRef; use crate::plans::Operator; use crate::plans::RelOp; -use std::hash::Hasher; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CTEConsumer { From f8f4d7abb13defce72a791cd3efc20320e4d2670 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 16:55:37 +0800 Subject: [PATCH 13/80] fix --- src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index f587d69c5ca1a..a63586634106e 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -487,10 +487,13 @@ impl SExpr { | crate::plans::RelOp::ConstantTableScan | crate::plans::RelOp::ExpressionScan | crate::plans::RelOp::CacheScan + | crate::plans::RelOp::CTEConsumer | crate::plans::RelOp::RecursiveCteScan => Ok(None), crate::plans::RelOp::Join => self.probe_side_child().get_data_distribution(), + crate::plans::RelOp::MaterializedCTE => self.child(1)?.get_data_distribution(), + crate::plans::RelOp::Exchange => { Ok(Some(self.plan.as_ref().clone().try_into().unwrap())) } From 4c75ded3978fd36e012ae2dd3f07115eceb8130f Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 20:36:14 +0800 Subject: [PATCH 14/80] fix join --- .../builders/builder_cte_consumer.rs | 20 ++++++++----------- .../src/pipelines/builders/builder_join.rs | 2 ++ .../builders/builder_materialized_cte.rs | 8 ++++++-- .../service/src/pipelines/pipeline_builder.rs | 5 +++-- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs index adfe50a2b0fad..1fa78f91fbc7d 100644 --- a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs +++ b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs @@ -30,19 +30,15 @@ impl PipelineBuilder { })? .clone(); - let current_consumer_id = - *self - .next_cte_consumer_id - .get(&cte.cte_name) - .ok_or_else(|| { - ErrorCode::Internal(format!( - "CTE consumer id not found for name: {}", - cte.cte_name - )) - })?; + let mut next_cte_consumer_id = self.next_cte_consumer_id.lock(); + let current_consumer_id = *next_cte_consumer_id.get(&cte.cte_name).ok_or_else(|| { + ErrorCode::Internal(format!( + "CTE consumer id not found for name: {}", + cte.cte_name + )) + })?; - self.next_cte_consumer_id - .insert(cte.cte_name.clone(), current_consumer_id + 1); + next_cte_consumer_id.insert(cte.cte_name.clone(), current_consumer_id + 1); self.main_pipeline.add_source( |output_port| { diff --git a/src/query/service/src/pipelines/builders/builder_join.rs b/src/query/service/src/pipelines/builders/builder_join.rs index 7937b11b3344c..09b1b9a7bb3d7 100644 --- a/src/query/service/src/pipelines/builders/builder_join.rs +++ b/src/query/service/src/pipelines/builders/builder_join.rs @@ -41,6 +41,8 @@ impl PipelineBuilder { let mut sub_builder = PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); sub_builder.hash_join_states = self.hash_join_states.clone(); + sub_builder.cte_receivers = self.cte_receivers.clone(); + sub_builder.next_cte_consumer_id = self.next_cte_consumer_id.clone(); sub_builder } diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 01db9afb7cf61..4336c2691920e 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -25,15 +25,19 @@ impl PipelineBuilder { pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { // init builder for cte pipeline let sub_context = QueryContext::create_from(self.ctx.as_ref()); - let sub_builder = + let mut sub_builder = PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); + sub_builder.cte_receivers = self.cte_receivers.clone(); + sub_builder.next_cte_consumer_id = self.next_cte_consumer_id.clone(); // build cte pipeline let mut build_res = sub_builder.finalize(&cte.left)?; build_res.main_pipeline.try_resize(1)?; let (tx, rx) = tokio::sync::watch::channel(Arc::new(MaterializedCteData::default())); self.cte_receivers.insert(cte.cte_name.clone(), rx); - self.next_cte_consumer_id.insert(cte.cte_name.clone(), 0); + self.next_cte_consumer_id + .lock() + .insert(cte.cte_name.clone(), 0); build_res .main_pipeline .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index ac7262cb90fbc..c3b909d72075c 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -27,6 +27,7 @@ use databend_common_pipeline_core::ExecutionInfo; use databend_common_pipeline_core::Pipeline; use databend_common_settings::Settings; use databend_common_sql::executor::PhysicalPlan; +use parking_lot::Mutex; use tokio::sync::watch::Receiver; use super::PipelineBuilderData; @@ -60,7 +61,7 @@ pub struct PipelineBuilder { pub contain_sink_processor: bool, pub cte_receivers: HashMap>>, - pub next_cte_consumer_id: HashMap, + pub next_cte_consumer_id: Arc>>, } impl PipelineBuilder { @@ -83,7 +84,7 @@ impl PipelineBuilder { contain_sink_processor: false, is_exchange_stack: vec![], cte_receivers: HashMap::new(), - next_cte_consumer_id: HashMap::new(), + next_cte_consumer_id: Arc::new(Mutex::new(HashMap::new())), } } From cc89312ada1fdf0811f7f36c612a30abf9bdfd63 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 30 Jun 2025 23:54:20 +0800 Subject: [PATCH 15/80] fix --- src/query/sql/src/planner/binder/bind_query/bind.rs | 5 ++++- .../planner/binder/bind_table_reference/bind_table.rs | 9 +++++++++ src/query/sql/src/planner/binder/table.rs | 9 --------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 988e98a76fc02..1d7750bfdfd6c 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -245,9 +245,12 @@ impl Binder { if cte.materialized { let cte_name = self.normalize_identifier(&cte.alias.name).name; + let mut cte_context = cte_context.clone(); + cte_context.cte_name = Some(cte_name.clone()); + // Create a new bind context for the CTE definition let mut cte_bind_context = BindContext { - cte_context: cte_context.clone(), + cte_context, ..Default::default() }; diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 8b7ae6d7cb160..4a0610ac5c1f0 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -68,6 +68,15 @@ impl Binder { table_identifier.table_name_alias(), ); + if let Some(cte_name) = &bind_context.cte_context.cte_name { + if cte_name == &table_name { + return Err(ErrorCode::SemanticError(format!( + "The cte {table_name} is not recursive, but it references itself.", + )) + .set_span(*span)); + } + } + let (consume, max_batch_size, with_opts_str) = if let Some(with_options) = with_options { check_with_opt_valid(with_options)?; let consume = get_with_opt_consume(with_options)?; diff --git a/src/query/sql/src/planner/binder/table.rs b/src/query/sql/src/planner/binder/table.rs index 5420257a63797..fcd5ac7b6e82c 100644 --- a/src/query/sql/src/planner/binder/table.rs +++ b/src/query/sql/src/planner/binder/table.rs @@ -163,15 +163,6 @@ impl Binder { alias: &Option, cte_info: &CteInfo, ) -> Result<(SExpr, BindContext)> { - if let Some(cte_name) = &bind_context.cte_context.cte_name { - if cte_name == table_name { - return Err(ErrorCode::SemanticError(format!( - "The cte {table_name} is not recursive, but it references itself.", - )) - .set_span(span)); - } - } - let mut new_bind_context = BindContext { parent: Some(Box::new(bind_context.clone())), bound_internal_columns: BTreeMap::new(), From 291204aa7c4bcf6749af4e7acc5a4988ca293d93 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 1 Jul 2025 00:12:56 +0800 Subject: [PATCH 16/80] refine explain --- src/query/sql/src/executor/format.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 8a5f11a61244a..0fa30d2531459 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -533,7 +533,7 @@ fn to_format_tree( children.push(to_format_tree(&plan.left, metadata, profs, context)?); children.push(to_format_tree(&plan.right, metadata, profs, context)?); Ok(FormatTreeNode::with_children( - "MaterializedCTE".to_string(), + format!("MaterializedCTE: {}", plan.cte_name), children, )) } From 4835fb85208993a4d201163efcd9cdfda14da63f Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 1 Jul 2025 19:30:31 +0800 Subject: [PATCH 17/80] fix --- .../sql/src/planner/binder/bind_context.rs | 5 ++- .../sql/src/planner/binder/bind_query/bind.rs | 11 ++++- .../binder/bind_table_reference/bind_table.rs | 9 +++- .../src/planner/optimizer/ir/expr/s_expr.rs | 17 ++++++++ .../sql/src/planner/plans/cte_consumer.rs | 43 ++++++------------- .../sql/src/planner/plans/materialized_cte.rs | 26 +++++++++++ 6 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index d1ac8e3430a99..b1493b1a78cc4 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -16,6 +16,7 @@ use std::collections::btree_map; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::hash::Hash; +use std::sync::Arc; use dashmap::DashMap; use databend_common_ast::ast::Identifier; @@ -47,6 +48,8 @@ use crate::ColumnSet; use crate::IndexType; use crate::MetadataRef; use crate::NameResolutionContext; +use std::sync::Mutex; +use crate::optimizer::ir::StatInfo; /// Context of current expression, this is used to check if /// the expression is valid in current context. @@ -160,7 +163,6 @@ pub struct CteContext { /// If the `BindContext` is created from a CTE, record the cte name pub cte_name: Option, pub cte_map: Box>, - pub is_binding_materialized_cte: bool, } impl CteContext { @@ -199,6 +201,7 @@ pub struct CteInfo { pub recursive: bool, // If cte is materialized, save its columns pub columns: Vec, + pub stat_info: Arc>>>, } impl BindContext { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 1d7750bfdfd6c..e752d19faf0fe 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; use std::sync::Arc; +use std::sync::Mutex; use databend_common_ast::ast::Expr; use databend_common_ast::ast::Query; @@ -27,6 +28,7 @@ use derive_visitor::Visitor; use crate::binder::CteInfo; use crate::normalize_identifier; +use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; @@ -154,6 +156,7 @@ impl Binder { recursive: with.recursive, columns: vec![], materialized: cte.materialized, + stat_info: Arc::new(Mutex::new(None)), }; bind_context .cte_context @@ -236,10 +239,9 @@ impl Binder { &mut self, with: &With, main_query_expr: SExpr, - mut cte_context: CteContext, + cte_context: CteContext, ) -> Result { let mut current_expr = main_query_expr; - cte_context.is_binding_materialized_cte = true; for cte in with.ctes.iter().rev() { if cte.materialized { @@ -247,6 +249,7 @@ impl Binder { let mut cte_context = cte_context.clone(); cte_context.cte_name = Some(cte_name.clone()); + let shared_stat_info = cte_context.cte_map.get(&cte_name).unwrap().stat_info.clone(); // Create a new bind context for the CTE definition let mut cte_bind_context = BindContext { @@ -265,6 +268,10 @@ impl Binder { Arc::new(cte_definition_expr), Arc::new(current_expr), ); + + // Derive cardinality statistics and share with consumer + let stats = RelExpr::with_s_expr(¤t_expr).derive_cardinality()?; + *shared_stat_info.lock().unwrap() = Some(stats); } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 4a0610ac5c1f0..faf51e525658d 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -177,10 +177,15 @@ impl Binder { new_bind_context.add_column_binding(column); } - let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { + let s_expr = SExpr::create_with_shared_stat_info(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, cte_schema, - }))); + })), + vec![], + None, + None, + cte_info.stat_info.clone(), + ); return Ok((s_expr, new_bind_context)); } else { if self diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index a63586634106e..3f89fddc445d0 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -83,6 +83,23 @@ impl SExpr { } } + pub fn create_with_shared_stat_info( + plan: Arc, + children: impl Into>>, + original_group: Option, + rel_prop: Option>, + stat_info: Arc>>>, + ) -> Self { + SExpr { + plan, + children: children.into(), + original_group, + rel_prop: Arc::new(Mutex::new(rel_prop)), + stat_info, + applied_rules: AppliedRules::default(), + } + } + pub fn create_unary(plan: Arc, child: impl Into>) -> Self { Self::create(plan, [child.into()], None, None, None) } diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index d0aa0c0ec6020..b9226478b5695 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -17,8 +17,13 @@ use std::hash::Hasher; use databend_common_expression::DataSchemaRef; +use crate::optimizer::ir::RelExpr; use crate::plans::Operator; use crate::plans::RelOp; +use std::sync::Arc; + +use databend_common_exception::Result; +use crate::optimizer::ir::StatInfo; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CTEConsumer { @@ -37,35 +42,13 @@ impl Operator for CTEConsumer { RelOp::CTEConsumer } - // fn arity(&self) -> usize { - // 0 - // } - - // fn derive_relational_prop(&self, _rel_expr: &RelExpr) -> Result> { - // Ok(Arc::new(RelationalProperty { - // output_columns: self.used_columns()?, - // outer_columns: ColumnSet::new(), - // used_columns: self.used_columns()?, - // orderings: vec![], - // partition_orderings: None, - // })) - // } - - // fn derive_physical_prop(&self, _rel_expr: &RelExpr) -> Result { - // Ok(PhysicalProperty { - // distribution: Distribution::Serial, - // }) - // } + /// Get arity of this operator + fn arity(&self) -> usize { + 0 + } - // fn compute_required_prop_child( - // &self, - // _ctx: Arc, - // _rel_expr: &RelExpr, - // _child_index: usize, - // _required: &RequiredProperty, - // ) -> Result { - // Err(ErrorCode::Internal( - // "Cannot compute required property for CTEConsumer".to_string(), - // )) - // } + /// Derive statistics information + fn derive_stats(&self, rel_expr: &RelExpr) -> Result> { + rel_expr.derive_cardinality() + } } diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index c37b286ef9436..cbdefc6ccf2d5 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -14,10 +14,16 @@ use std::hash::Hash; +use crate::optimizer::ir::RelExpr; use crate::plans::Operator; use crate::plans::RelOp; use crate::ColumnSet; +use std::sync::Arc; +use databend_common_exception::Result; +use crate::optimizer::ir::PhysicalProperty; +use crate::optimizer::ir::RelationalProperty; +use crate::optimizer::ir::StatInfo; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, @@ -34,4 +40,24 @@ impl Operator for MaterializedCTE { fn rel_op(&self) -> RelOp { RelOp::MaterializedCTE } + + /// Get arity of this operator + fn arity(&self) -> usize { + 2 + } + + /// Derive relational property + fn derive_relational_prop(&self, rel_expr: &RelExpr) -> Result> { + rel_expr.derive_relational_prop_child(1) + } + + /// Derive physical property + fn derive_physical_prop(&self, rel_expr: &RelExpr) -> Result { + rel_expr.derive_physical_prop_child(1) + } + + /// Derive statistics information + fn derive_stats(&self, rel_expr: &RelExpr) -> Result> { + rel_expr.derive_cardinality_child(1) + } } From 046bd041150901f72781c50ab43cced0e1b93837 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 2 Jul 2025 16:12:00 +0800 Subject: [PATCH 18/80] fix --- .../sql/src/planner/binder/bind_context.rs | 14 +++- .../planner/binder/bind_mutation/delete.rs | 2 +- .../planner/binder/bind_mutation/update.rs | 2 +- .../sql/src/planner/binder/bind_query/bind.rs | 84 ++++++++++--------- .../binder/bind_table_reference/bind_table.rs | 70 ++++------------ .../src/planner/binder/copy_into_location.rs | 2 +- .../sql/src/planner/binder/copy_into_table.rs | 2 +- src/query/sql/src/planner/binder/insert.rs | 2 +- .../src/planner/optimizer/ir/expr/s_expr.rs | 17 ---- .../sql/src/planner/plans/cte_consumer.rs | 7 +- .../sql/src/planner/plans/materialized_cte.rs | 10 +-- 11 files changed, 85 insertions(+), 127 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index b1493b1a78cc4..688afe9424fd0 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -43,13 +43,13 @@ use crate::binder::project_set::SetReturningInfo; use crate::binder::window::WindowInfo; use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; +use crate::optimizer::ir::SExpr; +use crate::optimizer::ir::StatInfo; use crate::plans::ScalarExpr; use crate::ColumnSet; use crate::IndexType; use crate::MetadataRef; use crate::NameResolutionContext; -use std::sync::Mutex; -use crate::optimizer::ir::StatInfo; /// Context of current expression, this is used to check if /// the expression is valid in current context. @@ -197,11 +197,17 @@ impl CteContext { pub struct CteInfo { pub columns_alias: Vec, pub query: Query, + pub bind_result: CteBindResult, pub materialized: bool, pub recursive: bool, - // If cte is materialized, save its columns pub columns: Vec, - pub stat_info: Arc>>>, +} + +#[derive(Clone, Debug)] +pub struct CteBindResult { + pub s_expr: SExpr, + pub bind_context: BindContext, + pub stat_info: Arc, } impl BindContext { diff --git a/src/query/sql/src/planner/binder/bind_mutation/delete.rs b/src/query/sql/src/planner/binder/bind_mutation/delete.rs index 0e8fcb8d7d671..b1b0f07e74529 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/delete.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/delete.rs @@ -41,7 +41,7 @@ impl Binder { .. } = stamt; - self.init_cte(bind_context, with)?; + self.bind_cte_def(bind_context, with)?; let target_table_identifier = if let TableReference::Table { catalog, diff --git a/src/query/sql/src/planner/binder/bind_mutation/update.rs b/src/query/sql/src/planner/binder/bind_mutation/update.rs index 56de12c2c2921..c267af8dc25a5 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/update.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/update.rs @@ -63,7 +63,7 @@ impl Binder { .. } = stmt; - self.init_cte(bind_context, with)?; + self.bind_cte_def(bind_context, with)?; let target_table_identifier = TableIdentifier::new(self, catalog, database, table, table_alias); diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index e752d19faf0fe..9e1bb0c69c86f 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -14,7 +14,6 @@ use std::collections::HashMap; use std::sync::Arc; -use std::sync::Mutex; use databend_common_ast::ast::Expr; use databend_common_ast::ast::Query; @@ -26,6 +25,7 @@ use databend_common_exception::Result; use derive_visitor::Drive; use derive_visitor::Visitor; +use crate::binder::CteBindResult; use crate::binder::CteInfo; use crate::normalize_identifier; use crate::optimizer::ir::RelExpr; @@ -73,8 +73,9 @@ impl Binder { } } } - // Initialize cte map. - self.init_cte(bind_context, &with)?; + + // Bind cte definition. + self.bind_cte_def(bind_context, &with)?; // Extract limit and offset from query. let (limit, offset) = self.extract_limit_and_offset(query)?; @@ -125,43 +126,53 @@ impl Binder { Ok(()) } - // Initialize cte map. - pub(crate) fn init_cte( + pub(crate) fn bind_cte_def( &mut self, bind_context: &mut BindContext, with: &Option, ) -> Result<()> { - let with = if let Some(with) = with { - with - } else { + let Some(with) = with else { return Ok(()); }; - for cte in with.ctes.iter() { - let table_name = self.normalize_identifier(&cte.alias.name).name; - if bind_context.cte_context.cte_map.contains_key(&table_name) { + let cte_name = self.normalize_identifier(&cte.alias.name).name; + if bind_context.cte_context.cte_map.contains_key(&cte_name) { return Err(ErrorCode::SemanticError(format!( - "Duplicate common table expression: {table_name}" + "Duplicate common table expression: {cte_name}" ))); } + let column_name = cte .alias .columns .iter() .map(|ident| self.normalize_identifier(ident).name) .collect(); + + let mut cte_bind_context = BindContext { + cte_context: CteContext { + cte_name: Some(cte_name.clone()), + cte_map: bind_context.cte_context.cte_map.clone(), + }, + ..Default::default() + }; + let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; + let stat_info = RelExpr::with_s_expr(&s_expr).derive_cardinality()?; + let bind_result = CteBindResult { + s_expr, + bind_context: cte_bind_context, + stat_info, + }; + let cte_info = CteInfo { columns_alias: column_name, query: *cte.query.clone(), recursive: with.recursive, - columns: vec![], materialized: cte.materialized, - stat_info: Arc::new(Mutex::new(None)), + columns: vec![], + bind_result, }; - bind_context - .cte_context - .cte_map - .insert(table_name, cte_info); + bind_context.cte_context.cte_map.insert(cte_name, cte_info); } Ok(()) @@ -246,32 +257,25 @@ impl Binder { for cte in with.ctes.iter().rev() { if cte.materialized { let cte_name = self.normalize_identifier(&cte.alias.name).name; + let CteBindResult { + s_expr, + bind_context, + stat_info, + } = cte_context + .cte_map + .get(&cte_name) + .unwrap() + .bind_result + .clone(); - let mut cte_context = cte_context.clone(); - cte_context.cte_name = Some(cte_name.clone()); - let shared_stat_info = cte_context.cte_map.get(&cte_name).unwrap().stat_info.clone(); - - // Create a new bind context for the CTE definition - let mut cte_bind_context = BindContext { - cte_context, - ..Default::default() - }; - - // Bind the CTE definition - let (cte_definition_expr, bind_context) = - self.bind_query(&mut cte_bind_context, &cte.query)?; - - // Create the MaterializedCTE operator let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); - current_expr = SExpr::create_binary( + current_expr = SExpr::create( Arc::new(materialized_cte.into()), - Arc::new(cte_definition_expr), - Arc::new(current_expr), + vec![Arc::new(s_expr), Arc::new(current_expr)], + None, + None, + Some(stat_info), ); - - // Derive cardinality statistics and share with consumer - let stats = RelExpr::with_s_expr(¤t_expr).derive_cardinality()?; - *shared_stat_info.lock().unwrap() = Some(stats); } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index faf51e525658d..b1da2f35ca22a 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -12,10 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; use std::sync::Arc; -use dashmap::DashMap; use databend_common_ast::ast::Identifier; use databend_common_ast::ast::SampleConfig; use databend_common_ast::ast::Statement; @@ -38,7 +36,6 @@ use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; -use crate::binder::ExprContext; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::CTEConsumer; @@ -90,38 +87,7 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - // For materialized CTE, we need to add column bindings to the bind_context - // so that column references can be resolved correctly - let mut new_bind_context = bind_context.clone(); - - // We need to bind the CTE definition to get the columns - // This is similar to how non-materialized CTEs are handled - let mut cte_bind_context = BindContext { - parent: Some(Box::new(bind_context.clone())), - bound_internal_columns: BTreeMap::new(), - columns: vec![], - aggregate_info: Default::default(), - windows: Default::default(), - srf_info: Default::default(), - cte_context: bind_context.cte_context.clone(), - in_grouping: false, - view_info: None, - have_async_func: false, - have_udf_script: false, - have_udf_server: false, - inverted_index_map: Box::default(), - allow_virtual_column: false, - expr_context: ExprContext::default(), - planning_agg_index: false, - window_definitions: DashMap::new(), - }; - - cte_bind_context.cte_context.cte_name = Some(table_name.to_string()); - - // Bind the CTE definition to get the columns - let (_, mut res_bind_context) = - self.bind_query(&mut cte_bind_context, &cte_info.query)?; - + let mut cte_bind_context = cte_info.bind_result.bind_context.clone(); // Apply column aliases let mut cols_alias = cte_info.columns_alias.clone(); if let Some(alias) = alias { @@ -139,26 +105,26 @@ impl Binder { .map(|alias| normalize_identifier(&alias.name, &self.name_resolution_ctx).name) .unwrap_or_else(|| table_name.to_string()); - for column in res_bind_context.columns.iter_mut() { + for column in cte_bind_context.columns.iter_mut() { column.database_name = None; column.table_name = Some(alias_table_name.clone()); } - if cols_alias.len() > res_bind_context.columns.len() { + if cols_alias.len() > cte_bind_context.columns.len() { return Err(ErrorCode::SemanticError(format!( "The CTE '{}' has {} columns, but {} aliases were provided. Ensure the number of aliases matches the number of columns in the CTE.", table_name, - res_bind_context.columns.len(), + cte_bind_context.columns.len(), cols_alias.len() )) .set_span(*span)); } for (index, column_name) in cols_alias.iter().enumerate() { - res_bind_context.columns[index].column_name = column_name.clone(); + cte_bind_context.columns[index].column_name = column_name.clone(); } - let fields = res_bind_context + let fields = cte_bind_context .columns .iter() .map(|column_binding| { @@ -170,22 +136,22 @@ impl Binder { .collect(); let cte_schema = DataSchemaRefExt::create(fields); - log::info!("[CTE] columns: {:?}", res_bind_context.columns); - // Add the columns to the new bind context - for column in res_bind_context.columns { + let mut new_bind_context = bind_context.clone(); + for column in cte_bind_context.columns { new_bind_context.add_column_binding(column); } - let s_expr = SExpr::create_with_shared_stat_info(Arc::new(RelOperator::CTEConsumer(CTEConsumer { - cte_name: table_name, - cte_schema, - })), - vec![], - None, - None, - cte_info.stat_info.clone(), - ); + let s_expr = SExpr::create( + Arc::new(RelOperator::CTEConsumer(CTEConsumer { + cte_name: table_name, + cte_schema, + })), + vec![], + None, + None, + Some(cte_info.bind_result.stat_info.clone()), + ); return Ok((s_expr, new_bind_context)); } else { if self diff --git a/src/query/sql/src/planner/binder/copy_into_location.rs b/src/query/sql/src/planner/binder/copy_into_location.rs index a6dd9e8d83c7a..01171a5102ea0 100644 --- a/src/query/sql/src/planner/binder/copy_into_location.rs +++ b/src/query/sql/src/planner/binder/copy_into_location.rs @@ -105,7 +105,7 @@ impl Binder { } } CopyIntoLocationSource::Query(query) => { - self.init_cte(bind_context, &stmt.with)?; + self.bind_cte_def(bind_context, &stmt.with)?; self.bind_statement(bind_context, &Statement::Query(query.clone())) .await } diff --git a/src/query/sql/src/planner/binder/copy_into_table.rs b/src/query/sql/src/planner/binder/copy_into_table.rs index 4c2652759705b..d10dea193440c 100644 --- a/src/query/sql/src/planner/binder/copy_into_table.rs +++ b/src/query/sql/src/planner/binder/copy_into_table.rs @@ -99,7 +99,7 @@ impl Binder { .await } CopyIntoTableSource::Query(query) => { - self.init_cte(bind_context, &stmt.with)?; + self.bind_cte_def(bind_context, &stmt.with)?; let mut max_column_position = MaxColumnPosition::new(); query.drive(&mut max_column_position); diff --git a/src/query/sql/src/planner/binder/insert.rs b/src/query/sql/src/planner/binder/insert.rs index 9cbfd4dc9b7df..aa374835cbb82 100644 --- a/src/query/sql/src/planner/binder/insert.rs +++ b/src/query/sql/src/planner/binder/insert.rs @@ -111,7 +111,7 @@ impl Binder { .. } = stmt; - self.init_cte(bind_context, with)?; + self.bind_cte_def(bind_context, with)?; let table_identifier = TableIdentifier::new(self, catalog, database, table, &None); let (catalog_name, database_name, table_name) = ( diff --git a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs index 3f89fddc445d0..a63586634106e 100644 --- a/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs +++ b/src/query/sql/src/planner/optimizer/ir/expr/s_expr.rs @@ -83,23 +83,6 @@ impl SExpr { } } - pub fn create_with_shared_stat_info( - plan: Arc, - children: impl Into>>, - original_group: Option, - rel_prop: Option>, - stat_info: Arc>>>, - ) -> Self { - SExpr { - plan, - children: children.into(), - original_group, - rel_prop: Arc::new(Mutex::new(rel_prop)), - stat_info, - applied_rules: AppliedRules::default(), - } - } - pub fn create_unary(plan: Arc, child: impl Into>) -> Self { Self::create(plan, [child.into()], None, None, None) } diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index b9226478b5695..41ed7ef60eb65 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -14,16 +14,15 @@ use std::hash::Hash; use std::hash::Hasher; +use std::sync::Arc; +use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::optimizer::ir::RelExpr; +use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; -use std::sync::Arc; - -use databend_common_exception::Result; -use crate::optimizer::ir::StatInfo; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CTEConsumer { diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index cbdefc6ccf2d5..dba74c7f23d49 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -13,17 +13,17 @@ // limitations under the License. use std::hash::Hash; - -use crate::optimizer::ir::RelExpr; -use crate::plans::Operator; -use crate::plans::RelOp; -use crate::ColumnSet; use std::sync::Arc; use databend_common_exception::Result; + use crate::optimizer::ir::PhysicalProperty; +use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::StatInfo; +use crate::plans::Operator; +use crate::plans::RelOp; +use crate::ColumnSet; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, From 4979e0b62a708611ff282e0aac93a2628432f80a Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 2 Jul 2025 17:32:42 +0800 Subject: [PATCH 19/80] fix --- src/query/sql/src/planner/binder/bind_context.rs | 3 --- .../sql/src/planner/binder/bind_query/bind.rs | 12 +++--------- .../binder/bind_table_reference/bind_table.rs | 15 +++++---------- src/query/sql/src/planner/plans/cte_consumer.rs | 6 ++++-- 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 688afe9424fd0..544c64f2f7339 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -16,7 +16,6 @@ use std::collections::btree_map; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::hash::Hash; -use std::sync::Arc; use dashmap::DashMap; use databend_common_ast::ast::Identifier; @@ -44,7 +43,6 @@ use crate::binder::window::WindowInfo; use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; -use crate::optimizer::ir::StatInfo; use crate::plans::ScalarExpr; use crate::ColumnSet; use crate::IndexType; @@ -207,7 +205,6 @@ pub struct CteInfo { pub struct CteBindResult { pub s_expr: SExpr, pub bind_context: BindContext, - pub stat_info: Arc, } impl BindContext { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 9e1bb0c69c86f..71ec123b54fe9 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -28,7 +28,6 @@ use derive_visitor::Visitor; use crate::binder::CteBindResult; use crate::binder::CteInfo; use crate::normalize_identifier; -use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; @@ -157,11 +156,9 @@ impl Binder { ..Default::default() }; let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; - let stat_info = RelExpr::with_s_expr(&s_expr).derive_cardinality()?; let bind_result = CteBindResult { s_expr, bind_context: cte_bind_context, - stat_info, }; let cte_info = CteInfo { @@ -260,7 +257,6 @@ impl Binder { let CteBindResult { s_expr, bind_context, - stat_info, } = cte_context .cte_map .get(&cte_name) @@ -269,12 +265,10 @@ impl Binder { .clone(); let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); - current_expr = SExpr::create( + current_expr = SExpr::create_binary( Arc::new(materialized_cte.into()), - vec![Arc::new(s_expr), Arc::new(current_expr)], - None, - None, - Some(stat_info), + Arc::new(s_expr), + Arc::new(current_expr), ); } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index b1da2f35ca22a..113c33a54333e 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -142,16 +142,11 @@ impl Binder { new_bind_context.add_column_binding(column); } - let s_expr = SExpr::create( - Arc::new(RelOperator::CTEConsumer(CTEConsumer { - cte_name: table_name, - cte_schema, - })), - vec![], - None, - None, - Some(cte_info.bind_result.stat_info.clone()), - ); + let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { + cte_name: table_name, + cte_schema, + def: cte_info.bind_result.s_expr.clone(), + }))); return Ok((s_expr, new_bind_context)); } else { if self diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index 41ed7ef60eb65..e439b5b558636 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -20,6 +20,7 @@ use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; use crate::optimizer::ir::RelExpr; +use crate::optimizer::ir::SExpr; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; @@ -28,6 +29,7 @@ use crate::plans::RelOp; pub struct CTEConsumer { pub cte_name: String, pub cte_schema: DataSchemaRef, + pub def: SExpr, } impl Hash for CTEConsumer { @@ -47,7 +49,7 @@ impl Operator for CTEConsumer { } /// Derive statistics information - fn derive_stats(&self, rel_expr: &RelExpr) -> Result> { - rel_expr.derive_cardinality() + fn derive_stats(&self, _rel_expr: &RelExpr) -> Result> { + RelExpr::with_s_expr(&self.def).derive_cardinality() } } From af0eeb7f3bb22887508a9e90e97b65541a87d80d Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 2 Jul 2025 17:51:11 +0800 Subject: [PATCH 20/80] fix --- .../sql/src/planner/binder/bind_context.rs | 2 +- .../sql/src/planner/binder/bind_query/bind.rs | 30 +++++++++++-------- .../binder/bind_table_reference/bind_table.rs | 4 +-- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 544c64f2f7339..790ab40698d8f 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -195,7 +195,7 @@ impl CteContext { pub struct CteInfo { pub columns_alias: Vec, pub query: Query, - pub bind_result: CteBindResult, + pub bind_result: Option, pub materialized: bool, pub recursive: bool, pub columns: Vec, diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 71ec123b54fe9..e2c6473bc0c0f 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -148,17 +148,22 @@ impl Binder { .map(|ident| self.normalize_identifier(ident).name) .collect(); - let mut cte_bind_context = BindContext { - cte_context: CteContext { - cte_name: Some(cte_name.clone()), - cte_map: bind_context.cte_context.cte_map.clone(), - }, - ..Default::default() - }; - let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; - let bind_result = CteBindResult { - s_expr, - bind_context: cte_bind_context, + let bind_result = if with.recursive { + None + } else { + let mut cte_bind_context = BindContext { + cte_context: CteContext { + cte_name: Some(cte_name.clone()), + cte_map: bind_context.cte_context.cte_map.clone(), + }, + ..Default::default() + }; + let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; + let bind_result = CteBindResult { + s_expr, + bind_context: cte_bind_context, + }; + Some(bind_result) }; let cte_info = CteInfo { @@ -262,7 +267,8 @@ impl Binder { .get(&cte_name) .unwrap() .bind_result - .clone(); + .clone() + .unwrap(); let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); current_expr = SExpr::create_binary( diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 113c33a54333e..1ce523d2e4d10 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -87,7 +87,7 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - let mut cte_bind_context = cte_info.bind_result.bind_context.clone(); + let mut cte_bind_context = cte_info.bind_result.as_ref().unwrap().bind_context.clone(); // Apply column aliases let mut cols_alias = cte_info.columns_alias.clone(); if let Some(alias) = alias { @@ -145,7 +145,7 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name, cte_schema, - def: cte_info.bind_result.s_expr.clone(), + def: cte_info.bind_result.as_ref().unwrap().s_expr.clone(), }))); return Ok((s_expr, new_bind_context)); } else { From 67c2bc3950aa867ccc5408406f31c4bb56e2ee02 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 3 Jul 2025 16:18:30 +0800 Subject: [PATCH 21/80] fix --- .../sql/src/planner/binder/bind_query/bind.rs | 3 +- .../binder/bind_table_reference/bind_table.rs | 3 +- .../sql/src/planner/plans/cte_consumer.rs | 40 +++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index e2c6473bc0c0f..3295167e99f3b 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -158,7 +158,8 @@ impl Binder { }, ..Default::default() }; - let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, &cte.query)?; + let (s_expr, cte_bind_context) = + self.bind_query(&mut cte_bind_context, &cte.query)?; let bind_result = CteBindResult { s_expr, bind_context: cte_bind_context, diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index 1ce523d2e4d10..fca9c63c11bb2 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -87,7 +87,8 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - let mut cte_bind_context = cte_info.bind_result.as_ref().unwrap().bind_context.clone(); + let mut cte_bind_context = + cte_info.bind_result.as_ref().unwrap().bind_context.clone(); // Apply column aliases let mut cols_alias = cte_info.columns_alias.clone(); if let Some(alias) = alias { diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index e439b5b558636..01580bc10133c 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -16,10 +16,15 @@ use std::hash::Hash; use std::hash::Hasher; use std::sync::Arc; +use databend_common_catalog::table_context::TableContext; +use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; +use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; +use crate::optimizer::ir::RelationalProperty; +use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::SExpr; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; @@ -52,4 +57,39 @@ impl Operator for CTEConsumer { fn derive_stats(&self, _rel_expr: &RelExpr) -> Result> { RelExpr::with_s_expr(&self.def).derive_cardinality() } + + /// Derive relational property + fn derive_relational_prop(&self, _rel_expr: &RelExpr) -> Result> { + RelExpr::with_s_expr(&self.def).derive_relational_prop() + } + + /// Derive physical property + fn derive_physical_prop(&self, _rel_expr: &RelExpr) -> Result { + RelExpr::with_s_expr(&self.def).derive_physical_prop() + } + + /// Compute required property for child with index `child_index` + fn compute_required_prop_child( + &self, + _ctx: Arc, + _rel_expr: &RelExpr, + _child_index: usize, + _required: &RequiredProperty, + ) -> Result { + Err(ErrorCode::Internal( + "Cannot compute required property for children of cte_consumer".to_string(), + )) + } + + /// Enumerate all possible combinations of required property for children + fn compute_required_prop_children( + &self, + _ctx: Arc, + _rel_expr: &RelExpr, + _required: &RequiredProperty, + ) -> Result>> { + Err(ErrorCode::Internal( + "Cannot compute required property for children of cte_consumer".to_string(), + )) + } } From 9a8eb3bdc932e397862e3fe3a4b07c75830bfc61 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 8 Jul 2025 16:27:39 +0800 Subject: [PATCH 22/80] fix --- .../sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 4d8c1d4daac5a..183b4db69c21e 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -275,6 +275,10 @@ impl DPhpyOptimizer { s_expr: &SExpr, ) -> Result<(Arc, bool)> { let new_s_expr = self.new_children(s_expr).await?; + self.join_relations.push(JoinRelation::new( + &new_s_expr, + self.sample_executor().clone(), + )); Ok((Arc::new(new_s_expr), true)) } From 30aa9f386e17a05b7f8aa4701f22bebacb8e460f Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 9 Jul 2025 10:26:34 +0800 Subject: [PATCH 23/80] fix --- .../optimizer/optimizers/hyper_dp/dphyp.rs | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 183b4db69c21e..3db2af0c0ae53 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -51,6 +51,8 @@ pub struct DPhpyOptimizer { join_relations: Vec, // base table index -> index of join_relations table_index_map: HashMap, + // cte name -> table indexes referenced by cte + cte_table_index_map: HashMap>, dp_table: HashMap, JoinNode>, query_graph: QueryGraph, relation_set_tree: RelationSetTree, @@ -66,6 +68,7 @@ impl DPhpyOptimizer { opt_ctx, join_relations: vec![], table_index_map: Default::default(), + cte_table_index_map: Default::default(), dp_table: Default::default(), query_graph: QueryGraph::new(), relation_set_tree: Default::default(), @@ -274,7 +277,28 @@ impl DPhpyOptimizer { &mut self, s_expr: &SExpr, ) -> Result<(Arc, bool)> { - let new_s_expr = self.new_children(s_expr).await?; + let m_cte = match s_expr.plan() { + RelOperator::MaterializedCTE(cte) => cte, + _ => unreachable!(), + }; + + let mut left_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + let left_expr = left_dphyp.optimize_async(s_expr.child(0)?).await?; + + let mut right_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + right_dphyp.cte_table_index_map.insert( + m_cte.cte_name.clone(), + self.table_index_map.keys().cloned().collect(), + ); + let right_expr = right_dphyp.optimize_async(s_expr.child(1)?).await?; + + // Merge table_index_map from right child into current table_index_map + let relation_idx = self.join_relations.len() as IndexType; + for table_index in right_dphyp.table_index_map.keys() { + self.table_index_map.insert(*table_index, relation_idx); + } + + let new_s_expr = s_expr.replace_children([Arc::new(left_expr), Arc::new(right_expr)]); self.join_relations.push(JoinRelation::new( &new_s_expr, self.sample_executor().clone(), @@ -282,7 +306,35 @@ impl DPhpyOptimizer { Ok((Arc::new(new_s_expr), true)) } - async fn process_cte_consumer_node(&mut self, s_expr: &SExpr) -> Result<(Arc, bool)> { + async fn process_cte_consumer_node( + &mut self, + s_expr: &SExpr, + join_relation: Option<&SExpr>, + ) -> Result<(Arc, bool)> { + let cte_consumer = match s_expr.plan() { + RelOperator::CTEConsumer(consumer) => consumer, + _ => unreachable!(), + }; + + let join_relation = if let Some(relation) = join_relation { + // Check if relation contains filter, if exists, check if the filter in `filters` + // If exists, remove it from `filters` + self.check_filter(relation); + JoinRelation::new(relation, self.sample_executor().clone()) + } else { + JoinRelation::new(s_expr, self.sample_executor().clone()) + }; + + // Get table indexes from cte_table_index_map for this CTE + if let Some(table_indexes) = self.cte_table_index_map.get(&cte_consumer.cte_name) { + // Map each table index to current join_relations index + let relation_idx = self.join_relations.len() as IndexType; + for table_index in table_indexes { + self.table_index_map.insert(*table_index, relation_idx); + } + } + + self.join_relations.push(join_relation); Ok((Arc::new(s_expr.clone()), true)) } @@ -349,7 +401,9 @@ impl DPhpyOptimizer { RelOperator::Join(_) => self.process_join_node(s_expr, join_conditions).await, RelOperator::MaterializedCTE(_) => self.process_materialized_cte_node(s_expr).await, - RelOperator::CTEConsumer(_) => self.process_cte_consumer_node(s_expr).await, + RelOperator::CTEConsumer(_) => { + self.process_cte_consumer_node(s_expr, join_relation).await + } RelOperator::ProjectSet(_) | RelOperator::Aggregate(_) From e39af8e7a4fea922ab5ebd7db30e50d8bbdc5f92 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 9 Jul 2025 10:57:39 +0800 Subject: [PATCH 24/80] fix --- .../optimizer/optimizers/hyper_dp/dphyp.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 3db2af0c0ae53..9ca9bb4cdcfc3 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -94,8 +94,10 @@ impl DPhpyOptimizer { // Parallel process children: start a new dphyp for each child. let left_expr = s_expr.children[0].clone(); let opt_ctx = self.opt_ctx.clone(); + let cte_table_index_map = self.cte_table_index_map.clone(); let left_res = spawn(async move { let mut dphyp = DPhpyOptimizer::new(opt_ctx.clone()); + dphyp.cte_table_index_map = cte_table_index_map; ( dphyp.optimize_async(&left_expr).await, dphyp.table_index_map, @@ -104,8 +106,10 @@ impl DPhpyOptimizer { let right_expr = s_expr.children[1].clone(); let opt_ctx = self.opt_ctx.clone(); + let cte_table_index_map = self.cte_table_index_map.clone(); let right_res = spawn(async move { let mut dphyp = DPhpyOptimizer::new(opt_ctx.clone()); + dphyp.cte_table_index_map = cte_table_index_map; ( dphyp.optimize_async(&right_expr).await, dphyp.table_index_map, @@ -137,6 +141,7 @@ impl DPhpyOptimizer { /// Process a subquery expression async fn process_subquery(&mut self, s_expr: &SExpr) -> Result<(Arc, bool)> { let mut dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + dphyp.cte_table_index_map = self.cte_table_index_map.clone(); let new_s_expr = Arc::new(dphyp.optimize_async(s_expr).await?); // Merge `table_index_map` of subquery into current `table_index_map`. @@ -283,12 +288,15 @@ impl DPhpyOptimizer { }; let mut left_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + left_dphyp.cte_table_index_map = self.cte_table_index_map.clone(); let left_expr = left_dphyp.optimize_async(s_expr.child(0)?).await?; let mut right_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); + right_dphyp.cte_table_index_map = self.cte_table_index_map.clone(); + // Use left_dphyp's table_index_map for the CTE definition right_dphyp.cte_table_index_map.insert( m_cte.cte_name.clone(), - self.table_index_map.keys().cloned().collect(), + left_dphyp.table_index_map.keys().cloned().collect(), ); let right_expr = right_dphyp.optimize_async(s_expr.child(1)?).await?; @@ -325,10 +333,9 @@ impl DPhpyOptimizer { JoinRelation::new(s_expr, self.sample_executor().clone()) }; - // Get table indexes from cte_table_index_map for this CTE + // Map table indexes before adding to join_relations + let relation_idx = self.join_relations.len() as IndexType; if let Some(table_indexes) = self.cte_table_index_map.get(&cte_consumer.cte_name) { - // Map each table index to current join_relations index - let relation_idx = self.join_relations.len() as IndexType; for table_index in table_indexes { self.table_index_map.insert(*table_index, relation_idx); } From a39a569c885599e68a3403d7dc0c56d980dafb07 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 9 Jul 2025 12:04:52 +0800 Subject: [PATCH 25/80] CleanupUnusedCTE --- .../sql/src/planner/optimizer/optimizer.rs | 5 +- .../src/planner/optimizer/optimizers/mod.rs | 1 + .../operator/cte/cleanup_unused_cte.rs | 93 +++++++++++++++ .../optimizer/optimizers/operator/cte/mod.rs | 17 +++ .../optimizer/optimizers/operator/mod.rs | 2 + .../suites/query/cleanup_unused_cte.test | 112 ++++++++++++++++++ 6 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs create mode 100644 src/query/sql/src/planner/optimizer/optimizers/operator/cte/mod.rs create mode 100644 tests/sqllogictests/suites/query/cleanup_unused_cte.test diff --git a/src/query/sql/src/planner/optimizer/optimizer.rs b/src/query/sql/src/planner/optimizer/optimizer.rs index 73213ecd31832..2141a9e9bbe0c 100644 --- a/src/query/sql/src/planner/optimizer/optimizer.rs +++ b/src/query/sql/src/planner/optimizer/optimizer.rs @@ -26,6 +26,7 @@ use crate::binder::MutationType; use crate::optimizer::ir::Memo; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::distributed::BroadcastToShuffleOptimizer; +use crate::optimizer::optimizers::operator::CleanupUnusedCTEOptimizer; use crate::optimizer::optimizers::operator::DeduplicateJoinConditionOptimizer; use crate::optimizer::optimizers::operator::PullUpFilterOptimizer; use crate::optimizer::optimizers::operator::RuleNormalizeAggregateOptimizer; @@ -273,7 +274,9 @@ pub async fn optimize_query(opt_ctx: Arc, s_expr: SExpr) -> Re .add_if( !opt_ctx.get_planning_agg_index(), RecursiveRuleOptimizer::new(opt_ctx.clone(), [RuleID::EliminateEvalScalar].as_slice()), - ); + ) + // 15. Clean up unused CTEs + .add(CleanupUnusedCTEOptimizer); // 15. Execute the pipeline let s_expr = pipeline.execute().await?; diff --git a/src/query/sql/src/planner/optimizer/optimizers/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/mod.rs index c4f5e6998b631..aa38f80db04fc 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/mod.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/mod.rs @@ -21,3 +21,4 @@ pub mod rule; pub use cascades::CascadesOptimizer; pub use hyper_dp::DPhpyOptimizer; +pub use operator::CleanupUnusedCTEOptimizer; diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs new file mode 100644 index 0000000000000..b29d06afe460d --- /dev/null +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs @@ -0,0 +1,93 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashSet; +use std::sync::Arc; + +use databend_common_exception::Result; + +use crate::optimizer::ir::SExpr; +use crate::optimizer::Optimizer; +use crate::plans::RelOperator; + +/// Optimizer that removes unused CTEs from the query plan. +/// This optimizer should be applied at the end of the optimization pipeline +/// to clean up any CTEs that are not referenced by any CTEConsumer. +pub struct CleanupUnusedCTEOptimizer; + +impl CleanupUnusedCTEOptimizer { + /// Collect all CTE names that are referenced by CTEConsumer nodes + fn collect_referenced_ctes(s_expr: &SExpr) -> Result> { + let mut referenced_ctes = HashSet::new(); + Self::collect_referenced_ctes_recursive(s_expr, &mut referenced_ctes)?; + Ok(referenced_ctes) + } + + /// Recursively traverse the expression tree to find CTEConsumer nodes + fn collect_referenced_ctes_recursive( + s_expr: &SExpr, + referenced_ctes: &mut HashSet, + ) -> Result<()> { + // Check if current node is a CTEConsumer + if let RelOperator::CTEConsumer(consumer) = s_expr.plan() { + referenced_ctes.insert(consumer.cte_name.clone()); + } + + // Recursively process children + for child in s_expr.children() { + Self::collect_referenced_ctes_recursive(child, referenced_ctes)?; + } + + Ok(()) + } + + /// Remove unused CTEs from the expression tree + fn remove_unused_ctes(s_expr: &SExpr, referenced_ctes: &HashSet) -> Result { + if let RelOperator::MaterializedCTE(m_cte) = s_expr.plan() { + // If this CTE is not referenced, remove it by returning the right child + if !referenced_ctes.contains(&m_cte.cte_name) { + // Return the right child (main query) and skip the left child (CTE definition) + let right_child = s_expr.child(1)?; + return Self::remove_unused_ctes(right_child, referenced_ctes); + } + } + + // Process children recursively + let mut optimized_children = Vec::with_capacity(s_expr.arity()); + for child in s_expr.children() { + let optimized_child = Self::remove_unused_ctes(child, referenced_ctes)?; + optimized_children.push(Arc::new(optimized_child)); + } + + // Create new expression with optimized children + let mut new_expr = s_expr.clone(); + new_expr.children = optimized_children; + Ok(new_expr) + } +} + +#[async_trait::async_trait] +impl Optimizer for CleanupUnusedCTEOptimizer { + fn name(&self) -> String { + "CleanupUnusedCTEOptimizer".to_string() + } + + async fn optimize(&mut self, s_expr: &SExpr) -> Result { + // Collect all referenced CTEs + let referenced_ctes = Self::collect_referenced_ctes(s_expr)?; + + // Remove unused CTEs + Self::remove_unused_ctes(s_expr, &referenced_ctes) + } +} diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/mod.rs new file mode 100644 index 0000000000000..0aeab30d3574b --- /dev/null +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/mod.rs @@ -0,0 +1,17 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod cleanup_unused_cte; + +pub use cleanup_unused_cte::CleanupUnusedCTEOptimizer; diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs index 0c0566a886f9e..93c8833bec214 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/mod.rs @@ -13,12 +13,14 @@ // limitations under the License. mod aggregate; +mod cte; mod decorrelate; mod filter; mod join; pub use aggregate::RuleNormalizeAggregateOptimizer; pub use aggregate::RuleStatsAggregateOptimizer; +pub use cte::CleanupUnusedCTEOptimizer; pub use decorrelate::FlattenInfo; pub use decorrelate::SubqueryDecorrelatorOptimizer; pub use decorrelate::UnnestResult; diff --git a/tests/sqllogictests/suites/query/cleanup_unused_cte.test b/tests/sqllogictests/suites/query/cleanup_unused_cte.test new file mode 100644 index 0000000000000..4c38d8a4e6845 --- /dev/null +++ b/tests/sqllogictests/suites/query/cleanup_unused_cte.test @@ -0,0 +1,112 @@ +# Test for CleanupUnusedCTEOptimizer +# This test verifies that unused CTEs are properly removed from the query plan + +# Test case 1: CTE is used, should not be removed +query I +with t1 as materialized (select number as a from numbers(10)) select t1.a from t1; +---- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +# Test case 2: CTE is not used, should be removed +query I +with t1 as materialized (select number as a from numbers(10)) select number as b from numbers(5); +---- +0 +1 +2 +3 +4 + +# Test case 3: Multiple CTEs, some used and some unused +query I +with t1 as materialized (select number as a from numbers(10)), + t2 as materialized (select number as b from numbers(20)), + t3 as materialized (select number as c from numbers(30)) +select t1.a from t1 join t2 on t1.a = t2.b; +---- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +# Test case 4: Nested CTEs, inner CTE is unused +query I +with t1 as materialized (select number as a from numbers(10)), + t2 as materialized (select a as b from t1), + t3 as materialized (select number as c from numbers(5)) +select t2.b from t2; +---- +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 + +# Test case 5: All CTEs are unused +query I +with t1 as materialized (select number as a from numbers(10)), + t2 as materialized (select number as b from numbers(20)) +select number as c from numbers(3); +---- +0 +1 +2 + +# Test case 6: CTE with complex query, should be removed when unused +query I +with t1 as materialized ( + select number as a, number * 2 as b + from numbers(10) + where number > 5 +) +select number as c from numbers(3); +---- +0 +1 +2 + +# Test case 7: CTE with aggregation, should be removed when unused +query I +with t1 as materialized ( + select number as a, count(*) as cnt + from numbers(10) + group by number +) +select number as b from numbers(3); +---- +0 +1 +2 + +# Test case 8: CTE with join, should be removed when unused +query I +with t1 as materialized ( + select n1.number as a, n2.number as b + from numbers(5) n1 + join numbers(5) n2 on n1.number = n2.number +) +select number as c from numbers(3); +---- +0 +1 +2 \ No newline at end of file From 36866865f0e31ff8fbca0d13f679c2430b4b8a12 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 14:41:09 +0800 Subject: [PATCH 26/80] fix --- .../binder/bind_table_reference/bind_cte.rs | 132 ++++++++++++++++++ .../binder/bind_table_reference/bind_table.rs | 70 +--------- .../binder/bind_table_reference/mod.rs | 1 + 3 files changed, 134 insertions(+), 69 deletions(-) create mode 100644 src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs new file mode 100644 index 0000000000000..9a066ed2617aa --- /dev/null +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -0,0 +1,132 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_ast::ast::Query; +use databend_common_ast::ast::TableAlias; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::DataField; +use databend_common_expression::DataSchemaRefExt; +use indexmap::IndexMap; + +use crate::binder::BindContext; +use crate::binder::Binder; +use crate::binder::CteContext; +use crate::binder::CteInfo; +use crate::normalize_identifier; +use crate::optimizer::ir::SExpr; +use crate::plans::CTEConsumer; +use crate::plans::RelOperator; + +impl Binder { + pub fn bind_cte_consumer( + &mut self, + bind_context: &mut BindContext, + table_name: &str, + alias: &Option, + cte_info: &CteInfo, + ) -> Result<(SExpr, BindContext)> { + let (_, cte_bind_context) = self.bind_cte_definition( + table_name, + bind_context.cte_context.cte_map.as_ref(), + &cte_info.query, + )?; + + let (table_alias, column_alias) = match alias { + Some(alias) => { + let table_alias = normalize_identifier(&alias.name, &self.name_resolution_ctx).name; + let column_alias = if alias.columns.is_empty() { + cte_info.columns_alias.clone() + } else { + alias + .columns + .iter() + .map(|column| normalize_identifier(column, &self.name_resolution_ctx).name) + .collect() + }; + (table_alias, column_alias) + } + None => (table_name.to_string(), cte_info.columns_alias.clone()), + }; + + if !column_alias.is_empty() && column_alias.len() != cte_bind_context.columns.len() { + return Err(ErrorCode::SemanticError(format!( + "The CTE '{}' has {} columns ({:?}), but {} aliases ({:?}) were provided. Ensure the number of aliases matches the number of columns in the CTE.", + table_name, + cte_bind_context.columns.len(), + cte_bind_context.columns.iter().map(|c| &c.column_name).collect::>(), + column_alias.len(), + column_alias, + ))); + } + + let mut cte_output_columns = cte_bind_context.columns.clone(); + for column in cte_output_columns.iter_mut() { + column.database_name = None; + column.table_name = Some(table_alias.clone()); + } + for (index, column_name) in column_alias.iter().enumerate() { + cte_output_columns[index].column_name = column_name.clone(); + } + + let fields = cte_output_columns + .iter() + .map(|column_binding| { + DataField::new( + &column_binding.index.to_string(), + *column_binding.data_type.clone(), + ) + }) + .collect(); + let cte_schema = DataSchemaRefExt::create(fields); + + let mut new_bind_context = bind_context.clone(); + for column in cte_output_columns { + new_bind_context.add_column_binding(column); + } + + let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { + cte_name: table_name.to_string(), + cte_schema, + def: cte_info.bind_result.as_ref().unwrap().s_expr.clone(), + }))); + Ok((s_expr, new_bind_context)) + } + + fn bind_cte_definition( + &mut self, + cte_name: &str, + cte_map: &IndexMap, + query: &Query, + ) -> Result<(SExpr, BindContext)> { + let mut prev_cte_map = Box::new(IndexMap::new()); + for (name, cte_info) in cte_map.iter() { + if name == cte_name { + break; + } + prev_cte_map.insert(name.clone(), cte_info.clone()); + } + let mut cte_bind_context = BindContext { + cte_context: CteContext { + cte_name: Some(cte_name.to_string()), + cte_map: prev_cte_map, + }, + ..Default::default() + }; + let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, query)?; + Ok((s_expr, cte_bind_context)) + } +} diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs index fca9c63c11bb2..d43bc3f7dfbba 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_table.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; - use databend_common_ast::ast::Identifier; use databend_common_ast::ast::SampleConfig; use databend_common_ast::ast::Statement; @@ -29,17 +27,12 @@ use databend_common_catalog::table_with_options::get_with_opt_consume; use databend_common_catalog::table_with_options::get_with_opt_max_batch_size; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::DataField; -use databend_common_expression::DataSchemaRefExt; use databend_common_storages_view::view_table::QUERY; use databend_storages_common_table_meta::table::get_change_type; use crate::binder::util::TableIdentifier; use crate::binder::Binder; -use crate::normalize_identifier; use crate::optimizer::ir::SExpr; -use crate::plans::CTEConsumer; -use crate::plans::RelOperator; use crate::BindContext; impl Binder { /// Bind a base table. @@ -87,68 +80,7 @@ impl Binder { let cte_map = bind_context.cte_context.cte_map.clone(); if let Some(cte_info) = cte_map.get(&table_name) { if cte_info.materialized { - let mut cte_bind_context = - cte_info.bind_result.as_ref().unwrap().bind_context.clone(); - // Apply column aliases - let mut cols_alias = cte_info.columns_alias.clone(); - if let Some(alias) = alias { - for (idx, col_alias) in alias.columns.iter().enumerate() { - if idx < cte_info.columns_alias.len() { - cols_alias[idx] = col_alias.name.clone(); - } else { - cols_alias.push(col_alias.name.clone()); - } - } - } - - let alias_table_name = alias - .as_ref() - .map(|alias| normalize_identifier(&alias.name, &self.name_resolution_ctx).name) - .unwrap_or_else(|| table_name.to_string()); - - for column in cte_bind_context.columns.iter_mut() { - column.database_name = None; - column.table_name = Some(alias_table_name.clone()); - } - - if cols_alias.len() > cte_bind_context.columns.len() { - return Err(ErrorCode::SemanticError(format!( - "The CTE '{}' has {} columns, but {} aliases were provided. Ensure the number of aliases matches the number of columns in the CTE.", - table_name, - cte_bind_context.columns.len(), - cols_alias.len() - )) - .set_span(*span)); - } - - for (index, column_name) in cols_alias.iter().enumerate() { - cte_bind_context.columns[index].column_name = column_name.clone(); - } - - let fields = cte_bind_context - .columns - .iter() - .map(|column_binding| { - DataField::new( - &column_binding.index.to_string(), - *column_binding.data_type.clone(), - ) - }) - .collect(); - let cte_schema = DataSchemaRefExt::create(fields); - - // Add the columns to the new bind context - let mut new_bind_context = bind_context.clone(); - for column in cte_bind_context.columns { - new_bind_context.add_column_binding(column); - } - - let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { - cte_name: table_name, - cte_schema, - def: cte_info.bind_result.as_ref().unwrap().s_expr.clone(), - }))); - return Ok((s_expr, new_bind_context)); + return self.bind_cte_consumer(bind_context, &table_name, alias, cte_info); } else { if self .metadata diff --git a/src/query/sql/src/planner/binder/bind_table_reference/mod.rs b/src/query/sql/src/planner/binder/bind_table_reference/mod.rs index 542911474aa34..75a42e6f60ddf 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/mod.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. mod bind; +mod bind_cte; mod bind_join; mod bind_location; mod bind_obfuscate; From fb7bfbd9d9e1d34892d5f019d612c359d5533f04 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 15:30:56 +0800 Subject: [PATCH 27/80] fix --- .../sql/src/planner/binder/bind_table_reference/bind_cte.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index 9a066ed2617aa..c009c786ff707 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -39,7 +39,7 @@ impl Binder { alias: &Option, cte_info: &CteInfo, ) -> Result<(SExpr, BindContext)> { - let (_, cte_bind_context) = self.bind_cte_definition( + let (s_expr, cte_bind_context) = self.bind_cte_definition( table_name, bind_context.cte_context.cte_map.as_ref(), &cte_info.query, @@ -101,7 +101,7 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name.to_string(), cte_schema, - def: cte_info.bind_result.as_ref().unwrap().s_expr.clone(), + def: s_expr, }))); Ok((s_expr, new_bind_context)) } From e600f31a39d6347bda7e1613626785c4d7b67e73 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 15:46:08 +0800 Subject: [PATCH 28/80] fix --- .../optimizer/optimizers/hyper_dp/dphyp.rs | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index 9ca9bb4cdcfc3..d4a665be1b700 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -51,8 +51,6 @@ pub struct DPhpyOptimizer { join_relations: Vec, // base table index -> index of join_relations table_index_map: HashMap, - // cte name -> table indexes referenced by cte - cte_table_index_map: HashMap>, dp_table: HashMap, JoinNode>, query_graph: QueryGraph, relation_set_tree: RelationSetTree, @@ -68,7 +66,6 @@ impl DPhpyOptimizer { opt_ctx, join_relations: vec![], table_index_map: Default::default(), - cte_table_index_map: Default::default(), dp_table: Default::default(), query_graph: QueryGraph::new(), relation_set_tree: Default::default(), @@ -94,10 +91,8 @@ impl DPhpyOptimizer { // Parallel process children: start a new dphyp for each child. let left_expr = s_expr.children[0].clone(); let opt_ctx = self.opt_ctx.clone(); - let cte_table_index_map = self.cte_table_index_map.clone(); let left_res = spawn(async move { let mut dphyp = DPhpyOptimizer::new(opt_ctx.clone()); - dphyp.cte_table_index_map = cte_table_index_map; ( dphyp.optimize_async(&left_expr).await, dphyp.table_index_map, @@ -106,10 +101,8 @@ impl DPhpyOptimizer { let right_expr = s_expr.children[1].clone(); let opt_ctx = self.opt_ctx.clone(); - let cte_table_index_map = self.cte_table_index_map.clone(); let right_res = spawn(async move { let mut dphyp = DPhpyOptimizer::new(opt_ctx.clone()); - dphyp.cte_table_index_map = cte_table_index_map; ( dphyp.optimize_async(&right_expr).await, dphyp.table_index_map, @@ -141,7 +134,6 @@ impl DPhpyOptimizer { /// Process a subquery expression async fn process_subquery(&mut self, s_expr: &SExpr) -> Result<(Arc, bool)> { let mut dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); - dphyp.cte_table_index_map = self.cte_table_index_map.clone(); let new_s_expr = Arc::new(dphyp.optimize_async(s_expr).await?); // Merge `table_index_map` of subquery into current `table_index_map`. @@ -282,22 +274,10 @@ impl DPhpyOptimizer { &mut self, s_expr: &SExpr, ) -> Result<(Arc, bool)> { - let m_cte = match s_expr.plan() { - RelOperator::MaterializedCTE(cte) => cte, - _ => unreachable!(), - }; - let mut left_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); - left_dphyp.cte_table_index_map = self.cte_table_index_map.clone(); let left_expr = left_dphyp.optimize_async(s_expr.child(0)?).await?; let mut right_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); - right_dphyp.cte_table_index_map = self.cte_table_index_map.clone(); - // Use left_dphyp's table_index_map for the CTE definition - right_dphyp.cte_table_index_map.insert( - m_cte.cte_name.clone(), - left_dphyp.table_index_map.keys().cloned().collect(), - ); let right_expr = right_dphyp.optimize_async(s_expr.child(1)?).await?; // Merge table_index_map from right child into current table_index_map @@ -335,16 +315,31 @@ impl DPhpyOptimizer { // Map table indexes before adding to join_relations let relation_idx = self.join_relations.len() as IndexType; - if let Some(table_indexes) = self.cte_table_index_map.get(&cte_consumer.cte_name) { - for table_index in table_indexes { - self.table_index_map.insert(*table_index, relation_idx); - } + let table_indexes = Self::get_table_indexes(&cte_consumer.def); + for table_index in table_indexes { + self.table_index_map.insert(table_index, relation_idx); } self.join_relations.push(join_relation); Ok((Arc::new(s_expr.clone()), true)) } + fn get_table_indexes(s_expr: &SExpr) -> Vec { + let mut table_indexes = Vec::new(); + Self::collect_table_indexes_recursive(s_expr, &mut table_indexes); + table_indexes + } + + fn collect_table_indexes_recursive(s_expr: &SExpr, table_indexes: &mut Vec) { + if let RelOperator::Scan(scan) = s_expr.plan() { + table_indexes.push(scan.table_index); + } + + for child in s_expr.children() { + Self::collect_table_indexes_recursive(child, table_indexes); + } + } + /// Process a unary operator node async fn process_unary_node( &mut self, From 5e0752a7cbcb0fa466fc9489c5e8b5ac30228191 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 16:10:40 +0800 Subject: [PATCH 29/80] fix --- .../sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index d4a665be1b700..bb17f389104a0 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -335,6 +335,10 @@ impl DPhpyOptimizer { table_indexes.push(scan.table_index); } + if let RelOperator::CTEConsumer(cte_consumer) = s_expr.plan() { + Self::collect_table_indexes_recursive(&cte_consumer.def, table_indexes); + } + for child in s_expr.children() { Self::collect_table_indexes_recursive(child, table_indexes); } From 4606b12d7e36b0b8592edc7686d75acccdbfc6d9 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 19:12:23 +0800 Subject: [PATCH 30/80] refine --- .../sql/src/planner/binder/bind_context.rs | 1 - .../planner/binder/bind_mutation/delete.rs | 2 +- .../planner/binder/bind_mutation/update.rs | 2 +- .../sql/src/planner/binder/bind_query/bind.rs | 96 +------------------ .../binder/bind_table_reference/bind_cte.rs | 61 +++++++++++- .../src/planner/binder/copy_into_location.rs | 2 +- .../sql/src/planner/binder/copy_into_table.rs | 2 +- src/query/sql/src/planner/binder/insert.rs | 2 +- 8 files changed, 66 insertions(+), 102 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 790ab40698d8f..3a0d4179e77c4 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -195,7 +195,6 @@ impl CteContext { pub struct CteInfo { pub columns_alias: Vec, pub query: Query, - pub bind_result: Option, pub materialized: bool, pub recursive: bool, pub columns: Vec, diff --git a/src/query/sql/src/planner/binder/bind_mutation/delete.rs b/src/query/sql/src/planner/binder/bind_mutation/delete.rs index b1b0f07e74529..0e8fcb8d7d671 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/delete.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/delete.rs @@ -41,7 +41,7 @@ impl Binder { .. } = stamt; - self.bind_cte_def(bind_context, with)?; + self.init_cte(bind_context, with)?; let target_table_identifier = if let TableReference::Table { catalog, diff --git a/src/query/sql/src/planner/binder/bind_mutation/update.rs b/src/query/sql/src/planner/binder/bind_mutation/update.rs index c267af8dc25a5..56de12c2c2921 100644 --- a/src/query/sql/src/planner/binder/bind_mutation/update.rs +++ b/src/query/sql/src/planner/binder/bind_mutation/update.rs @@ -63,7 +63,7 @@ impl Binder { .. } = stmt; - self.bind_cte_def(bind_context, with)?; + self.init_cte(bind_context, with)?; let target_table_identifier = TableIdentifier::new(self, catalog, database, table, table_alias); diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 3295167e99f3b..315dbad48db93 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -25,16 +25,12 @@ use databend_common_exception::Result; use derive_visitor::Drive; use derive_visitor::Visitor; -use crate::binder::CteBindResult; -use crate::binder::CteInfo; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::planner::binder::scalar::ScalarBinder; use crate::planner::binder::BindContext; use crate::planner::binder::Binder; -use crate::planner::binder::CteContext; use crate::plans::BoundColumnRef; -use crate::plans::MaterializedCTE; use crate::plans::ScalarExpr; use crate::plans::Sort; use crate::plans::SortItem; @@ -74,7 +70,7 @@ impl Binder { } // Bind cte definition. - self.bind_cte_def(bind_context, &with)?; + self.init_cte(bind_context, &with)?; // Extract limit and offset from query. let (limit, offset) = self.extract_limit_and_offset(query)?; @@ -125,62 +121,6 @@ impl Binder { Ok(()) } - pub(crate) fn bind_cte_def( - &mut self, - bind_context: &mut BindContext, - with: &Option, - ) -> Result<()> { - let Some(with) = with else { - return Ok(()); - }; - for cte in with.ctes.iter() { - let cte_name = self.normalize_identifier(&cte.alias.name).name; - if bind_context.cte_context.cte_map.contains_key(&cte_name) { - return Err(ErrorCode::SemanticError(format!( - "Duplicate common table expression: {cte_name}" - ))); - } - - let column_name = cte - .alias - .columns - .iter() - .map(|ident| self.normalize_identifier(ident).name) - .collect(); - - let bind_result = if with.recursive { - None - } else { - let mut cte_bind_context = BindContext { - cte_context: CteContext { - cte_name: Some(cte_name.clone()), - cte_map: bind_context.cte_context.cte_map.clone(), - }, - ..Default::default() - }; - let (s_expr, cte_bind_context) = - self.bind_query(&mut cte_bind_context, &cte.query)?; - let bind_result = CteBindResult { - s_expr, - bind_context: cte_bind_context, - }; - Some(bind_result) - }; - - let cte_info = CteInfo { - columns_alias: column_name, - query: *cte.query.clone(), - recursive: with.recursive, - materialized: cte.materialized, - columns: vec![], - bind_result, - }; - bind_context.cte_context.cte_map.insert(cte_name, cte_info); - } - - Ok(()) - } - pub(crate) fn bind_query_order_by( &mut self, bind_context: &mut BindContext, @@ -248,38 +188,4 @@ impl Binder { Arc::new(child), )) } - - fn bind_materialized_cte( - &mut self, - with: &With, - main_query_expr: SExpr, - cte_context: CteContext, - ) -> Result { - let mut current_expr = main_query_expr; - - for cte in with.ctes.iter().rev() { - if cte.materialized { - let cte_name = self.normalize_identifier(&cte.alias.name).name; - let CteBindResult { - s_expr, - bind_context, - } = cte_context - .cte_map - .get(&cte_name) - .unwrap() - .bind_result - .clone() - .unwrap(); - - let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); - current_expr = SExpr::create_binary( - Arc::new(materialized_cte.into()), - Arc::new(s_expr), - Arc::new(current_expr), - ); - } - } - - Ok(current_expr) - } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index c009c786ff707..925117e218436 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use databend_common_ast::ast::Query; use databend_common_ast::ast::TableAlias; +use databend_common_ast::ast::With; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataField; @@ -29,9 +30,41 @@ use crate::binder::CteInfo; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; use crate::plans::CTEConsumer; +use crate::plans::MaterializedCTE; use crate::plans::RelOperator; impl Binder { + pub fn init_cte(&mut self, bind_context: &mut BindContext, with: &Option) -> Result<()> { + let Some(with) = with else { + return Ok(()); + }; + for cte in with.ctes.iter() { + let cte_name = self.normalize_identifier(&cte.alias.name).name; + if bind_context.cte_context.cte_map.contains_key(&cte_name) { + return Err(ErrorCode::SemanticError(format!( + "Duplicate common table expression: {cte_name}" + ))); + } + + let column_name = cte + .alias + .columns + .iter() + .map(|ident| self.normalize_identifier(ident).name) + .collect(); + + let cte_info = CteInfo { + columns_alias: column_name, + query: *cte.query.clone(), + recursive: with.recursive, + materialized: cte.materialized, + columns: vec![], + }; + bind_context.cte_context.cte_map.insert(cte_name, cte_info); + } + + Ok(()) + } pub fn bind_cte_consumer( &mut self, bind_context: &mut BindContext, @@ -106,7 +139,7 @@ impl Binder { Ok((s_expr, new_bind_context)) } - fn bind_cte_definition( + pub fn bind_cte_definition( &mut self, cte_name: &str, cte_map: &IndexMap, @@ -129,4 +162,30 @@ impl Binder { let (s_expr, cte_bind_context) = self.bind_query(&mut cte_bind_context, query)?; Ok((s_expr, cte_bind_context)) } + + pub fn bind_materialized_cte( + &mut self, + with: &With, + main_query_expr: SExpr, + cte_context: CteContext, + ) -> Result { + let mut current_expr = main_query_expr; + + for cte in with.ctes.iter().rev() { + if cte.materialized { + let cte_name = self.normalize_identifier(&cte.alias.name).name; + let (s_expr, bind_context) = + self.bind_cte_definition(&cte_name, &cte_context.cte_map, &cte.query)?; + + let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); + current_expr = SExpr::create_binary( + Arc::new(materialized_cte.into()), + Arc::new(s_expr), + Arc::new(current_expr), + ); + } + } + + Ok(current_expr) + } } diff --git a/src/query/sql/src/planner/binder/copy_into_location.rs b/src/query/sql/src/planner/binder/copy_into_location.rs index 01171a5102ea0..a6dd9e8d83c7a 100644 --- a/src/query/sql/src/planner/binder/copy_into_location.rs +++ b/src/query/sql/src/planner/binder/copy_into_location.rs @@ -105,7 +105,7 @@ impl Binder { } } CopyIntoLocationSource::Query(query) => { - self.bind_cte_def(bind_context, &stmt.with)?; + self.init_cte(bind_context, &stmt.with)?; self.bind_statement(bind_context, &Statement::Query(query.clone())) .await } diff --git a/src/query/sql/src/planner/binder/copy_into_table.rs b/src/query/sql/src/planner/binder/copy_into_table.rs index d10dea193440c..4c2652759705b 100644 --- a/src/query/sql/src/planner/binder/copy_into_table.rs +++ b/src/query/sql/src/planner/binder/copy_into_table.rs @@ -99,7 +99,7 @@ impl Binder { .await } CopyIntoTableSource::Query(query) => { - self.bind_cte_def(bind_context, &stmt.with)?; + self.init_cte(bind_context, &stmt.with)?; let mut max_column_position = MaxColumnPosition::new(); query.drive(&mut max_column_position); diff --git a/src/query/sql/src/planner/binder/insert.rs b/src/query/sql/src/planner/binder/insert.rs index aa374835cbb82..9cbfd4dc9b7df 100644 --- a/src/query/sql/src/planner/binder/insert.rs +++ b/src/query/sql/src/planner/binder/insert.rs @@ -111,7 +111,7 @@ impl Binder { .. } = stmt; - self.bind_cte_def(bind_context, with)?; + self.init_cte(bind_context, with)?; let table_identifier = TableIdentifier::new(self, catalog, database, table, &None); let (catalog_name, database_name, table_name) = ( From 70df77de3372f966e6bcd3296ff3cc226da1d545 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 19:14:33 +0800 Subject: [PATCH 31/80] refine --- src/query/sql/src/planner/binder/bind_context.rs | 6 ------ src/query/sql/src/planner/binder/bind_query/bind.rs | 1 - 2 files changed, 7 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 3a0d4179e77c4..758eae2c15169 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -200,12 +200,6 @@ pub struct CteInfo { pub columns: Vec, } -#[derive(Clone, Debug)] -pub struct CteBindResult { - pub s_expr: SExpr, - pub bind_context: BindContext, -} - impl BindContext { pub fn new() -> Self { Self { diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 315dbad48db93..9090af1ae1bd3 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -69,7 +69,6 @@ impl Binder { } } - // Bind cte definition. self.init_cte(bind_context, &with)?; // Extract limit and offset from query. From e38c70fd3be919a7ecc1034c34f2d814b5d52811 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 10 Jul 2025 20:20:34 +0800 Subject: [PATCH 32/80] make lint --- src/query/sql/src/planner/binder/bind_context.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/sql/src/planner/binder/bind_context.rs b/src/query/sql/src/planner/binder/bind_context.rs index 758eae2c15169..b0c1c7a7d6608 100644 --- a/src/query/sql/src/planner/binder/bind_context.rs +++ b/src/query/sql/src/planner/binder/bind_context.rs @@ -42,7 +42,6 @@ use crate::binder::project_set::SetReturningInfo; use crate::binder::window::WindowInfo; use crate::binder::ColumnBindingBuilder; use crate::normalize_identifier; -use crate::optimizer::ir::SExpr; use crate::plans::ScalarExpr; use crate::ColumnSet; use crate::IndexType; From 6829408a52fd2f1f22f11981401a7132be301e10 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Fri, 11 Jul 2025 04:43:22 +0800 Subject: [PATCH 33/80] fix --- .../pipelines/builders/builder_materialized_cte.rs | 8 ++++++++ .../sql/src/executor/physical_plan_visitor.rs | 1 + .../physical_plans/physical_materialized_cte.rs | 14 ++++++++++++-- .../binder/bind_table_reference/bind_cte.rs | 2 +- .../sql/src/planner/plans/materialized_cte.rs | 11 +++++++---- 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 4336c2691920e..e90efe0b06163 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -32,6 +32,14 @@ impl PipelineBuilder { // build cte pipeline let mut build_res = sub_builder.finalize(&cte.left)?; + let input_schema = cte.left.output_schema()?; + Self::build_result_projection( + &self.func_ctx, + input_schema, + &cte.cte_output_columns, + &mut build_res.main_pipeline, + false, + )?; build_res.main_pipeline.try_resize(1)?; let (tx, rx) = tokio::sync::watch::channel(Arc::new(MaterializedCteData::default())); self.cte_receivers.insert(cte.cte_name.clone(), rx); diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index cedc2e63a75d0..a98cedc17f71a 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -657,6 +657,7 @@ pub trait PhysicalPlanReplacer { right: Box::new(right), stat_info: plan.stat_info.clone(), cte_name: plan.cte_name.clone(), + cte_output_columns: plan.cte_output_columns.clone(), }))) } diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index 401278e26280d..fc61ae7ab3bea 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -19,6 +19,7 @@ use crate::executor::explain::PlanStatsInfo; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; +use crate::ColumnBinding; use crate::ColumnSet; /// This is a binary operator that executes its children in order (left to right), and returns the results of the right child @@ -31,6 +32,7 @@ pub struct MaterializedCTE { pub left: Box, pub right: Box, pub cte_name: String, + pub cte_output_columns: Vec, } impl MaterializedCTE { @@ -48,8 +50,15 @@ impl PhysicalPlanBuilder { required: ColumnSet, ) -> Result { let left_side = Box::new( - self.build(s_expr.child(0)?, materialized_cte.required.clone()) - .await?, + self.build( + s_expr.child(0)?, + materialized_cte + .cte_output_columns + .iter() + .map(|c| c.index) + .collect(), + ) + .await?, ); let right_side = Box::new(self.build(s_expr.child(1)?, required).await?); Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { @@ -58,6 +67,7 @@ impl PhysicalPlanBuilder { left: left_side, right: right_side, cte_name: materialized_cte.cte_name.clone(), + cte_output_columns: materialized_cte.cte_output_columns.clone(), }))) } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index 925117e218436..e4d067cd0d40f 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -177,7 +177,7 @@ impl Binder { let (s_expr, bind_context) = self.bind_cte_definition(&cte_name, &cte_context.cte_map, &cte.query)?; - let materialized_cte = MaterializedCTE::new(cte_name, bind_context.column_set()); + let materialized_cte = MaterializedCTE::new(cte_name, bind_context.columns); current_expr = SExpr::create_binary( Arc::new(materialized_cte.into()), Arc::new(s_expr), diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index dba74c7f23d49..523f05afb0abc 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -23,16 +23,19 @@ use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; -use crate::ColumnSet; +use crate::ColumnBinding; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, - pub required: ColumnSet, + pub cte_output_columns: Vec, } impl MaterializedCTE { - pub fn new(cte_name: String, required: ColumnSet) -> Self { - Self { cte_name, required } + pub fn new(cte_name: String, output_columns: Vec) -> Self { + Self { + cte_name, + cte_output_columns: output_columns, + } } } From bc4efde38b6aed20a9921ffde0bcd8c24616ea8a Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 13 Jul 2025 09:38:52 +0800 Subject: [PATCH 34/80] add log --- .../src/schedulers/fragments/query_fragment_actions.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs index ea2dccf3d2379..010067c701ce1 100644 --- a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs +++ b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs @@ -130,6 +130,13 @@ impl QueryFragmentsActions { pub fn get_root_fragment_ids(&self) -> Result> { let mut fragment_ids = Vec::new(); for fragment_actions in &self.fragments_actions { + if fragment_actions.fragment_actions.is_empty() { + return Err(ErrorCode::Internal(format!( + "Fragment actions is empty for fragment_id: {}", + fragment_actions.fragment_id + ))); + } + let plan = &fragment_actions.fragment_actions[0].physical_plan; if !matches!(plan, PhysicalPlan::ExchangeSink(_)) { fragment_ids.push(fragment_actions.fragment_id); From b5ccca9332c0bef7a9aa1a2091afac7c893377c8 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 13 Jul 2025 15:01:59 +0800 Subject: [PATCH 35/80] fix --- .../src/schedulers/fragments/fragmenter.rs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 2e7a6e878b819..eaef119265cf4 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -26,6 +26,7 @@ use databend_common_sql::executor::physical_plans::ExchangeSink; use databend_common_sql::executor::physical_plans::ExchangeSource; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::HashJoin; +use databend_common_sql::executor::physical_plans::MaterializedCTE; use databend_common_sql::executor::physical_plans::MutationSource; use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceInto; @@ -341,4 +342,23 @@ impl PhysicalPlanReplacer for Fragmenter { source_fragment_id, })) } + + fn replace_materialized_cte(&mut self, plan: &MaterializedCTE) -> Result { + let mut fragments = vec![]; + let left = self.replace(plan.left.as_ref())?; + + fragments.append(&mut self.fragments); + let right = self.replace(plan.right.as_ref())?; + fragments.append(&mut self.fragments); + + self.fragments = fragments; + Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { + plan_id: plan.plan_id, + left: Box::new(left), + right: Box::new(right), + stat_info: plan.stat_info.clone(), + cte_name: plan.cte_name.clone(), + cte_output_columns: plan.cte_output_columns.clone(), + }))) + } } From 2f897f4e06556a410ff41a7e98c285d7338b01a3 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 13 Jul 2025 16:29:04 +0800 Subject: [PATCH 36/80] fix --- src/query/service/src/schedulers/fragments/fragmenter.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index eaef119265cf4..29b36f051b3ce 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_meta_types::NodeInfo; +use databend_common_sql::executor::physical_plans::CTEConsumer; use databend_common_sql::executor::physical_plans::CompactSource; use databend_common_sql::executor::physical_plans::ConstantTableScan; use databend_common_sql::executor::physical_plans::CopyIntoTable; @@ -68,6 +69,7 @@ enum State { Compact, Recluster, Other, + CTEConsumer, } impl Fragmenter { @@ -161,6 +163,11 @@ impl PhysicalPlanReplacer for Fragmenter { Ok(PhysicalPlan::TableScan(plan.clone())) } + fn replace_cte_consumer(&mut self, plan: &CTEConsumer) -> Result { + self.state = State::CTEConsumer; + Ok(PhysicalPlan::CTEConsumer(Box::new(plan.clone()))) + } + fn replace_constant_table_scan(&mut self, plan: &ConstantTableScan) -> Result { self.state = State::SelectLeaf; Ok(PhysicalPlan::ConstantTableScan(plan.clone())) @@ -311,6 +318,7 @@ impl PhysicalPlanReplacer for Fragmenter { State::ReplaceInto => FragmentType::ReplaceInto, State::Compact => FragmentType::Compact, State::Recluster => FragmentType::Recluster, + State::CTEConsumer => FragmentType::Intermediate, }; self.state = State::Other; let exchange = Self::get_exchange(self.ctx.clone(), &plan)?; From cf7e7f97cc4a1960f19519de17e68c776465fa83 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 13 Jul 2025 16:30:30 +0800 Subject: [PATCH 37/80] make lint --- .../service/src/schedulers/fragments/query_fragment_actions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs index 010067c701ce1..9c0d5d42f888e 100644 --- a/src/query/service/src/schedulers/fragments/query_fragment_actions.rs +++ b/src/query/service/src/schedulers/fragments/query_fragment_actions.rs @@ -136,7 +136,7 @@ impl QueryFragmentsActions { fragment_actions.fragment_id ))); } - + let plan = &fragment_actions.fragment_actions[0].physical_plan; if !matches!(plan, PhysicalPlan::ExchangeSink(_)) { fragment_ids.push(fragment_actions.fragment_id); From 2a5f3714e180a6da538a8893942d04dbad63264c Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 14 Jul 2025 06:36:00 +0800 Subject: [PATCH 38/80] fix --- src/query/sql/src/planner/plans/cte_consumer.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index 01580bc10133c..fe40ee8070215 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -21,6 +21,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataSchemaRef; +use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; @@ -65,7 +66,9 @@ impl Operator for CTEConsumer { /// Derive physical property fn derive_physical_prop(&self, _rel_expr: &RelExpr) -> Result { - RelExpr::with_s_expr(&self.def).derive_physical_prop() + Ok(PhysicalProperty { + distribution: Distribution::Random, + }) } /// Compute required property for child with index `child_index` From 8b0efe5df41cb7dd3069ecbfeefcb9782cf13c10 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 15 Jul 2025 10:54:35 +0800 Subject: [PATCH 39/80] fix --- .../builders/builder_cte_consumer.rs | 27 +------ .../src/pipelines/builders/builder_join.rs | 2 - .../builders/builder_materialized_cte.rs | 13 +--- .../service/src/pipelines/pipeline_builder.rs | 7 -- .../processors/transforms/materialized_cte.rs | 76 ++++++++++--------- .../pipelines/processors/transforms/mod.rs | 1 + src/query/service/src/sessions/query_ctx.rs | 27 +++++++ .../service/src/sessions/query_ctx_shared.rs | 5 ++ 8 files changed, 79 insertions(+), 79 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs index 1fa78f91fbc7d..7eba2117cad13 100644 --- a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs +++ b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_sql::executor::physical_plans::CTEConsumer; use databend_common_storages_fuse::TableContext; @@ -22,32 +21,10 @@ use crate::pipelines::PipelineBuilder; impl PipelineBuilder { pub(crate) fn build_cte_consumer(&mut self, cte: &CTEConsumer) -> Result<()> { - let receiver = self - .cte_receivers - .get(&cte.cte_name) - .ok_or_else(|| { - ErrorCode::Internal(format!("CTE receiver not found for name: {}", cte.cte_name)) - })? - .clone(); - - let mut next_cte_consumer_id = self.next_cte_consumer_id.lock(); - let current_consumer_id = *next_cte_consumer_id.get(&cte.cte_name).ok_or_else(|| { - ErrorCode::Internal(format!( - "CTE consumer id not found for name: {}", - cte.cte_name - )) - })?; - - next_cte_consumer_id.insert(cte.cte_name.clone(), current_consumer_id + 1); - + let receiver = self.ctx.get_materialized_cte_receiver(&cte.cte_name); self.main_pipeline.add_source( |output_port| { - CTESource::create( - self.ctx.clone(), - output_port.clone(), - receiver.clone(), - current_consumer_id, - ) + CTESource::create(self.ctx.clone(), output_port.clone(), receiver.clone()) }, self.ctx.get_settings().get_max_threads()? as usize, )?; diff --git a/src/query/service/src/pipelines/builders/builder_join.rs b/src/query/service/src/pipelines/builders/builder_join.rs index 09b1b9a7bb3d7..7937b11b3344c 100644 --- a/src/query/service/src/pipelines/builders/builder_join.rs +++ b/src/query/service/src/pipelines/builders/builder_join.rs @@ -41,8 +41,6 @@ impl PipelineBuilder { let mut sub_builder = PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); sub_builder.hash_join_states = self.hash_join_states.clone(); - sub_builder.cte_receivers = self.cte_receivers.clone(); - sub_builder.next_cte_consumer_id = self.next_cte_consumer_id.clone(); sub_builder } diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index e90efe0b06163..b514a0f20535c 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -12,12 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; - use databend_common_exception::Result; use databend_common_sql::executor::physical_plans::MaterializedCTE; -use crate::pipelines::processors::transforms::MaterializedCteData; use crate::pipelines::processors::transforms::MaterializedCteSink; use crate::pipelines::PipelineBuilder; use crate::sessions::QueryContext; @@ -25,10 +22,8 @@ impl PipelineBuilder { pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { // init builder for cte pipeline let sub_context = QueryContext::create_from(self.ctx.as_ref()); - let mut sub_builder = + let sub_builder = PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); - sub_builder.cte_receivers = self.cte_receivers.clone(); - sub_builder.next_cte_consumer_id = self.next_cte_consumer_id.clone(); // build cte pipeline let mut build_res = sub_builder.finalize(&cte.left)?; @@ -41,11 +36,7 @@ impl PipelineBuilder { false, )?; build_res.main_pipeline.try_resize(1)?; - let (tx, rx) = tokio::sync::watch::channel(Arc::new(MaterializedCteData::default())); - self.cte_receivers.insert(cte.cte_name.clone(), rx); - self.next_cte_consumer_id - .lock() - .insert(cte.cte_name.clone(), 0); + let tx = self.ctx.get_materialized_cte_sender(&cte.cte_name); build_res .main_pipeline .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index c3b909d72075c..f6150f5466fe7 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -27,12 +27,9 @@ use databend_common_pipeline_core::ExecutionInfo; use databend_common_pipeline_core::Pipeline; use databend_common_settings::Settings; use databend_common_sql::executor::PhysicalPlan; -use parking_lot::Mutex; -use tokio::sync::watch::Receiver; use super::PipelineBuilderData; use crate::interpreters::CreateTableInterpreter; -use crate::pipelines::processors::transforms::MaterializedCteData; use crate::pipelines::processors::HashJoinBuildState; use crate::pipelines::processors::HashJoinState; use crate::pipelines::PipelineBuildResult; @@ -60,8 +57,6 @@ pub struct PipelineBuilder { pub(crate) is_exchange_stack: Vec, pub contain_sink_processor: bool, - pub cte_receivers: HashMap>>, - pub next_cte_consumer_id: Arc>>, } impl PipelineBuilder { @@ -83,8 +78,6 @@ impl PipelineBuilder { r_cte_scan_interpreters: vec![], contain_sink_processor: false, is_exchange_stack: vec![], - cte_receivers: HashMap::new(), - next_cte_consumer_id: Arc::new(Mutex::new(HashMap::new())), } } diff --git a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs index 092ae0ac94366..8670eb08b57c2 100644 --- a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs +++ b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs @@ -1,13 +1,3 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::Mutex; - -use databend_common_exception::ErrorCode; -use databend_common_exception::Result; -use databend_common_expression::DataBlock; -use databend_common_pipeline_core::processors::InputPort; -use databend_common_pipeline_core::processors::OutputPort; -use databend_common_pipeline_core::processors::ProcessorPtr; // Copyright 2021 Datafuse Labs // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,10 +11,23 @@ use databend_common_pipeline_core::processors::ProcessorPtr; // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_expression::DataBlock; +use databend_common_pipeline_core::processors::InputPort; +use databend_common_pipeline_core::processors::OutputPort; +use databend_common_pipeline_core::processors::ProcessorPtr; use databend_common_pipeline_sinks::Sink; use databend_common_pipeline_sinks::Sinker; use databend_common_pipeline_sources::AsyncSource; use databend_common_pipeline_sources::AsyncSourcer; +use databend_common_storages_fuse::TableContext; +use tokio::sync::watch; use tokio::sync::watch::Receiver; use tokio::sync::watch::Sender; @@ -33,32 +36,17 @@ pub struct MaterializedCteSink { blocks: Vec, } -#[derive(Default)] pub struct MaterializedCteData { blocks: Vec, - // consumer_id -> current_index - consumer_states: Arc>>, } impl MaterializedCteData { pub fn new(blocks: Vec) -> Self { - Self { - blocks, - consumer_states: Arc::new(Mutex::new(HashMap::new())), - } + Self { blocks } } - pub fn get_next_block(&self, consumer_id: usize) -> Option { - let mut states = self.consumer_states.lock().unwrap(); - let current_index = states.get(&consumer_id).copied().unwrap_or(0); - - if current_index < self.blocks.len() { - let block = self.blocks[current_index].clone(); - states.insert(consumer_id, current_index + 1); - Some(block) - } else { - None - } + pub fn get_data_block_at(&self, index: usize) -> Option { + self.blocks.get(index).cloned() } } @@ -84,7 +72,9 @@ impl Sink for MaterializedCteSink { fn on_finish(&mut self) -> Result<()> { self.sender - .send(Arc::new(MaterializedCteData::new(self.blocks.clone()))) + .send(Arc::new(MaterializedCteData::new(std::mem::take( + &mut self.blocks, + )))) .map_err(|_| { ErrorCode::Internal("Failed to send blocks to materialized cte consumer") })?; @@ -95,20 +85,19 @@ impl Sink for MaterializedCteSink { pub struct CTESource { receiver: Receiver>, data: Option>, - consumer_id: usize, + next_block_id: Arc, } impl CTESource { pub fn create( - ctx: Arc, + ctx: Arc, output_port: Arc, receiver: Receiver>, - consumer_id: usize, ) -> Result { AsyncSourcer::create(ctx, output_port, Self { receiver, data: None, - consumer_id, + next_block_id: Arc::new(AtomicUsize::new(0)), }) } } @@ -127,10 +116,29 @@ impl AsyncSource for CTESource { } if let Some(data) = &self.data { - if let Some(block) = data.get_next_block(self.consumer_id) { + let id = self.next_block_id.fetch_add(1, Ordering::Relaxed); + if let Some(block) = data.get_data_block_at(id) { return Ok(Some(block)); } } Ok(None) } } + +pub struct MaterializedCteChannel { + pub sender: Sender>, + pub receiver: Receiver>, +} + +impl MaterializedCteChannel { + pub fn new() -> Self { + let (sender, receiver) = watch::channel(Arc::new(MaterializedCteData::new(vec![]))); + Self { sender, receiver } + } +} + +impl Default for MaterializedCteChannel { + fn default() -> Self { + Self::new() + } +} diff --git a/src/query/service/src/pipelines/processors/transforms/mod.rs b/src/query/service/src/pipelines/processors/transforms/mod.rs index 2a619c974aaf4..f8beebf5134ee 100644 --- a/src/query/service/src/pipelines/processors/transforms/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/mod.rs @@ -47,6 +47,7 @@ pub use broadcast::BroadcastSinkProcessor; pub use broadcast::BroadcastSourceProcessor; pub use hash_join::*; pub use materialized_cte::CTESource; +pub use materialized_cte::MaterializedCteChannel; pub use materialized_cte::MaterializedCteData; pub use materialized_cte::MaterializedCteSink; pub use transform_add_computed_columns::TransformAddComputedColumns; diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 51e9018842023..9b172dfe516d3 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -139,6 +139,7 @@ use crate::clusters::Cluster; use crate::clusters::ClusterHelper; use crate::locks::LockManager; use crate::pipelines::executor::PipelineExecutor; +use crate::pipelines::processors::transforms::MaterializedCteData; use crate::servers::flight::v1::exchange::DataExchangeManager; use crate::sessions::query_affect::QueryAffect; use crate::sessions::query_ctx_shared::MemoryUpdater; @@ -584,6 +585,32 @@ impl QueryContext { pub fn clear_table_meta_timestamps_cache(&self) { self.shared.table_meta_timestamps.lock().clear(); } + + pub fn get_materialized_cte_sender( + &self, + cte_name: &str, + ) -> tokio::sync::watch::Sender> { + self.shared + .materialized_cte_channels + .lock() + .entry(cte_name.to_string()) + .or_default() + .sender + .clone() + } + + pub fn get_materialized_cte_receiver( + &self, + cte_name: &str, + ) -> tokio::sync::watch::Receiver> { + self.shared + .materialized_cte_channels + .lock() + .entry(cte_name.to_string()) + .or_default() + .receiver + .clone() + } } #[async_trait::async_trait] diff --git a/src/query/service/src/sessions/query_ctx_shared.rs b/src/query/service/src/sessions/query_ctx_shared.rs index 9547f4fe1d539..cd1f7b6215df7 100644 --- a/src/query/service/src/sessions/query_ctx_shared.rs +++ b/src/query/service/src/sessions/query_ctx_shared.rs @@ -72,6 +72,7 @@ use uuid::Uuid; use crate::clusters::Cluster; use crate::clusters::ClusterDiscovery; use crate::pipelines::executor::PipelineExecutor; +use crate::pipelines::processors::transforms::MaterializedCteChannel; use crate::sessions::query_affect::QueryAffect; use crate::sessions::Session; use crate::storages::Table; @@ -182,6 +183,9 @@ pub struct QueryContextShared { // QueryPerf used to draw flamegraph pub(in crate::sessions) perf_flag: AtomicBool, pub(in crate::sessions) nodes_perf: Arc>>, + + pub(in crate::sessions) materialized_cte_channels: + Arc>>, } #[derive(Default)] @@ -259,6 +263,7 @@ impl QueryContextShared { broadcast_channels: Arc::new(Mutex::new(HashMap::new())), perf_flag: AtomicBool::new(false), nodes_perf: Arc::new(Mutex::new(HashMap::new())), + materialized_cte_channels: Arc::new(Mutex::new(HashMap::new())), })) } From 38067f5f0d0d6ad1027d1a54810f4a10a2ab582e Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 15 Jul 2025 14:43:05 +0800 Subject: [PATCH 40/80] fix --- .../src/pipelines/builders/builder_cte_consumer.rs | 11 ++++++++++- .../processors/transforms/materialized_cte.rs | 3 ++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs index 7eba2117cad13..4cc684dde7d28 100644 --- a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs +++ b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::atomic::AtomicUsize; +use std::sync::Arc; + use databend_common_exception::Result; use databend_common_sql::executor::physical_plans::CTEConsumer; use databend_common_storages_fuse::TableContext; @@ -22,9 +25,15 @@ use crate::pipelines::PipelineBuilder; impl PipelineBuilder { pub(crate) fn build_cte_consumer(&mut self, cte: &CTEConsumer) -> Result<()> { let receiver = self.ctx.get_materialized_cte_receiver(&cte.cte_name); + let next_block_id = Arc::new(AtomicUsize::new(0)); self.main_pipeline.add_source( |output_port| { - CTESource::create(self.ctx.clone(), output_port.clone(), receiver.clone()) + CTESource::create( + self.ctx.clone(), + output_port.clone(), + receiver.clone(), + next_block_id.clone(), + ) }, self.ctx.get_settings().get_max_threads()? as usize, )?; diff --git a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs index 8670eb08b57c2..a0cfaa8fe5f33 100644 --- a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs +++ b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs @@ -93,11 +93,12 @@ impl CTESource { ctx: Arc, output_port: Arc, receiver: Receiver>, + next_block_id: Arc, ) -> Result { AsyncSourcer::create(ctx, output_port, Self { receiver, data: None, - next_block_id: Arc::new(AtomicUsize::new(0)), + next_block_id, }) } } From 3758bbd639140708782224501a93049be4ca2576 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 15 Jul 2025 15:39:59 +0800 Subject: [PATCH 41/80] fix --- src/query/service/src/schedulers/fragments/fragmenter.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 29b36f051b3ce..a43c9cbf29840 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -69,7 +69,6 @@ enum State { Compact, Recluster, Other, - CTEConsumer, } impl Fragmenter { @@ -164,7 +163,6 @@ impl PhysicalPlanReplacer for Fragmenter { } fn replace_cte_consumer(&mut self, plan: &CTEConsumer) -> Result { - self.state = State::CTEConsumer; Ok(PhysicalPlan::CTEConsumer(Box::new(plan.clone()))) } @@ -318,7 +316,6 @@ impl PhysicalPlanReplacer for Fragmenter { State::ReplaceInto => FragmentType::ReplaceInto, State::Compact => FragmentType::Compact, State::Recluster => FragmentType::Recluster, - State::CTEConsumer => FragmentType::Intermediate, }; self.state = State::Other; let exchange = Self::get_exchange(self.ctx.clone(), &plan)?; From 488a08d98e748fffc6c6d6d7b1c416759e7fd016 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 15 Jul 2025 16:35:25 +0800 Subject: [PATCH 42/80] fix --- src/query/service/src/schedulers/fragments/fragmenter.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index a43c9cbf29840..43c79ff82bbef 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -353,6 +353,7 @@ impl PhysicalPlanReplacer for Fragmenter { let left = self.replace(plan.left.as_ref())?; fragments.append(&mut self.fragments); + self.state = State::Other; let right = self.replace(plan.right.as_ref())?; fragments.append(&mut self.fragments); From f24bb7f666aafffaa316c7e3da3b17e38dc18ff3 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 16 Jul 2025 14:43:31 +0800 Subject: [PATCH 43/80] disable distributed optimization --- src/query/sql/src/planner/optimizer/pipeline/common.rs | 6 ++++++ src/query/sql/src/planner/optimizer/pipeline/pipeline.rs | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/query/sql/src/planner/optimizer/pipeline/common.rs b/src/query/sql/src/planner/optimizer/pipeline/common.rs index 26aa6e6556415..40306a21c0ca0 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/common.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/common.rs @@ -59,3 +59,9 @@ pub fn contains_warehouse_table_scan(s_expr: &SExpr, metadata: &MetadataRef) -> false } + +/// Check if a query contains MaterializedCTE operators. +pub fn contains_materialized_cte(s_expr: &SExpr) -> bool { + s_expr.children().any(contains_materialized_cte) + || matches!(s_expr.plan(), RelOperator::MaterializedCTE(_)) +} diff --git a/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs b/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs index f90580450f416..fd3397e6b42de 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs @@ -19,6 +19,7 @@ use databend_common_exception::Result; use log::info; use super::common::contains_local_table_scan; +use super::common::contains_materialized_cte; use super::common::contains_warehouse_table_scan; use crate::optimizer::ir::Memo; use crate::optimizer::ir::SExpr; @@ -98,6 +99,12 @@ impl OptimizerPipeline { } } + // Check if the plan contains MaterializedCTE, if so, disable distributed optimization + if contains_materialized_cte(s_expr) { + self.opt_ctx.set_enable_distributed_optimization(false); + info!("Disable distributed optimization due to MaterializedCTE."); + } + Ok(()) } From 00b299a80ffc70878f03224fd8644cdd2f6dde07 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 16 Jul 2025 15:17:32 +0800 Subject: [PATCH 44/80] fix merge --- src/query/service/src/sessions/query_ctx.rs | 1 - src/query/sql/src/planner/plans/operator.rs | 4 +++- src/query/sql/src/planner/plans/operator_macros.rs | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 0916ab4b57964..9b172dfe516d3 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -141,7 +141,6 @@ use crate::locks::LockManager; use crate::pipelines::executor::PipelineExecutor; use crate::pipelines::processors::transforms::MaterializedCteData; use crate::servers::flight::v1::exchange::DataExchangeManager; -use crate::servers::http::v1::ClientSessionManager; use crate::sessions::query_affect::QueryAffect; use crate::sessions::query_ctx_shared::MemoryUpdater; use crate::sessions::ProcessInfo; diff --git a/src/query/sql/src/planner/plans/operator.rs b/src/query/sql/src/planner/plans/operator.rs index bb820e4f6b34e..1d004f998974f 100644 --- a/src/query/sql/src/planner/plans/operator.rs +++ b/src/query/sql/src/planner/plans/operator.rs @@ -255,5 +255,7 @@ impl_try_from_rel_operator! { AsyncFunction, Mutation, CompactBlock, - MutationSource + MutationSource, + MaterializedCTE, + CTEConsumer } diff --git a/src/query/sql/src/planner/plans/operator_macros.rs b/src/query/sql/src/planner/plans/operator_macros.rs index 4b90525c53f43..4695bc7f6e4ca 100644 --- a/src/query/sql/src/planner/plans/operator_macros.rs +++ b/src/query/sql/src/planner/plans/operator_macros.rs @@ -110,6 +110,8 @@ macro_rules! impl_match_rel_op { RelOperator::Mutation($rel_op) => $rel_op.$method($($arg),*), RelOperator::CompactBlock($rel_op) => $rel_op.$method($($arg),*), RelOperator::MutationSource($rel_op) => $rel_op.$method($($arg),*), + RelOperator::MaterializedCTE($rel_op) => $rel_op.$method($($arg),*), + RelOperator::CTEConsumer($rel_op) => $rel_op.$method($($arg),*), } } } From ad405bbae993140f698c4578998e0e23f7398565 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 17 Jul 2025 09:52:09 +0800 Subject: [PATCH 45/80] fix explain join --- src/query/sql/src/executor/format.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 82b19df5404d2..1ebd9c2a7d6f0 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -176,6 +176,24 @@ impl PhysicalPlan { children, )) } + PhysicalPlan::MaterializedCTE(plan) => { + let left_child = plan.left.format_join(metadata)?; + let right_child = plan.right.format_join(metadata)?; + + let children = vec![ + FormatTreeNode::with_children("CTE".to_string(), vec![left_child]), + FormatTreeNode::with_children("Main".to_string(), vec![right_child]), + ]; + + Ok(FormatTreeNode::with_children( + format!("MaterializedCTE: {}", plan.cte_name), + children, + )) + } + PhysicalPlan::CTEConsumer(plan) => Ok(FormatTreeNode::with_children( + format!("CTEConsumer: {}", plan.cte_name), + vec![], + )), other => { let children = other .children() From a5853a074f38ab2945c0c948b92321fce10d3677 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 17 Jul 2025 10:46:55 +0800 Subject: [PATCH 46/80] fix logic test --- src/query/sql/src/executor/format.rs | 5 +---- .../sqllogictests/suites/tpch/join_order.test | 20 ++++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 1ebd9c2a7d6f0..285eb2df267ec 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -180,10 +180,7 @@ impl PhysicalPlan { let left_child = plan.left.format_join(metadata)?; let right_child = plan.right.format_join(metadata)?; - let children = vec![ - FormatTreeNode::with_children("CTE".to_string(), vec![left_child]), - FormatTreeNode::with_children("Main".to_string(), vec![right_child]), - ]; + let children = vec![left_child, right_child]; Ok(FormatTreeNode::with_children( format!("MaterializedCTE: {}", plan.cte_name), diff --git a/tests/sqllogictests/suites/tpch/join_order.test b/tests/sqllogictests/suites/tpch/join_order.test index 70eee27cd6481..8bddcf3a12c90 100644 --- a/tests/sqllogictests/suites/tpch/join_order.test +++ b/tests/sqllogictests/suites/tpch/join_order.test @@ -724,15 +724,17 @@ where order by s_suppkey; ---- -HashJoin: INNER -├── Build -│ └── Scan: default.tpch_test.revenue (#2) (read rows: 10000) -└── Probe - └── HashJoin: INNER - ├── Build - │ └── Scan: default.tpch_test.revenue (#1) (read rows: 10000) - └── Probe - └── Scan: default.tpch_test.supplier (#0) (read rows: 10000) +MaterializedCTE: revenue +├── Scan: default.tpch_test.lineitem (#3) (read rows: 6001215) +└── HashJoin: INNER + ├── Build + │ └── HashJoin: INNER + │ ├── Build + │ │ └── CTEConsumer: revenue + │ └── Probe + │ └── Scan: default.tpch_test.supplier (#0) (read rows: 10000) + └── Probe + └── CTEConsumer: revenue # Q16 From 0d8ee4ec4faf3af04c34ced849961f4fc038d382 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 17 Jul 2025 10:58:53 +0800 Subject: [PATCH 47/80] fix logic test --- .../mode/standalone/explain/explain.test | 1 - .../standalone/explain/materialized_cte.test | 98 +++++++++---------- .../push_down_filter_self_join.test | 71 ++++++++------ .../explain/selectivity/is_not_null.test | 2 +- 4 files changed, 91 insertions(+), 81 deletions(-) diff --git a/tests/sqllogictests/suites/mode/standalone/explain/explain.test b/tests/sqllogictests/suites/mode/standalone/explain/explain.test index 47b1e66801fd3..4847ef28b2caa 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/explain.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/explain.test @@ -1705,4 +1705,3 @@ HashJoin ├── push downs: [filters: [NOT CAST(numbers.number (#0) AS Boolean)], limit: NONE] ├── apply join filters: [#0] └── estimated rows: 10.00 - diff --git a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test index 029831f9d4334..280d28b5c02a8 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test @@ -1,65 +1,63 @@ query T explain with t1 as materialized (select number as a from numbers(10)), t2 as (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b; ---- -HashJoin -├── output columns: [t1.a (#0)] -├── join type: INNER -├── build keys: [t2.b (#1)] -├── probe keys: [t1.a (#0)] -├── keys is null equal: [false] -├── filters: [] -├── build join filters: -│ └── filter id:0, build key:t2.b (#1), probe key:t1.a (#0), filter type:inlist,min_max -├── estimated rows: 100.00 -├── TableScan(Build) -│ ├── table: default.default.t1 -│ ├── output columns: [a (#1)] +MaterializedCTE: t1 +├── TableScan +│ ├── table: default.system.numbers +│ ├── output columns: [number (#2)] │ ├── read rows: 10 │ ├── read size: < 1 KiB -│ ├── partitions total: 0 -│ ├── partitions scanned: 0 +│ ├── partitions total: 1 +│ ├── partitions scanned: 1 │ ├── push downs: [filters: [], limit: NONE] │ └── estimated rows: 10.00 -└── TableScan(Probe) - ├── table: default.default.t1 - ├── output columns: [a (#0)] - ├── read rows: 10 - ├── read size: < 1 KiB - ├── partitions total: 0 - ├── partitions scanned: 0 - ├── push downs: [filters: [], limit: NONE] - ├── apply join filters: [#0] - └── estimated rows: 10.00 +└── HashJoin + ├── output columns: [numbers.number (#0)] + ├── join type: INNER + ├── build keys: [t2.b (#1)] + ├── probe keys: [t1.a (#0)] + ├── keys is null equal: [false] + ├── filters: [] + ├── build join filters: + │ └── filter id:0, build key:t2.b (#1), probe key:t1.a (#0), filter type:bloom,inlist,min_max + ├── estimated rows: 0.00 + ├── CTEConsumer(Build) + │ ├── cte_name: t1 + │ └── cte_schema: [number (#1)] + └── CTEConsumer(Probe) + ├── cte_name: t1 + └── cte_schema: [number (#0)] query T explain with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b; ---- -HashJoin -├── output columns: [t1.a (#0)] -├── join type: INNER -├── build keys: [t2.b (#1)] -├── probe keys: [t1.a (#0)] -├── keys is null equal: [false] -├── filters: [] -├── build join filters: -│ └── filter id:0, build key:t2.b (#1), probe key:t1.a (#0), filter type:inlist,min_max -├── estimated rows: 100.00 -├── TableScan(Build) -│ ├── table: default.default.t2 -│ ├── output columns: [b (#1)] +MaterializedCTE: t1 +├── TableScan +│ ├── table: default.system.numbers +│ ├── output columns: [number (#3)] │ ├── read rows: 10 │ ├── read size: < 1 KiB -│ ├── partitions total: 0 -│ ├── partitions scanned: 0 +│ ├── partitions total: 1 +│ ├── partitions scanned: 1 │ ├── push downs: [filters: [], limit: NONE] │ └── estimated rows: 10.00 -└── TableScan(Probe) - ├── table: default.default.t1 - ├── output columns: [a (#0)] - ├── read rows: 10 - ├── read size: < 1 KiB - ├── partitions total: 0 - ├── partitions scanned: 0 - ├── push downs: [filters: [], limit: NONE] - ├── apply join filters: [#0] - └── estimated rows: 10.00 +└── MaterializedCTE: t2 + ├── CTEConsumer + │ ├── cte_name: t1 + │ └── cte_schema: [number (#2)] + └── HashJoin + ├── output columns: [numbers.number (#0)] + ├── join type: INNER + ├── build keys: [t2.b (#1)] + ├── probe keys: [t1.a (#0)] + ├── keys is null equal: [false] + ├── filters: [] + ├── build join filters: + │ └── filter id:0, build key:t2.b (#1), probe key:t1.a (#0), filter type:bloom,inlist,min_max + ├── estimated rows: 0.00 + ├── CTEConsumer(Build) + │ ├── cte_name: t2 + │ └── cte_schema: [number (#1)] + └── CTEConsumer(Probe) + ├── cte_name: t1 + └── cte_schema: [number (#0)] diff --git a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test index 221ebc9c3ac23..f803bc97d4530 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test @@ -11,32 +11,45 @@ C as (select * from B as b1 left outer join B as b2 on b1.a = b2.a where b1.b < D as (select * from C) select * from D; ---- -HashJoin -├── output columns: [b1.a (#0), b1.b (#1), b2.b (#3), b2.a (#2)] -├── join type: INNER -├── build keys: [b2.a (#2)] -├── probe keys: [b1.a (#0)] -├── keys is null equal: [false] -├── filters: [d.b (#1) < d.b (#3)] -├── build join filters: -│ └── filter id:0, build key:b2.a (#2), probe key:b1.a (#0), filter type:inlist,min_max -├── estimated rows: 400.00 -├── TableScan(Build) -│ ├── table: default.default.b -│ ├── output columns: [a (#2), b (#3)] -│ ├── read rows: 20 -│ ├── read size: < 1 KiB -│ ├── partitions total: 0 -│ ├── partitions scanned: 0 -│ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 20.00 -└── TableScan(Probe) - ├── table: default.default.b - ├── output columns: [a (#0), b (#1)] - ├── read rows: 20 - ├── read size: < 1 KiB - ├── partitions total: 0 - ├── partitions scanned: 0 - ├── push downs: [filters: [], limit: NONE] - ├── apply join filters: [#0] - └── estimated rows: 20.00 +MaterializedCTE: a +├── UnionAll +│ ├── output columns: [a (#22), b (#23)] +│ ├── estimated rows: 20.00 +│ ├── TableScan +│ │ ├── table: default.default.t1 +│ │ ├── output columns: [a (#18), b (#19)] +│ │ ├── read rows: 10 +│ │ ├── read size: < 1 KiB +│ │ ├── partitions total: 1 +│ │ ├── partitions scanned: 1 +│ │ ├── pruning stats: [segments: , blocks: ] +│ │ ├── push downs: [filters: [], limit: NONE] +│ │ └── estimated rows: 10.00 +│ └── TableScan +│ ├── table: default.default.t2 +│ ├── output columns: [a (#20), b (#21)] +│ ├── read rows: 10 +│ ├── read size: < 1 KiB +│ ├── partitions total: 1 +│ ├── partitions scanned: 1 +│ ├── pruning stats: [segments: , blocks: ] +│ ├── push downs: [filters: [], limit: NONE] +│ └── estimated rows: 10.00 +└── MaterializedCTE: b + ├── CTEConsumer + │ ├── cte_name: a + │ └── cte_schema: [a (#16), b (#17)] + └── HashJoin + ├── output columns: [a (#4), b (#5), b (#11), a (#10)] + ├── join type: INNER + ├── build keys: [b2.a (#10)] + ├── probe keys: [b1.a (#4)] + ├── keys is null equal: [false] + ├── filters: [d.b (#5) < d.b (#11)] + ├── estimated rows: 0.00 + ├── CTEConsumer(Build) + │ ├── cte_name: b + │ └── cte_schema: [a (#10), b (#11)] + └── CTEConsumer(Probe) + ├── cte_name: b + └── cte_schema: [a (#4), b (#5)] diff --git a/tests/sqllogictests/suites/mode/standalone/explain/selectivity/is_not_null.test b/tests/sqllogictests/suites/mode/standalone/explain/selectivity/is_not_null.test index a4c42da235e36..5c41e4a9a5e4c 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/selectivity/is_not_null.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/selectivity/is_not_null.test @@ -50,4 +50,4 @@ Filter └── estimated rows: 2.00 statement ok -drop table t_user; \ No newline at end of file +drop table t_user; From 321a70a4ff8ec7a71012bdcf38e884463febe6da Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 21 Jul 2025 19:14:38 +0800 Subject: [PATCH 48/80] add ref count --- .../src/schedulers/fragments/fragmenter.rs | 1 + .../sql/src/executor/physical_plan_visitor.rs | 1 + .../physical_materialized_cte.rs | 2 + .../sql/src/planner/binder/bind_query/bind.rs | 41 +++++++++++++------ .../binder/bind_table_reference/bind_cte.rs | 7 +++- .../sql/src/planner/plans/materialized_cte.rs | 4 +- 6 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 43c79ff82bbef..65f93b0316745 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -365,6 +365,7 @@ impl PhysicalPlanReplacer for Fragmenter { stat_info: plan.stat_info.clone(), cte_name: plan.cte_name.clone(), cte_output_columns: plan.cte_output_columns.clone(), + ref_count: plan.ref_count, }))) } } diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index a98cedc17f71a..fa54e29d080cc 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -658,6 +658,7 @@ pub trait PhysicalPlanReplacer { stat_info: plan.stat_info.clone(), cte_name: plan.cte_name.clone(), cte_output_columns: plan.cte_output_columns.clone(), + ref_count: plan.ref_count, }))) } diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index fc61ae7ab3bea..8b787e4179f13 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -33,6 +33,7 @@ pub struct MaterializedCTE { pub right: Box, pub cte_name: String, pub cte_output_columns: Vec, + pub ref_count: usize, } impl MaterializedCTE { @@ -68,6 +69,7 @@ impl PhysicalPlanBuilder { right: right_side, cte_name: materialized_cte.cte_name.clone(), cte_output_columns: materialized_cte.cte_output_columns.clone(), + ref_count: materialized_cte.ref_count, }))) } } diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index 9090af1ae1bd3..ec7190bbce4ae 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -85,13 +85,39 @@ impl Binder { s_expr = self.bind_query_limit(query, s_expr, limit, offset); if let Some(with) = &with { - s_expr = self.bind_materialized_cte(with, s_expr, bind_context.cte_context.clone())?; + let cte_ref_count = self.compute_cte_ref_count(with, query)?; + s_expr = self.bind_materialized_cte( + with, + s_expr, + bind_context.cte_context.clone(), + &cte_ref_count, + )?; } Ok((s_expr, bind_context)) } fn auto_materialize_cte(&mut self, with: &mut With, query: &Query) -> Result<()> { + let cte_ref_count = self.compute_cte_ref_count(with, query)?; + + // Update materialization based on reference count + for cte in with.ctes.iter_mut() { + let table_name = self.normalize_identifier(&cte.alias.name).name; + if let Some(count) = cte_ref_count.get(&table_name) { + log::info!("[CTE]cte_ref_count: {table_name} {count}"); + // Materialize if referenced more than once + cte.materialized |= *count > 1; + } + } + + Ok(()) + } + + pub fn compute_cte_ref_count( + &self, + with: &With, + query: &Query, + ) -> Result> { // Initialize the count of each CTE to 0 let mut cte_ref_count: HashMap = HashMap::new(); for cte in with.ctes.iter() { @@ -105,19 +131,8 @@ impl Binder { name_resolution_ctx: self.name_resolution_ctx.clone(), }; query.drive(&mut visitor); - cte_ref_count = visitor.cte_ref_count; - // Update materialization based on reference count - for cte in with.ctes.iter_mut() { - let table_name = self.normalize_identifier(&cte.alias.name).name; - if let Some(count) = cte_ref_count.get(&table_name) { - log::info!("[CTE]cte_ref_count: {table_name} {count}"); - // Materialize if referenced more than once - cte.materialized |= *count > 1; - } - } - - Ok(()) + Ok(visitor.cte_ref_count) } pub(crate) fn bind_query_order_by( diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index e4d067cd0d40f..f1c243bdbac22 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::sync::Arc; use databend_common_ast::ast::Query; @@ -168,6 +169,7 @@ impl Binder { with: &With, main_query_expr: SExpr, cte_context: CteContext, + cte_ref_count: &HashMap, ) -> Result { let mut current_expr = main_query_expr; @@ -177,7 +179,10 @@ impl Binder { let (s_expr, bind_context) = self.bind_cte_definition(&cte_name, &cte_context.cte_map, &cte.query)?; - let materialized_cte = MaterializedCTE::new(cte_name, bind_context.columns); + let ref_count = cte_ref_count.get(&cte_name).unwrap(); + + let materialized_cte = + MaterializedCTE::new(cte_name, bind_context.columns, *ref_count); current_expr = SExpr::create_binary( Arc::new(materialized_cte.into()), Arc::new(s_expr), diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index 523f05afb0abc..96ca542a7000a 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -28,13 +28,15 @@ use crate::ColumnBinding; pub struct MaterializedCTE { pub cte_name: String, pub cte_output_columns: Vec, + pub ref_count: usize, } impl MaterializedCTE { - pub fn new(cte_name: String, output_columns: Vec) -> Self { + pub fn new(cte_name: String, output_columns: Vec, ref_count: usize) -> Self { Self { cte_name, cte_output_columns: output_columns, + ref_count, } } } From a5949c1c06f64bd292b920e9751f15a942860383 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Mon, 21 Jul 2025 20:04:19 +0800 Subject: [PATCH 49/80] refactor: streaming CTE consumption --- .../builders/builder_cte_consumer.rs | 11 +- .../builders/builder_materialized_cte.rs | 4 +- .../processors/transforms/materialized_cte.rs | 108 +++++------------- .../pipelines/processors/transforms/mod.rs | 2 - src/query/service/src/sessions/query_ctx.rs | 37 +++--- .../service/src/sessions/query_ctx_shared.rs | 8 +- 6 files changed, 52 insertions(+), 118 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs index 4cc684dde7d28..7eba2117cad13 100644 --- a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs +++ b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::AtomicUsize; -use std::sync::Arc; - use databend_common_exception::Result; use databend_common_sql::executor::physical_plans::CTEConsumer; use databend_common_storages_fuse::TableContext; @@ -25,15 +22,9 @@ use crate::pipelines::PipelineBuilder; impl PipelineBuilder { pub(crate) fn build_cte_consumer(&mut self, cte: &CTEConsumer) -> Result<()> { let receiver = self.ctx.get_materialized_cte_receiver(&cte.cte_name); - let next_block_id = Arc::new(AtomicUsize::new(0)); self.main_pipeline.add_source( |output_port| { - CTESource::create( - self.ctx.clone(), - output_port.clone(), - receiver.clone(), - next_block_id.clone(), - ) + CTESource::create(self.ctx.clone(), output_port.clone(), receiver.clone()) }, self.ctx.get_settings().get_max_threads()? as usize, )?; diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index b514a0f20535c..b4ca67cf50dce 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -36,7 +36,9 @@ impl PipelineBuilder { false, )?; build_res.main_pipeline.try_resize(1)?; - let tx = self.ctx.get_materialized_cte_sender(&cte.cte_name); + let tx = self + .ctx + .get_materialized_cte_senders(&cte.cte_name, cte.ref_count); build_res .main_pipeline .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; diff --git a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs index a0cfaa8fe5f33..4a7317863de5d 100644 --- a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs +++ b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs @@ -12,94 +12,66 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering; use std::sync::Arc; +use async_channel::Receiver; +use async_channel::Sender; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_pipeline_core::processors::InputPort; use databend_common_pipeline_core::processors::OutputPort; use databend_common_pipeline_core::processors::ProcessorPtr; -use databend_common_pipeline_sinks::Sink; -use databend_common_pipeline_sinks::Sinker; +use databend_common_pipeline_sinks::AsyncSink; +use databend_common_pipeline_sinks::AsyncSinker; use databend_common_pipeline_sources::AsyncSource; use databend_common_pipeline_sources::AsyncSourcer; use databend_common_storages_fuse::TableContext; -use tokio::sync::watch; -use tokio::sync::watch::Receiver; -use tokio::sync::watch::Sender; pub struct MaterializedCteSink { - sender: Sender>, - blocks: Vec, -} - -pub struct MaterializedCteData { - blocks: Vec, -} - -impl MaterializedCteData { - pub fn new(blocks: Vec) -> Self { - Self { blocks } - } - - pub fn get_data_block_at(&self, index: usize) -> Option { - self.blocks.get(index).cloned() - } + senders: Vec>, } impl MaterializedCteSink { - pub fn create( - input: Arc, - sender: Sender>, - ) -> Result { - Ok(ProcessorPtr::create(Sinker::create(input, Self { - blocks: vec![], - sender, + pub fn create(input: Arc, senders: Vec>) -> Result { + Ok(ProcessorPtr::create(AsyncSinker::create(input, Self { + senders, }))) } } -impl Sink for MaterializedCteSink { +#[async_trait::async_trait] +impl AsyncSink for MaterializedCteSink { const NAME: &'static str = "MaterializedCteSink"; - fn consume(&mut self, data_block: DataBlock) -> Result<()> { - self.blocks.push(data_block); - Ok(()) - } - - fn on_finish(&mut self) -> Result<()> { - self.sender - .send(Arc::new(MaterializedCteData::new(std::mem::take( - &mut self.blocks, - )))) - .map_err(|_| { + async fn consume(&mut self, data_block: DataBlock) -> Result { + for sender in self.senders.iter() { + sender.send(data_block.clone()).await.map_err(|_| { ErrorCode::Internal("Failed to send blocks to materialized cte consumer") })?; + } + Ok(false) + } + + async fn on_finish(&mut self) -> Result<()> { + for sender in self.senders.iter() { + sender.close(); + } Ok(()) } } pub struct CTESource { - receiver: Receiver>, - data: Option>, - next_block_id: Arc, + receiver: Receiver, } impl CTESource { pub fn create( ctx: Arc, output_port: Arc, - receiver: Receiver>, - next_block_id: Arc, + receiver: Receiver, ) -> Result { - AsyncSourcer::create(ctx, output_port, Self { - receiver, - data: None, - next_block_id, - }) + AsyncSourcer::create(ctx, output_port, Self { receiver }) } } @@ -109,37 +81,9 @@ impl AsyncSource for CTESource { #[async_backtrace::framed] async fn generate(&mut self) -> Result> { - if self.data.is_none() { - self.receiver.changed().await.map_err(|_| { - ErrorCode::Internal("Failed to get data from receiver in CTEConsumerSource") - })?; - self.data = Some(self.receiver.borrow().clone()); - } - - if let Some(data) = &self.data { - let id = self.next_block_id.fetch_add(1, Ordering::Relaxed); - if let Some(block) = data.get_data_block_at(id) { - return Ok(Some(block)); - } + if let Ok(data) = self.receiver.recv().await { + return Ok(Some(data)); } Ok(None) } } - -pub struct MaterializedCteChannel { - pub sender: Sender>, - pub receiver: Receiver>, -} - -impl MaterializedCteChannel { - pub fn new() -> Self { - let (sender, receiver) = watch::channel(Arc::new(MaterializedCteData::new(vec![]))); - Self { sender, receiver } - } -} - -impl Default for MaterializedCteChannel { - fn default() -> Self { - Self::new() - } -} diff --git a/src/query/service/src/pipelines/processors/transforms/mod.rs b/src/query/service/src/pipelines/processors/transforms/mod.rs index f8beebf5134ee..51d0de172d7be 100644 --- a/src/query/service/src/pipelines/processors/transforms/mod.rs +++ b/src/query/service/src/pipelines/processors/transforms/mod.rs @@ -47,8 +47,6 @@ pub use broadcast::BroadcastSinkProcessor; pub use broadcast::BroadcastSourceProcessor; pub use hash_join::*; pub use materialized_cte::CTESource; -pub use materialized_cte::MaterializedCteChannel; -pub use materialized_cte::MaterializedCteData; pub use materialized_cte::MaterializedCteSink; pub use transform_add_computed_columns::TransformAddComputedColumns; pub use transform_add_const_columns::TransformAddConstColumns; diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 03de1cfd0e461..eb3432e4bfbbc 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -68,6 +68,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoPtr; use databend_common_expression::BlockThresholds; +use databend_common_expression::DataBlock; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; @@ -139,7 +140,6 @@ use crate::clusters::Cluster; use crate::clusters::ClusterHelper; use crate::locks::LockManager; use crate::pipelines::executor::PipelineExecutor; -use crate::pipelines::processors::transforms::MaterializedCteData; use crate::servers::flight::v1::exchange::DataExchangeManager; use crate::sessions::query_affect::QueryAffect; use crate::sessions::query_ctx_shared::MemoryUpdater; @@ -586,30 +586,29 @@ impl QueryContext { self.shared.table_meta_timestamps.lock().clear(); } - pub fn get_materialized_cte_sender( + pub fn get_materialized_cte_senders( &self, cte_name: &str, - ) -> tokio::sync::watch::Sender> { + cte_ref_count: usize, + ) -> Vec> { + let mut senders = vec![]; + let mut receivers = vec![]; + for _ in 0..cte_ref_count { + let (sender, receiver) = async_channel::unbounded(); + senders.push(sender); + receivers.push(receiver); + } self.shared - .materialized_cte_channels + .materialized_cte_receivers .lock() - .entry(cte_name.to_string()) - .or_default() - .sender - .clone() + .insert(cte_name.to_string(), receivers); + senders } - pub fn get_materialized_cte_receiver( - &self, - cte_name: &str, - ) -> tokio::sync::watch::Receiver> { - self.shared - .materialized_cte_channels - .lock() - .entry(cte_name.to_string()) - .or_default() - .receiver - .clone() + pub fn get_materialized_cte_receiver(&self, cte_name: &str) -> Receiver { + let mut receivers = self.shared.materialized_cte_receivers.lock(); + let receivers = receivers.get_mut(cte_name).unwrap(); + receivers.pop().unwrap() } } diff --git a/src/query/service/src/sessions/query_ctx_shared.rs b/src/query/service/src/sessions/query_ctx_shared.rs index cd1f7b6215df7..215745d322620 100644 --- a/src/query/service/src/sessions/query_ctx_shared.rs +++ b/src/query/service/src/sessions/query_ctx_shared.rs @@ -47,6 +47,7 @@ use databend_common_config::GlobalConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::BlockMetaInfoPtr; +use databend_common_expression::DataBlock; use databend_common_meta_app::principal::OnErrorMode; use databend_common_meta_app::principal::RoleInfo; use databend_common_meta_app::principal::UserDefinedConnection; @@ -72,7 +73,6 @@ use uuid::Uuid; use crate::clusters::Cluster; use crate::clusters::ClusterDiscovery; use crate::pipelines::executor::PipelineExecutor; -use crate::pipelines::processors::transforms::MaterializedCteChannel; use crate::sessions::query_affect::QueryAffect; use crate::sessions::Session; use crate::storages::Table; @@ -184,8 +184,8 @@ pub struct QueryContextShared { pub(in crate::sessions) perf_flag: AtomicBool, pub(in crate::sessions) nodes_perf: Arc>>, - pub(in crate::sessions) materialized_cte_channels: - Arc>>, + pub(in crate::sessions) materialized_cte_receivers: + Arc>>>>, } #[derive(Default)] @@ -263,7 +263,7 @@ impl QueryContextShared { broadcast_channels: Arc::new(Mutex::new(HashMap::new())), perf_flag: AtomicBool::new(false), nodes_perf: Arc::new(Mutex::new(HashMap::new())), - materialized_cte_channels: Arc::new(Mutex::new(HashMap::new())), + materialized_cte_receivers: Arc::new(Mutex::new(HashMap::new())), })) } From 9c146f80d6012893597fb7b31cac39a4913e47bb Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 22 Jul 2025 16:39:49 +0800 Subject: [PATCH 50/80] refactor plan --- .../builders/builder_materialized_cte.rs | 25 ++---- .../pipelines/builders/builder_sequence.rs | 38 +++++++++ .../service/src/pipelines/builders/mod.rs | 1 + .../service/src/pipelines/pipeline_builder.rs | 1 + .../transform_recursive_cte_source.rs | 3 + .../src/schedulers/fragments/fragmenter.rs | 9 +- src/query/sql/src/executor/format.rs | 19 +++-- src/query/sql/src/executor/physical_plan.rs | 24 ++++-- .../sql/src/executor/physical_plan_builder.rs | 6 +- .../sql/src/executor/physical_plan_visitor.rs | 22 ++++- .../sql/src/executor/physical_plans/mod.rs | 2 + .../physical_materialized_cte.rs | 14 +--- .../physical_plans/physical_sequence.rs | 58 +++++++++++++ .../binder/bind_table_reference/bind_cte.rs | 8 +- .../sql/src/planner/optimizer/ir/format.rs | 1 + .../optimizer/optimizers/hyper_dp/dphyp.rs | 10 +-- .../hyper_dp/dynamic_sample/dynamic_sample.rs | 1 + .../decorrelate/subquery_decorrelator.rs | 3 +- .../join_rules/rule_semi_to_inner_join.rs | 3 +- .../sql/src/planner/plans/materialized_cte.rs | 32 +++++++- src/query/sql/src/planner/plans/mod.rs | 2 + src/query/sql/src/planner/plans/operator.rs | 6 +- .../sql/src/planner/plans/operator_macros.rs | 1 + src/query/sql/src/planner/plans/sequence.rs | 82 +++++++++++++++++++ 24 files changed, 305 insertions(+), 66 deletions(-) create mode 100644 src/query/service/src/pipelines/builders/builder_sequence.rs create mode 100644 src/query/sql/src/executor/physical_plans/physical_sequence.rs create mode 100644 src/query/sql/src/planner/plans/sequence.rs diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index b4ca67cf50dce..387516c7c224c 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -17,38 +17,23 @@ use databend_common_sql::executor::physical_plans::MaterializedCTE; use crate::pipelines::processors::transforms::MaterializedCteSink; use crate::pipelines::PipelineBuilder; -use crate::sessions::QueryContext; impl PipelineBuilder { pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { - // init builder for cte pipeline - let sub_context = QueryContext::create_from(self.ctx.as_ref()); - let sub_builder = - PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); - - // build cte pipeline - let mut build_res = sub_builder.finalize(&cte.left)?; - let input_schema = cte.left.output_schema()?; + self.build_pipeline(&cte.input)?; + let input_schema = cte.input.output_schema()?; Self::build_result_projection( &self.func_ctx, input_schema, &cte.cte_output_columns, - &mut build_res.main_pipeline, + &mut self.main_pipeline, false, )?; - build_res.main_pipeline.try_resize(1)?; + self.main_pipeline.try_resize(1)?; let tx = self .ctx .get_materialized_cte_senders(&cte.cte_name, cte.ref_count); - build_res - .main_pipeline + self.main_pipeline .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; - - // add cte pipeline to pipelines - self.pipelines.push(build_res.main_pipeline); - self.pipelines.extend(build_res.sources_pipelines); - - // build main pipeline - self.build_pipeline(&cte.right)?; Ok(()) } } diff --git a/src/query/service/src/pipelines/builders/builder_sequence.rs b/src/query/service/src/pipelines/builders/builder_sequence.rs new file mode 100644 index 0000000000000..7f0c3f1c3bc9d --- /dev/null +++ b/src/query/service/src/pipelines/builders/builder_sequence.rs @@ -0,0 +1,38 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::Result; +use databend_common_sql::executor::physical_plans::Sequence; + +use crate::pipelines::PipelineBuilder; +use crate::sessions::QueryContext; +impl PipelineBuilder { + pub(crate) fn build_sequence(&mut self, sequence: &Sequence) -> Result<()> { + // init builder for cte pipeline + let sub_context = QueryContext::create_from(self.ctx.as_ref()); + let sub_builder = + PipelineBuilder::create(self.func_ctx.clone(), self.settings.clone(), sub_context); + + // build cte pipeline + let build_res = sub_builder.finalize(&sequence.left)?; + + // add cte pipeline to pipelines + self.pipelines.push(build_res.main_pipeline); + self.pipelines.extend(build_res.sources_pipelines); + + // build main pipeline + self.build_pipeline(&sequence.right)?; + Ok(()) + } +} diff --git a/src/query/service/src/pipelines/builders/mod.rs b/src/query/service/src/pipelines/builders/mod.rs index ba86e21fd6dc0..6fd4bf024bc4c 100644 --- a/src/query/service/src/pipelines/builders/mod.rs +++ b/src/query/service/src/pipelines/builders/mod.rs @@ -45,6 +45,7 @@ mod builder_replace_into; mod builder_row_fetch; mod builder_scalar; mod builder_scan; +mod builder_sequence; mod builder_sort; mod builder_udf; mod builder_union_all; diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index f6150f5466fe7..64dba326e941f 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -298,6 +298,7 @@ impl PipelineBuilder { PhysicalPlan::MaterializedCTE(cte) => self.build_materialized_cte(cte), PhysicalPlan::CTEConsumer(cte_consumer) => self.build_cte_consumer(cte_consumer), + PhysicalPlan::Sequence(sequence) => self.build_sequence(sequence), }?; self.is_exchange_stack.pop(); diff --git a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs index 8f61f51e61778..4e515788c27da 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs @@ -330,6 +330,9 @@ async fn create_memory_table_for_cte_scan( create_memory_table_for_cte_scan(ctx, plan.input.as_ref()).await?; } PhysicalPlan::MaterializedCTE(plan) => { + create_memory_table_for_cte_scan(ctx, plan.input.as_ref()).await?; + } + PhysicalPlan::Sequence(plan) => { create_memory_table_for_cte_scan(ctx, plan.left.as_ref()).await?; create_memory_table_for_cte_scan(ctx, plan.right.as_ref()).await?; } diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 65f93b0316745..147c1d49b97b7 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -27,10 +27,10 @@ use databend_common_sql::executor::physical_plans::ExchangeSink; use databend_common_sql::executor::physical_plans::ExchangeSource; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::HashJoin; -use databend_common_sql::executor::physical_plans::MaterializedCTE; use databend_common_sql::executor::physical_plans::MutationSource; use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceInto; +use databend_common_sql::executor::physical_plans::Sequence; use databend_common_sql::executor::physical_plans::TableScan; use databend_common_sql::executor::physical_plans::UnionAll; use databend_common_sql::executor::PhysicalPlanReplacer; @@ -348,7 +348,7 @@ impl PhysicalPlanReplacer for Fragmenter { })) } - fn replace_materialized_cte(&mut self, plan: &MaterializedCTE) -> Result { + fn replace_sequence(&mut self, plan: &Sequence) -> Result { let mut fragments = vec![]; let left = self.replace(plan.left.as_ref())?; @@ -358,14 +358,11 @@ impl PhysicalPlanReplacer for Fragmenter { fragments.append(&mut self.fragments); self.fragments = fragments; - Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { + Ok(PhysicalPlan::Sequence(Box::new(Sequence { plan_id: plan.plan_id, left: Box::new(left), right: Box::new(right), stat_info: plan.stat_info.clone(), - cte_name: plan.cte_name.clone(), - cte_output_columns: plan.cte_output_columns.clone(), - ref_count: plan.ref_count, }))) } } diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 285eb2df267ec..7dc4db3f0160d 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -177,10 +177,9 @@ impl PhysicalPlan { )) } PhysicalPlan::MaterializedCTE(plan) => { - let left_child = plan.left.format_join(metadata)?; - let right_child = plan.right.format_join(metadata)?; + let input = plan.input.format_join(metadata)?; - let children = vec![left_child, right_child]; + let children = vec![input]; Ok(FormatTreeNode::with_children( format!("MaterializedCTE: {}", plan.cte_name), @@ -545,8 +544,7 @@ fn to_format_tree( PhysicalPlan::MaterializedCTE(plan) => { let mut children = Vec::new(); append_profile_info(&mut children, profs, plan.plan_id); - children.push(to_format_tree(&plan.left, metadata, profs, context)?); - children.push(to_format_tree(&plan.right, metadata, profs, context)?); + children.push(to_format_tree(&plan.input, metadata, profs, context)?); Ok(FormatTreeNode::with_children( format!("MaterializedCTE: {}", plan.cte_name), children, @@ -568,6 +566,17 @@ fn to_format_tree( children, )) } + PhysicalPlan::Sequence(plan) => { + let mut children = Vec::new(); + children.push(FormatTreeNode::new(format!("Sequence"))); + append_profile_info(&mut children, profs, plan.plan_id); + children.push(to_format_tree(&plan.left, metadata, profs, context)?); + children.push(to_format_tree(&plan.right, metadata, profs, context)?); + Ok(FormatTreeNode::with_children( + "Sequence".to_string(), + children, + )) + } } } diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 7899b6aca9633..37899282ab246 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -73,6 +73,7 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; +use crate::executor::physical_plans::Sequence; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; @@ -164,8 +165,10 @@ pub enum PhysicalPlan { BroadcastSource(BroadcastSource), BroadcastSink(BroadcastSink), + // CTE MaterializedCTE(Box), CTEConsumer(Box), + Sequence(Box), } impl PhysicalPlan { @@ -430,13 +433,18 @@ impl PhysicalPlan { PhysicalPlan::MaterializedCTE(plan) => { plan.plan_id = *next_id; *next_id += 1; - plan.left.adjust_plan_id(next_id); - plan.right.adjust_plan_id(next_id); + plan.input.adjust_plan_id(next_id); } PhysicalPlan::CTEConsumer(plan) => { plan.plan_id = *next_id; *next_id += 1; } + PhysicalPlan::Sequence(plan) => { + plan.plan_id = *next_id; + *next_id += 1; + plan.left.adjust_plan_id(next_id); + plan.right.adjust_plan_id(next_id); + } } } @@ -497,6 +505,7 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSink(v) => v.plan_id, PhysicalPlan::MaterializedCTE(v) => v.plan_id, PhysicalPlan::CTEConsumer(v) => v.plan_id, + PhysicalPlan::Sequence(v) => v.plan_id, } } @@ -556,6 +565,7 @@ impl PhysicalPlan { PhysicalPlan::ChunkCommitInsert(_) => todo!(), PhysicalPlan::MaterializedCTE(plan) => plan.output_schema(), PhysicalPlan::CTEConsumer(plan) => plan.output_schema(), + PhysicalPlan::Sequence(plan) => plan.output_schema(), } } @@ -621,6 +631,7 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSink(_) => "RuntimeFilterSink".to_string(), PhysicalPlan::MaterializedCTE(_) => "MaterializedCTE".to_string(), PhysicalPlan::CTEConsumer(_) => "CTEConsumer".to_string(), + PhysicalPlan::Sequence(_) => "Sequence".to_string(), } } @@ -696,7 +707,8 @@ impl PhysicalPlan { CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v.as_ref())), CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v.as_ref())), }, - PhysicalPlan::MaterializedCTE(plan) => Box::new( + PhysicalPlan::MaterializedCTE(plan) => Box::new(std::iter::once(plan.input.as_ref())), + PhysicalPlan::Sequence(plan) => Box::new( std::iter::once(plan.left.as_ref()).chain(std::iter::once(plan.right.as_ref())), ), } @@ -773,7 +785,8 @@ impl PhysicalPlan { CopyIntoTableSource::Query(v) => Box::new(std::iter::once(v.as_mut())), CopyIntoTableSource::Stage(v) => Box::new(std::iter::once(v.as_mut())), }, - PhysicalPlan::MaterializedCTE(plan) => Box::new( + PhysicalPlan::MaterializedCTE(plan) => Box::new(std::iter::once(plan.input.as_mut())), + PhysicalPlan::Sequence(plan) => Box::new( std::iter::once(plan.left.as_mut()).chain(std::iter::once(plan.right.as_mut())), ), PhysicalPlan::BroadcastSink(plan) => Box::new(std::iter::once(plan.input.as_mut())), @@ -836,7 +849,8 @@ impl PhysicalPlan { | PhysicalPlan::BroadcastSource(_) | PhysicalPlan::BroadcastSink(_) | PhysicalPlan::MaterializedCTE(_) - | PhysicalPlan::CTEConsumer(_) => None, + | PhysicalPlan::CTEConsumer(_) + | PhysicalPlan::Sequence(_) => None, } } diff --git a/src/query/sql/src/executor/physical_plan_builder.rs b/src/query/sql/src/executor/physical_plan_builder.rs index aa948994a22b5..c97ff00113cc1 100644 --- a/src/query/sql/src/executor/physical_plan_builder.rs +++ b/src/query/sql/src/executor/physical_plan_builder.rs @@ -130,12 +130,16 @@ impl PhysicalPlanBuilder { } RelOperator::CompactBlock(compact) => self.build_compact_block(compact).await, RelOperator::MaterializedCTE(materialized_cte) => { - self.build_materialized_cte(s_expr, materialized_cte, stat_info, required) + self.build_materialized_cte(s_expr, materialized_cte, stat_info) .await } RelOperator::CTEConsumer(cte_consumer) => { self.build_cte_consumer(cte_consumer, stat_info).await } + RelOperator::Sequence(sequence) => { + self.build_sequence(s_expr, sequence, stat_info, required) + .await + } } } diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index fa54e29d080cc..a4de69cee8f36 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -63,6 +63,7 @@ use crate::executor::physical_plans::ReplaceAsyncSourcer; use crate::executor::physical_plans::ReplaceDeduplicate; use crate::executor::physical_plans::ReplaceInto; use crate::executor::physical_plans::RowFetch; +use crate::executor::physical_plans::Sequence; use crate::executor::physical_plans::Shuffle; use crate::executor::physical_plans::Sort; use crate::executor::physical_plans::TableScan; @@ -128,6 +129,7 @@ pub trait PhysicalPlanReplacer { PhysicalPlan::BroadcastSink(plan) => self.replace_runtime_filter_sink(plan), PhysicalPlan::MaterializedCTE(plan) => self.replace_materialized_cte(plan), PhysicalPlan::CTEConsumer(plan) => self.replace_cte_consumer(plan), + PhysicalPlan::Sequence(plan) => self.replace_sequence(plan), } } @@ -649,12 +651,10 @@ pub trait PhysicalPlanReplacer { } fn replace_materialized_cte(&mut self, plan: &MaterializedCTE) -> Result { - let left = self.replace(&plan.left)?; - let right = self.replace(&plan.right)?; + let input = self.replace(&plan.input)?; Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { plan_id: plan.plan_id, - left: Box::new(left), - right: Box::new(right), + input: Box::new(input), stat_info: plan.stat_info.clone(), cte_name: plan.cte_name.clone(), cte_output_columns: plan.cte_output_columns.clone(), @@ -665,6 +665,17 @@ pub trait PhysicalPlanReplacer { fn replace_cte_consumer(&mut self, plan: &CTEConsumer) -> Result { Ok(PhysicalPlan::CTEConsumer(Box::new(plan.clone()))) } + + fn replace_sequence(&mut self, plan: &Sequence) -> Result { + let left = self.replace(&plan.left)?; + let right = self.replace(&plan.right)?; + Ok(PhysicalPlan::Sequence(Box::new(Sequence { + plan_id: plan.plan_id, + stat_info: plan.stat_info.clone(), + left: Box::new(left), + right: Box::new(right), + }))) + } } impl PhysicalPlan { @@ -819,6 +830,9 @@ impl PhysicalPlan { Self::traverse(&plan.input, pre_visit, visit, post_visit); } PhysicalPlan::MaterializedCTE(plan) => { + Self::traverse(&plan.input, pre_visit, visit, post_visit); + } + PhysicalPlan::Sequence(plan) => { Self::traverse(&plan.left, pre_visit, visit, post_visit); Self::traverse(&plan.right, pre_visit, visit, post_visit); } diff --git a/src/query/sql/src/executor/physical_plans/mod.rs b/src/query/sql/src/executor/physical_plans/mod.rs index db1051333ba03..38fe1d1b617b3 100644 --- a/src/query/sql/src/executor/physical_plans/mod.rs +++ b/src/query/sql/src/executor/physical_plans/mod.rs @@ -54,6 +54,7 @@ mod physical_replace_async_source; mod physical_replace_deduplicate; mod physical_replace_into; mod physical_row_fetch; +mod physical_sequence; mod physical_sort; mod physical_table_scan; mod physical_udf; @@ -118,3 +119,4 @@ pub use physical_union_all::UnionAll; pub use physical_window::*; pub use physical_window_partition::*; mod physical_asof_join; +pub use physical_sequence::Sequence; diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index 8b787e4179f13..7833d3a022ff2 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -20,17 +20,14 @@ use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; use crate::optimizer::ir::SExpr; use crate::ColumnBinding; -use crate::ColumnSet; -/// This is a binary operator that executes its children in order (left to right), and returns the results of the right child #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct MaterializedCTE { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, // Only used for explain pub stat_info: Option, - pub left: Box, - pub right: Box, + pub input: Box, pub cte_name: String, pub cte_output_columns: Vec, pub ref_count: usize, @@ -38,7 +35,7 @@ pub struct MaterializedCTE { impl MaterializedCTE { pub fn output_schema(&self) -> Result { - self.right.output_schema() + self.input.output_schema() } } @@ -48,9 +45,8 @@ impl PhysicalPlanBuilder { s_expr: &SExpr, materialized_cte: &crate::plans::MaterializedCTE, stat_info: PlanStatsInfo, - required: ColumnSet, ) -> Result { - let left_side = Box::new( + let input = Box::new( self.build( s_expr.child(0)?, materialized_cte @@ -61,12 +57,10 @@ impl PhysicalPlanBuilder { ) .await?, ); - let right_side = Box::new(self.build(s_expr.child(1)?, required).await?); Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { plan_id: 0, stat_info: Some(stat_info), - left: left_side, - right: right_side, + input, cte_name: materialized_cte.cte_name.clone(), cte_output_columns: materialized_cte.cte_output_columns.clone(), ref_count: materialized_cte.ref_count, diff --git a/src/query/sql/src/executor/physical_plans/physical_sequence.rs b/src/query/sql/src/executor/physical_plans/physical_sequence.rs new file mode 100644 index 0000000000000..375eda4e36d07 --- /dev/null +++ b/src/query/sql/src/executor/physical_plans/physical_sequence.rs @@ -0,0 +1,58 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_exception::Result; +use databend_common_expression::DataSchemaRef; + +use crate::executor::explain::PlanStatsInfo; +use crate::executor::PhysicalPlan; +use crate::executor::PhysicalPlanBuilder; +use crate::optimizer::ir::SExpr; +use crate::ColumnSet; + +/// This is a binary operator that executes its children in order (left to right), and returns the results of the right child +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct Sequence { + // A unique id of operator in a `PhysicalPlan` tree, only used for display. + pub plan_id: u32, + // Only used for explain + pub stat_info: Option, + pub left: Box, + pub right: Box, +} + +impl Sequence { + pub fn output_schema(&self) -> Result { + self.right.output_schema() + } +} + +impl PhysicalPlanBuilder { + pub(crate) async fn build_sequence( + &mut self, + s_expr: &SExpr, + _sequence: &crate::plans::Sequence, + stat_info: PlanStatsInfo, + required: ColumnSet, + ) -> Result { + let left_side = Box::new(self.build(s_expr.child(0)?, Default::default()).await?); + let right_side = Box::new(self.build(s_expr.child(1)?, required).await?); + Ok(PhysicalPlan::Sequence(Box::new(Sequence { + plan_id: 0, + stat_info: Some(stat_info), + left: left_side, + right: right_side, + }))) + } +} diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index f1c243bdbac22..17dee5dfcbd78 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -33,6 +33,7 @@ use crate::optimizer::ir::SExpr; use crate::plans::CTEConsumer; use crate::plans::MaterializedCTE; use crate::plans::RelOperator; +use crate::plans::Sequence; impl Binder { pub fn init_cte(&mut self, bind_context: &mut BindContext, with: &Option) -> Result<()> { @@ -183,9 +184,12 @@ impl Binder { let materialized_cte = MaterializedCTE::new(cte_name, bind_context.columns, *ref_count); + let materialized_cte = + SExpr::create_unary(Arc::new(materialized_cte.into()), Arc::new(s_expr)); + let sequence = Sequence {}; current_expr = SExpr::create_binary( - Arc::new(materialized_cte.into()), - Arc::new(s_expr), + Arc::new(sequence.into()), + materialized_cte, Arc::new(current_expr), ); } diff --git a/src/query/sql/src/planner/optimizer/ir/format.rs b/src/query/sql/src/planner/optimizer/ir/format.rs index 5d6909f2cf374..7e7ec8b63bec4 100644 --- a/src/query/sql/src/planner/optimizer/ir/format.rs +++ b/src/query/sql/src/planner/optimizer/ir/format.rs @@ -82,6 +82,7 @@ fn display_rel_op(rel_op: &RelOperator) -> String { RelOperator::CompactBlock(_) => "CompactBlock".to_string(), RelOperator::MaterializedCTE(_) => "MaterializedCTE".to_string(), RelOperator::CTEConsumer(_) => "CTEConsumer".to_string(), + RelOperator::Sequence(_) => "Sequence".to_string(), } } diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index bb17f389104a0..a31376ae5b045 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -270,10 +270,7 @@ impl DPhpyOptimizer { Ok((new_s_expr, left_res.1 && right_res.1)) } - async fn process_materialized_cte_node( - &mut self, - s_expr: &SExpr, - ) -> Result<(Arc, bool)> { + async fn process_sequence_node(&mut self, s_expr: &SExpr) -> Result<(Arc, bool)> { let mut left_dphyp = DPhpyOptimizer::new(self.opt_ctx.clone()); let left_expr = left_dphyp.optimize_async(s_expr.child(0)?).await?; @@ -406,7 +403,7 @@ impl DPhpyOptimizer { RelOperator::Join(_) => self.process_join_node(s_expr, join_conditions).await, - RelOperator::MaterializedCTE(_) => self.process_materialized_cte_node(s_expr).await, + RelOperator::Sequence(_) => self.process_sequence_node(s_expr).await, RelOperator::CTEConsumer(_) => { self.process_cte_consumer_node(s_expr, join_relation).await } @@ -418,7 +415,8 @@ impl DPhpyOptimizer { | RelOperator::EvalScalar(_) | RelOperator::Window(_) | RelOperator::Udf(_) - | RelOperator::Filter(_) => { + | RelOperator::Filter(_) + | RelOperator::MaterializedCTE(_) => { self.process_unary_node(s_expr, join_conditions, join_child, join_relation) .await } diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs index cdc9071d13c84..3f1d2ace543f7 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs @@ -94,6 +94,7 @@ pub async fn dynamic_sample( | RelOperator::CompactBlock(_) | RelOperator::MaterializedCTE(_) | RelOperator::CTEConsumer(_) + | RelOperator::Sequence(_) | RelOperator::MutationSource(_) => { s_expr.plan().derive_stats(&RelExpr::with_s_expr(s_expr)) } diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs index a3f091cda6831..c191e043fb6ae 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs @@ -341,7 +341,8 @@ impl SubqueryDecorrelatorOptimizer { | RelOperator::Mutation(_) | RelOperator::MutationSource(_) | RelOperator::CTEConsumer(_) - | RelOperator::CompactBlock(_) => Ok(s_expr.clone()), + | RelOperator::CompactBlock(_) + | RelOperator::Sequence(_) => Ok(s_expr.clone()), } } diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs index 1cfa72a2ed63c..7ec7e8742ec94 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs @@ -154,7 +154,8 @@ fn find_group_by_keys( | RelOperator::MutationSource(_) | RelOperator::MaterializedCTE(_) | RelOperator::CTEConsumer(_) - | RelOperator::CompactBlock(_) => {} + | RelOperator::CompactBlock(_) + | RelOperator::Sequence(_) => {} } Ok(()) } diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index 96ca542a7000a..d2bc704fa93cf 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -15,15 +15,18 @@ use std::hash::Hash; use std::sync::Arc; +use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use crate::optimizer::ir::PhysicalProperty; use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RelationalProperty; +use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::StatInfo; use crate::plans::Operator; use crate::plans::RelOp; use crate::ColumnBinding; + #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, @@ -48,21 +51,42 @@ impl Operator for MaterializedCTE { /// Get arity of this operator fn arity(&self) -> usize { - 2 + 1 } /// Derive relational property fn derive_relational_prop(&self, rel_expr: &RelExpr) -> Result> { - rel_expr.derive_relational_prop_child(1) + rel_expr.derive_relational_prop_child(0) } /// Derive physical property fn derive_physical_prop(&self, rel_expr: &RelExpr) -> Result { - rel_expr.derive_physical_prop_child(1) + rel_expr.derive_physical_prop_child(0) } /// Derive statistics information fn derive_stats(&self, rel_expr: &RelExpr) -> Result> { - rel_expr.derive_cardinality_child(1) + rel_expr.derive_cardinality_child(0) + } + + /// Compute required property for child with index `child_index` + fn compute_required_prop_child( + &self, + _ctx: Arc, + _rel_expr: &RelExpr, + _child_index: usize, + required: &RequiredProperty, + ) -> Result { + Ok(required.clone()) + } + + /// Enumerate all possible combinations of required property for children + fn compute_required_prop_children( + &self, + _ctx: Arc, + _rel_expr: &RelExpr, + _required: &RequiredProperty, + ) -> Result>> { + Ok(vec![vec![RequiredProperty::default(); self.arity()]]) } } diff --git a/src/query/sql/src/planner/plans/mod.rs b/src/query/sql/src/planner/plans/mod.rs index 487b041b82db7..a139f1243cf4a 100644 --- a/src/query/sql/src/planner/plans/mod.rs +++ b/src/query/sql/src/planner/plans/mod.rs @@ -47,6 +47,7 @@ mod replace; mod revert_table; mod scalar_expr; mod scan; +mod sequence; mod set; mod set_priority; mod sort; @@ -96,6 +97,7 @@ pub use replace::Replace; pub use revert_table::RevertTablePlan; pub use scalar_expr::*; pub use scan::*; +pub use sequence::*; pub use set::*; pub use set_priority::SetPriorityPlan; pub use sort::*; diff --git a/src/query/sql/src/planner/plans/operator.rs b/src/query/sql/src/planner/plans/operator.rs index 1d004f998974f..189bdedd18336 100644 --- a/src/query/sql/src/planner/plans/operator.rs +++ b/src/query/sql/src/planner/plans/operator.rs @@ -31,6 +31,7 @@ use crate::optimizer::ir::RelationalProperty; use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::StatInfo; use crate::plans::r_cte_scan::RecursiveCteScan; +use crate::plans::sequence::Sequence; use crate::plans::Aggregate; use crate::plans::AsyncFunction; use crate::plans::CTEConsumer; @@ -131,6 +132,7 @@ pub enum RelOp { MutationSource, MaterializedCTE, CTEConsumer, + Sequence, } /// Relational operators @@ -166,6 +168,7 @@ pub enum RelOperator { MutationSource(MutationSource), MaterializedCTE(MaterializedCTE), CTEConsumer(CTEConsumer), + Sequence(Sequence), } impl RelOperator { @@ -257,5 +260,6 @@ impl_try_from_rel_operator! { CompactBlock, MutationSource, MaterializedCTE, - CTEConsumer + CTEConsumer, + Sequence } diff --git a/src/query/sql/src/planner/plans/operator_macros.rs b/src/query/sql/src/planner/plans/operator_macros.rs index 4695bc7f6e4ca..a81779d997c58 100644 --- a/src/query/sql/src/planner/plans/operator_macros.rs +++ b/src/query/sql/src/planner/plans/operator_macros.rs @@ -112,6 +112,7 @@ macro_rules! impl_match_rel_op { RelOperator::MutationSource($rel_op) => $rel_op.$method($($arg),*), RelOperator::MaterializedCTE($rel_op) => $rel_op.$method($($arg),*), RelOperator::CTEConsumer($rel_op) => $rel_op.$method($($arg),*), + RelOperator::Sequence($rel_op) => $rel_op.$method($($arg),*), } } } diff --git a/src/query/sql/src/planner/plans/sequence.rs b/src/query/sql/src/planner/plans/sequence.rs new file mode 100644 index 0000000000000..b99c7b4d1338a --- /dev/null +++ b/src/query/sql/src/planner/plans/sequence.rs @@ -0,0 +1,82 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_catalog::table_context::TableContext; +use databend_common_exception::Result; + +use crate::optimizer::ir::PhysicalProperty; +use crate::optimizer::ir::RelExpr; +use crate::optimizer::ir::RelationalProperty; +use crate::optimizer::ir::RequiredProperty; +use crate::optimizer::ir::StatInfo; +use crate::plans::Operator; +use crate::plans::RelOp; +use crate::ScalarExpr; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Sequence; + +impl Operator for Sequence { + /// Get relational operator kind + fn rel_op(&self) -> RelOp { + RelOp::Sequence + } + + /// Get arity of this operator + fn arity(&self) -> usize { + 2 + } + + fn scalar_expr_iter(&self) -> Box + '_> { + Box::new(std::iter::empty()) + } + + /// Derive relational property + fn derive_relational_prop(&self, rel_expr: &RelExpr) -> Result> { + rel_expr.derive_relational_prop_child(1) + } + + /// Derive physical property + fn derive_physical_prop(&self, rel_expr: &RelExpr) -> Result { + rel_expr.derive_physical_prop_child(1) + } + + /// Derive statistics information + fn derive_stats(&self, rel_expr: &RelExpr) -> Result> { + rel_expr.derive_cardinality_child(1) + } + + /// Compute required property for child with index `child_index` + fn compute_required_prop_child( + &self, + _ctx: Arc, + _rel_expr: &RelExpr, + _child_index: usize, + required: &RequiredProperty, + ) -> Result { + Ok(required.clone()) + } + + /// Enumerate all possible combinations of required property for children + fn compute_required_prop_children( + &self, + _ctx: Arc, + _rel_expr: &RelExpr, + _required: &RequiredProperty, + ) -> Result>> { + Ok(vec![vec![RequiredProperty::default(); self.arity()]]) + } +} From 993659a08f0ad6d048f2466a75c337a981a36c14 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 22 Jul 2025 18:43:59 +0800 Subject: [PATCH 51/80] fix --- .../operator/decorrelate/subquery_decorrelator.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs index c191e043fb6ae..261dd5cc17312 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs @@ -312,25 +312,19 @@ impl SubqueryDecorrelatorOptimizer { )); } - RelOperator::UnionAll(_) => Ok(SExpr::create_binary( + RelOperator::UnionAll(_) | RelOperator::Sequence(_) => Ok(SExpr::create_binary( s_expr.plan.clone(), Arc::new(self.optimize_sync(s_expr.left_child())?), Arc::new(self.optimize_sync(s_expr.right_child())?), )), - RelOperator::Limit(_) | RelOperator::Udf(_) | RelOperator::AsyncFunction(_) => { + RelOperator::Limit(_) | RelOperator::Udf(_) | RelOperator::AsyncFunction(_) | RelOperator::MaterializedCTE(_) => { Ok(SExpr::create_unary( s_expr.plan.clone(), Arc::new(self.optimize_sync(s_expr.unary_child())?), )) } - RelOperator::MaterializedCTE(_) => Ok(SExpr::create_binary( - s_expr.plan.clone(), - Arc::new(self.optimize_sync(s_expr.left_child())?), - Arc::new(self.optimize_sync(s_expr.right_child())?), - )), - RelOperator::DummyTableScan(_) | RelOperator::Scan(_) | RelOperator::ConstantTableScan(_) @@ -341,8 +335,7 @@ impl SubqueryDecorrelatorOptimizer { | RelOperator::Mutation(_) | RelOperator::MutationSource(_) | RelOperator::CTEConsumer(_) - | RelOperator::CompactBlock(_) - | RelOperator::Sequence(_) => Ok(s_expr.clone()), + | RelOperator::CompactBlock(_) => Ok(s_expr.clone()), } } From db1c089630276a572a8f82ed20b4c9e41c988a73 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Tue, 22 Jul 2025 19:00:58 +0800 Subject: [PATCH 52/80] fix --- .../optimizers/operator/cte/cleanup_unused_cte.rs | 7 +++++-- .../operator/decorrelate/subquery_decorrelator.rs | 13 +++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs index b29d06afe460d..cab43b979205e 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs @@ -19,6 +19,7 @@ use databend_common_exception::Result; use crate::optimizer::ir::SExpr; use crate::optimizer::Optimizer; +use crate::plans::MaterializedCTE; use crate::plans::RelOperator; /// Optimizer that removes unused CTEs from the query plan. @@ -54,9 +55,11 @@ impl CleanupUnusedCTEOptimizer { /// Remove unused CTEs from the expression tree fn remove_unused_ctes(s_expr: &SExpr, referenced_ctes: &HashSet) -> Result { - if let RelOperator::MaterializedCTE(m_cte) = s_expr.plan() { + if let RelOperator::Sequence(_) = s_expr.plan() { + let left_child = s_expr.child(0)?.plan().clone(); + let cte: MaterializedCTE = left_child.try_into()?; // If this CTE is not referenced, remove it by returning the right child - if !referenced_ctes.contains(&m_cte.cte_name) { + if !referenced_ctes.contains(&cte.cte_name) { // Return the right child (main query) and skip the left child (CTE definition) let right_child = s_expr.child(1)?; return Self::remove_unused_ctes(right_child, referenced_ctes); diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs index 261dd5cc17312..bbcd120ee9fd7 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs @@ -318,12 +318,13 @@ impl SubqueryDecorrelatorOptimizer { Arc::new(self.optimize_sync(s_expr.right_child())?), )), - RelOperator::Limit(_) | RelOperator::Udf(_) | RelOperator::AsyncFunction(_) | RelOperator::MaterializedCTE(_) => { - Ok(SExpr::create_unary( - s_expr.plan.clone(), - Arc::new(self.optimize_sync(s_expr.unary_child())?), - )) - } + RelOperator::Limit(_) + | RelOperator::Udf(_) + | RelOperator::AsyncFunction(_) + | RelOperator::MaterializedCTE(_) => Ok(SExpr::create_unary( + s_expr.plan.clone(), + Arc::new(self.optimize_sync(s_expr.unary_child())?), + )), RelOperator::DummyTableScan(_) | RelOperator::Scan(_) From 082eccf0846f4da79d980d8238266f2c7e9cca4c Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 23 Jul 2025 10:08:13 +0800 Subject: [PATCH 53/80] enable distributed --- .../src/schedulers/fragments/fragmenter.rs | 48 +++++++++++++++---- .../src/planner/optimizer/pipeline/common.rs | 6 --- .../planner/optimizer/pipeline/pipeline.rs | 7 --- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 147c1d49b97b7..16ecb2a87793a 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -27,6 +27,7 @@ use databend_common_sql::executor::physical_plans::ExchangeSink; use databend_common_sql::executor::physical_plans::ExchangeSource; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::HashJoin; +use databend_common_sql::executor::physical_plans::MaterializedCTE; use databend_common_sql::executor::physical_plans::MutationSource; use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceInto; @@ -350,19 +351,50 @@ impl PhysicalPlanReplacer for Fragmenter { fn replace_sequence(&mut self, plan: &Sequence) -> Result { let mut fragments = vec![]; - let left = self.replace(plan.left.as_ref())?; + let _left = self.replace(plan.left.as_ref())?; fragments.append(&mut self.fragments); - self.state = State::Other; let right = self.replace(plan.right.as_ref())?; fragments.append(&mut self.fragments); self.fragments = fragments; - Ok(PhysicalPlan::Sequence(Box::new(Sequence { - plan_id: plan.plan_id, - left: Box::new(left), - right: Box::new(right), - stat_info: plan.stat_info.clone(), - }))) + Ok(right) + } + + fn replace_materialized_cte(&mut self, plan: &MaterializedCTE) -> Result { + let input = self.replace(plan.input.as_ref())?; + let plan = PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { + input: Box::new(input), + ..plan.clone() + })); + + let fragment_type = match self.state { + State::SelectLeaf => FragmentType::Source, + State::MutationSource => FragmentType::MutationSource, + State::Other => FragmentType::Intermediate, + State::ReplaceInto => FragmentType::ReplaceInto, + State::Compact => FragmentType::Compact, + State::Recluster => FragmentType::Recluster, + }; + self.state = State::Other; + + let fragment_id = self.ctx.get_fragment_id(); + + let mut fragment = PlanFragment { + plan: plan.clone(), + fragment_type, + + fragment_id, + exchange: None, + query_id: self.query_id.clone(), + + source_fragments: self.fragments.drain(..).collect(), + }; + + // Fill the destination_fragment_id for source fragments of `fragment`. + Self::resolve_fragment_connection(&mut fragment); + + self.fragments.push(fragment); + Ok(plan) } } diff --git a/src/query/sql/src/planner/optimizer/pipeline/common.rs b/src/query/sql/src/planner/optimizer/pipeline/common.rs index 40306a21c0ca0..26aa6e6556415 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/common.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/common.rs @@ -59,9 +59,3 @@ pub fn contains_warehouse_table_scan(s_expr: &SExpr, metadata: &MetadataRef) -> false } - -/// Check if a query contains MaterializedCTE operators. -pub fn contains_materialized_cte(s_expr: &SExpr) -> bool { - s_expr.children().any(contains_materialized_cte) - || matches!(s_expr.plan(), RelOperator::MaterializedCTE(_)) -} diff --git a/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs b/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs index fd3397e6b42de..f90580450f416 100644 --- a/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs +++ b/src/query/sql/src/planner/optimizer/pipeline/pipeline.rs @@ -19,7 +19,6 @@ use databend_common_exception::Result; use log::info; use super::common::contains_local_table_scan; -use super::common::contains_materialized_cte; use super::common::contains_warehouse_table_scan; use crate::optimizer::ir::Memo; use crate::optimizer::ir::SExpr; @@ -99,12 +98,6 @@ impl OptimizerPipeline { } } - // Check if the plan contains MaterializedCTE, if so, disable distributed optimization - if contains_materialized_cte(s_expr) { - self.opt_ctx.set_enable_distributed_optimization(false); - info!("Disable distributed optimization due to MaterializedCTE."); - } - Ok(()) } From 2ad7a25c615d6eaaf4f486d231ebecca65de1188 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 23 Jul 2025 10:13:24 +0800 Subject: [PATCH 54/80] fix logic test --- .../suites/query/materialized_cte.test | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/sqllogictests/suites/query/materialized_cte.test b/tests/sqllogictests/suites/query/materialized_cte.test index ac33cfe71071e..f516c3835349f 100644 --- a/tests/sqllogictests/suites/query/materialized_cte.test +++ b/tests/sqllogictests/suites/query/materialized_cte.test @@ -1,6 +1,6 @@ # two materialized ctes and select one column query I -with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select number as b from numbers(20)) select t1.a from t1 join t2 on t1.a = t2.b; +with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select number as b from numbers(20)) select t1.a from t1 join t2 on t1.a = t2.b order by t1.a; ---- 0 1 @@ -15,7 +15,7 @@ with t1 as materialized (select number as a from numbers(10)), t2 as materialize # two materialized ctes and select all columns query I -with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select number as b from numbers(20)) select * from t1 join t2 on t1.a = t2.b; +with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select number as b from numbers(20)) select * from t1 join t2 on t1.a = t2.b order by t1.a; ---- 0 0 1 1 @@ -30,7 +30,7 @@ with t1 as materialized (select number as a from numbers(10)), t2 as materialize # only one materialized query I -with t1 as materialized (select number as a from numbers(10)) select t1.a from t1; +with t1 as materialized (select number as a from numbers(10)) select t1.a from t1 order by t1.a; ---- 0 1 @@ -45,7 +45,7 @@ with t1 as materialized (select number as a from numbers(10)) select t1.a from t # one materialized and one not query I -with t1 as materialized (select number as a from numbers(10)), t2 as (select number as b from numbers(20)) select t1.a from t1 join t2 on t1.a = t2.b; +with t1 as materialized (select number as a from numbers(10)), t2 as (select number as b from numbers(20)) select t1.a from t1 join t2 on t1.a = t2.b order by t1.a; ---- 0 1 @@ -60,7 +60,7 @@ with t1 as materialized (select number as a from numbers(10)), t2 as (select num # one cte uses a materialized cte query I -with t1 as materialized (select number as a from numbers(10)), t2 as (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b; +with t1 as materialized (select number as a from numbers(10)), t2 as (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b order by t1.a; ---- 0 1 @@ -74,7 +74,7 @@ with t1 as materialized (select number as a from numbers(10)), t2 as (select a a 9 query I -with t1 as materialized (select number as a from numbers(10)) select t2.a from t1 as t2 where t2.a in (select * from t1 as t3 where t2.a = t3.a); +with t1 as materialized (select number as a from numbers(10)) select t2.a from t1 as t2 where t2.a in (select * from t1 as t3 where t2.a = t3.a) order by t2.a; ---- 0 1 @@ -135,7 +135,7 @@ with t1 as materialized (select number as a from numbers(10)), t2 as materialize # one materialized cte uses a materialized cte query I -with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b; +with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b order by t1.a; ---- 0 1 @@ -149,7 +149,7 @@ with t1 as materialized (select number as a from numbers(10)), t2 as materialize 9 query II -with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select number as b from numbers(20)), t3 as (select b as c from t2) select * from t1 join t3 on t1.a = t3.c; +with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select number as b from numbers(20)), t3 as (select b as c from t2) select * from t1 join t3 on t1.a = t3.c order by t1.a; ---- 0 0 1 1 @@ -163,7 +163,7 @@ with t1 as materialized (select number as a from numbers(10)), t2 as materialize 9 9 query I -with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select a as b from t1) select b from t2; +with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select a as b from t1) select b from t2 order by b; ---- 0 1 From b2d42af9ddaf3563c6475cf8c0b9a37f31db0829 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 23 Jul 2025 12:45:00 +0800 Subject: [PATCH 55/80] fix serial cte --- .../optimizer/optimizers/cascades/cascade.rs | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs b/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs index a79f76dba9497..5667a667ff565 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/cascades/cascade.rs @@ -22,6 +22,7 @@ use log::info; use crate::optimizer::cost::CostModel; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::Memo; +use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::RequiredProperty; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::cascades::cost::DefaultCostModel; @@ -36,6 +37,7 @@ use crate::optimizer::optimizers::rule::RuleSet; use crate::optimizer::optimizers::rule::TransformResult; use crate::optimizer::Optimizer; use crate::optimizer::OptimizerContext; +use crate::plans::RelOperator; use crate::IndexType; /// A cascades-style search engine to enumerate possible alternations of a relational expression and @@ -91,7 +93,7 @@ impl CascadesOptimizer { let result = self.optimize_internal(s_expr.clone()); // Process different cases based on the result - let optimized_expr = match result { + let mut optimized_expr = match result { Ok(expr) => { // After successful optimization, apply sort and limit push down if distributed optimization is enabled if opt_ctx.get_enable_distributed_optimization() { @@ -120,9 +122,54 @@ impl CascadesOptimizer { } }; + optimized_expr = Self::remove_exchanges_for_serial_sequence(optimized_expr)?; + Ok(optimized_expr) } + fn remove_exchanges_for_serial_sequence(s_expr: SExpr) -> Result { + if Self::has_sequence_with_serial_left_child(&s_expr)? { + Self::remove_all_exchanges(s_expr) + } else { + Ok(s_expr) + } + } + + fn has_sequence_with_serial_left_child(s_expr: &SExpr) -> Result { + if let RelOperator::Sequence(_) = s_expr.plan.as_ref() { + let left_child = s_expr.left_child(); + let rel_expr = RelExpr::with_s_expr(left_child); + let physical_prop = rel_expr.derive_physical_prop()?; + + if physical_prop.distribution == Distribution::Serial { + return Ok(true); + } + } + + for child in s_expr.children() { + if Self::has_sequence_with_serial_left_child(child)? { + return Ok(true); + } + } + + Ok(false) + } + + fn remove_all_exchanges(s_expr: SExpr) -> Result { + if let RelOperator::Exchange(_) = s_expr.plan.as_ref() { + return Self::remove_all_exchanges(s_expr.unary_child().clone()); + } + + let mut new_children = Vec::new(); + for child in s_expr.children() { + let processed_child = Self::remove_all_exchanges(child.clone())?; + new_children.push(Arc::new(processed_child)); + } + + let result = s_expr.replace_children(new_children); + Ok(result) + } + fn optimize_internal(&mut self, s_expr: SExpr) -> Result { // Update rule set based on current flags // This ensures we use the most up-to-date flag values, regardless of when the optimizer was created From 5e20f1cbb58b44061260f03753d3202a2d6227cf Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 23 Jul 2025 14:22:25 +0800 Subject: [PATCH 56/80] fix test --- .../suites/query/cleanup_unused_cte.test | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/sqllogictests/suites/query/cleanup_unused_cte.test b/tests/sqllogictests/suites/query/cleanup_unused_cte.test index 4c38d8a4e6845..a50fcc0c958f9 100644 --- a/tests/sqllogictests/suites/query/cleanup_unused_cte.test +++ b/tests/sqllogictests/suites/query/cleanup_unused_cte.test @@ -3,7 +3,7 @@ # Test case 1: CTE is used, should not be removed query I -with t1 as materialized (select number as a from numbers(10)) select t1.a from t1; +with t1 as materialized (select number as a from numbers(10)) select t1.a from t1 order by t1.a; ---- 0 1 @@ -18,7 +18,7 @@ with t1 as materialized (select number as a from numbers(10)) select t1.a from t # Test case 2: CTE is not used, should be removed query I -with t1 as materialized (select number as a from numbers(10)) select number as b from numbers(5); +with t1 as materialized (select number as a from numbers(10)) select number as b from numbers(5) order by b; ---- 0 1 @@ -31,7 +31,7 @@ query I with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select number as b from numbers(20)), t3 as materialized (select number as c from numbers(30)) -select t1.a from t1 join t2 on t1.a = t2.b; +select t1.a from t1 join t2 on t1.a = t2.b order by t1.a; ---- 0 1 @@ -49,7 +49,7 @@ query I with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select a as b from t1), t3 as materialized (select number as c from numbers(5)) -select t2.b from t2; +select t2.b from t2 order by t2.b; ---- 0 1 @@ -66,7 +66,7 @@ select t2.b from t2; query I with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select number as b from numbers(20)) -select number as c from numbers(3); +select number as c from numbers(3) order by c; ---- 0 1 @@ -79,7 +79,7 @@ with t1 as materialized ( from numbers(10) where number > 5 ) -select number as c from numbers(3); +select number as c from numbers(3) order by c; ---- 0 1 @@ -92,7 +92,7 @@ with t1 as materialized ( from numbers(10) group by number ) -select number as b from numbers(3); +select number as b from numbers(3) order by b; ---- 0 1 @@ -105,7 +105,7 @@ with t1 as materialized ( from numbers(5) n1 join numbers(5) n2 on n1.number = n2.number ) -select number as c from numbers(3); +select number as c from numbers(3) order by c; ---- 0 1 From c612c567999f443ba3c84117453dd471a956ab0f Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 23 Jul 2025 15:15:16 +0800 Subject: [PATCH 57/80] fix fragment type --- src/query/service/src/schedulers/fragments/fragmenter.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 16ecb2a87793a..2fe8fef502aa9 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -164,6 +164,7 @@ impl PhysicalPlanReplacer for Fragmenter { } fn replace_cte_consumer(&mut self, plan: &CTEConsumer) -> Result { + self.state = State::Other; Ok(PhysicalPlan::CTEConsumer(Box::new(plan.clone()))) } From 5fc45ba182494bf76b2f53be8f9b310d80631667 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 23 Jul 2025 16:07:03 +0800 Subject: [PATCH 58/80] fix replace range join --- .../src/schedulers/fragments/fragmenter.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index 2fe8fef502aa9..ac5dfda0c8645 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -29,6 +29,7 @@ use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::HashJoin; use databend_common_sql::executor::physical_plans::MaterializedCTE; use databend_common_sql::executor::physical_plans::MutationSource; +use databend_common_sql::executor::physical_plans::RangeJoin; use databend_common_sql::executor::physical_plans::Recluster; use databend_common_sql::executor::physical_plans::ReplaceInto; use databend_common_sql::executor::physical_plans::Sequence; @@ -258,6 +259,28 @@ impl PhysicalPlanReplacer for Fragmenter { })) } + fn replace_range_join(&mut self, plan: &RangeJoin) -> Result { + let mut fragments = vec![]; + let left = self.replace(&plan.left)?; + fragments.append(&mut self.fragments); + let right = self.replace(&plan.right)?; + fragments.append(&mut self.fragments); + + self.fragments = fragments; + + Ok(PhysicalPlan::RangeJoin(RangeJoin { + plan_id: plan.plan_id, + left: Box::new(left), + right: Box::new(right), + conditions: plan.conditions.clone(), + other_conditions: plan.other_conditions.clone(), + join_type: plan.join_type.clone(), + range_join_type: plan.range_join_type.clone(), + output_schema: plan.output_schema.clone(), + stat_info: plan.stat_info.clone(), + })) + } + fn replace_union(&mut self, plan: &UnionAll) -> Result { let mut fragments = vec![]; let left_input = self.replace(plan.left.as_ref())?; From 7b38411a65055a1de09c7486512d8ad74e851960 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 23 Jul 2025 16:57:53 +0800 Subject: [PATCH 59/80] fix explain join order --- src/query/sql/src/executor/format.rs | 36 ++++++++++++++----- .../sqllogictests/suites/tpch/join_order.test | 15 +++++--- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index 7dc4db3f0160d..cdcfc7b7a6e76 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -178,18 +178,38 @@ impl PhysicalPlan { } PhysicalPlan::MaterializedCTE(plan) => { let input = plan.input.format_join(metadata)?; - - let children = vec![input]; - + let mut children = vec![ + FormatTreeNode::new(format!("cte_name: {}", plan.cte_name)), + FormatTreeNode::new(format!("ref_count: {}", plan.ref_count)), + input, + ]; + Ok(FormatTreeNode::with_children( + format!("MaterializedCTE"), + children, + )) + } + PhysicalPlan::CTEConsumer(plan) => { + let mut children = vec![ + FormatTreeNode::new(format!("cte_name: {}", plan.cte_name)), + FormatTreeNode::new(format!( + "cte_schema: [{}]", + format_output_columns(plan.cte_schema.clone(), &metadata.read(), false) + )), + ]; + Ok(FormatTreeNode::with_children( + "CTEConsumer".to_string(), + children, + )) + } + PhysicalPlan::Sequence(plan) => { + let left = plan.left.format_join(metadata)?; + let right = plan.right.format_join(metadata)?; + let mut children = vec![left, right]; Ok(FormatTreeNode::with_children( - format!("MaterializedCTE: {}", plan.cte_name), + "Sequence".to_string(), children, )) } - PhysicalPlan::CTEConsumer(plan) => Ok(FormatTreeNode::with_children( - format!("CTEConsumer: {}", plan.cte_name), - vec![], - )), other => { let children = other .children() diff --git a/tests/sqllogictests/suites/tpch/join_order.test b/tests/sqllogictests/suites/tpch/join_order.test index 8bddcf3a12c90..21b3b5e4a4786 100644 --- a/tests/sqllogictests/suites/tpch/join_order.test +++ b/tests/sqllogictests/suites/tpch/join_order.test @@ -724,17 +724,24 @@ where order by s_suppkey; ---- -MaterializedCTE: revenue -├── Scan: default.tpch_test.lineitem (#3) (read rows: 6001215) +Sequence +├── MaterializedCTE +│ ├── cte_name: revenue +│ ├── ref_count: 2 +│ └── Scan: default.tpch_test.lineitem (#3) (read rows: 6001215) └── HashJoin: INNER ├── Build │ └── HashJoin: INNER │ ├── Build - │ │ └── CTEConsumer: revenue + │ │ └── CTEConsumer + │ │ ├── cte_name: revenue + │ │ └── cte_schema: [l_suppkey (#9), total_revenue (#25)] │ └── Probe │ └── Scan: default.tpch_test.supplier (#0) (read rows: 10000) └── Probe - └── CTEConsumer: revenue + └── CTEConsumer + ├── cte_name: revenue + └── cte_schema: [l_suppkey (#28), total_revenue (#44)] # Q16 From e385732b28e33d2a353f084e800ded40904821af Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Wed, 23 Jul 2025 17:07:58 +0800 Subject: [PATCH 60/80] fix logic test --- src/query/sql/src/executor/format.rs | 7 +-- .../standalone/explain/materialized_cte.test | 51 ++++++++-------- .../push_down_filter_self_join.test | 58 ++++++++++--------- 3 files changed, 60 insertions(+), 56 deletions(-) diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index cdcfc7b7a6e76..cf13c2aeb2591 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -178,7 +178,7 @@ impl PhysicalPlan { } PhysicalPlan::MaterializedCTE(plan) => { let input = plan.input.format_join(metadata)?; - let mut children = vec![ + let children = vec![ FormatTreeNode::new(format!("cte_name: {}", plan.cte_name)), FormatTreeNode::new(format!("ref_count: {}", plan.ref_count)), input, @@ -189,7 +189,7 @@ impl PhysicalPlan { )) } PhysicalPlan::CTEConsumer(plan) => { - let mut children = vec![ + let children = vec![ FormatTreeNode::new(format!("cte_name: {}", plan.cte_name)), FormatTreeNode::new(format!( "cte_schema: [{}]", @@ -204,7 +204,7 @@ impl PhysicalPlan { PhysicalPlan::Sequence(plan) => { let left = plan.left.format_join(metadata)?; let right = plan.right.format_join(metadata)?; - let mut children = vec![left, right]; + let children = vec![left, right]; Ok(FormatTreeNode::with_children( "Sequence".to_string(), children, @@ -588,7 +588,6 @@ fn to_format_tree( } PhysicalPlan::Sequence(plan) => { let mut children = Vec::new(); - children.push(FormatTreeNode::new(format!("Sequence"))); append_profile_info(&mut children, profs, plan.plan_id); children.push(to_format_tree(&plan.left, metadata, profs, context)?); children.push(to_format_tree(&plan.right, metadata, profs, context)?); diff --git a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test index 280d28b5c02a8..354c1275c5443 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test @@ -1,16 +1,17 @@ query T explain with t1 as materialized (select number as a from numbers(10)), t2 as (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b; ---- -MaterializedCTE: t1 -├── TableScan -│ ├── table: default.system.numbers -│ ├── output columns: [number (#2)] -│ ├── read rows: 10 -│ ├── read size: < 1 KiB -│ ├── partitions total: 1 -│ ├── partitions scanned: 1 -│ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 10.00 +Sequence +├── MaterializedCTE: t1 +│ └── TableScan +│ ├── table: default.system.numbers +│ ├── output columns: [number (#2)] +│ ├── read rows: 10 +│ ├── read size: < 1 KiB +│ ├── partitions total: 1 +│ ├── partitions scanned: 1 +│ ├── push downs: [filters: [], limit: NONE] +│ └── estimated rows: 10.00 └── HashJoin ├── output columns: [numbers.number (#0)] ├── join type: INNER @@ -31,20 +32,22 @@ MaterializedCTE: t1 query T explain with t1 as materialized (select number as a from numbers(10)), t2 as materialized (select a as b from t1) select t1.a from t1 join t2 on t1.a = t2.b; ---- -MaterializedCTE: t1 -├── TableScan -│ ├── table: default.system.numbers -│ ├── output columns: [number (#3)] -│ ├── read rows: 10 -│ ├── read size: < 1 KiB -│ ├── partitions total: 1 -│ ├── partitions scanned: 1 -│ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 10.00 -└── MaterializedCTE: t2 - ├── CTEConsumer - │ ├── cte_name: t1 - │ └── cte_schema: [number (#2)] +Sequence +├── MaterializedCTE: t1 +│ └── TableScan +│ ├── table: default.system.numbers +│ ├── output columns: [number (#3)] +│ ├── read rows: 10 +│ ├── read size: < 1 KiB +│ ├── partitions total: 1 +│ ├── partitions scanned: 1 +│ ├── push downs: [filters: [], limit: NONE] +│ └── estimated rows: 10.00 +└── Sequence + ├── MaterializedCTE: t2 + │ └── CTEConsumer + │ ├── cte_name: t1 + │ └── cte_schema: [number (#2)] └── HashJoin ├── output columns: [numbers.number (#0)] ├── join type: INNER diff --git a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test index f803bc97d4530..09b64a79562cc 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test @@ -11,34 +11,36 @@ C as (select * from B as b1 left outer join B as b2 on b1.a = b2.a where b1.b < D as (select * from C) select * from D; ---- -MaterializedCTE: a -├── UnionAll -│ ├── output columns: [a (#22), b (#23)] -│ ├── estimated rows: 20.00 -│ ├── TableScan -│ │ ├── table: default.default.t1 -│ │ ├── output columns: [a (#18), b (#19)] -│ │ ├── read rows: 10 -│ │ ├── read size: < 1 KiB -│ │ ├── partitions total: 1 -│ │ ├── partitions scanned: 1 -│ │ ├── pruning stats: [segments: , blocks: ] -│ │ ├── push downs: [filters: [], limit: NONE] -│ │ └── estimated rows: 10.00 -│ └── TableScan -│ ├── table: default.default.t2 -│ ├── output columns: [a (#20), b (#21)] -│ ├── read rows: 10 -│ ├── read size: < 1 KiB -│ ├── partitions total: 1 -│ ├── partitions scanned: 1 -│ ├── pruning stats: [segments: , blocks: ] -│ ├── push downs: [filters: [], limit: NONE] -│ └── estimated rows: 10.00 -└── MaterializedCTE: b - ├── CTEConsumer - │ ├── cte_name: a - │ └── cte_schema: [a (#16), b (#17)] +Sequence +├── MaterializedCTE: a +│ └── UnionAll +│ ├── output columns: [a (#22), b (#23)] +│ ├── estimated rows: 20.00 +│ ├── TableScan +│ │ ├── table: default.default.t1 +│ │ ├── output columns: [a (#18), b (#19)] +│ │ ├── read rows: 10 +│ │ ├── read size: < 1 KiB +│ │ ├── partitions total: 1 +│ │ ├── partitions scanned: 1 +│ │ ├── pruning stats: [segments: , blocks: ] +│ │ ├── push downs: [filters: [], limit: NONE] +│ │ └── estimated rows: 10.00 +│ └── TableScan +│ ├── table: default.default.t2 +│ ├── output columns: [a (#20), b (#21)] +│ ├── read rows: 10 +│ ├── read size: < 1 KiB +│ ├── partitions total: 1 +│ ├── partitions scanned: 1 +│ ├── pruning stats: [segments: , blocks: ] +│ ├── push downs: [filters: [], limit: NONE] +│ └── estimated rows: 10.00 +└── Sequence + ├── MaterializedCTE: b + │ └── CTEConsumer + │ ├── cte_name: a + │ └── cte_schema: [a (#16), b (#17)] └── HashJoin ├── output columns: [a (#4), b (#5), b (#11), a (#10)] ├── join type: INNER From 4fcfba535694e228cdcdc55be42a60f08014433e Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Thu, 24 Jul 2025 09:10:36 +0800 Subject: [PATCH 61/80] feat(query): add rule_grouping_sets_to_union --- .../optimizers/rule/agg_rules/mod.rs | 2 + .../agg_rules/rule_grouping_sets_to_union.rs | 216 ++++++++++++++++++ .../optimizer/optimizers/rule/factory.rs | 2 + .../planner/optimizer/optimizers/rule/rule.rs | 3 + 4 files changed, 223 insertions(+) create mode 100644 src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/mod.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/mod.rs index b1a25d3218ada..715343c80c6af 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/mod.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/mod.rs @@ -15,6 +15,7 @@ mod agg_index; mod rule_eager_aggregation; mod rule_fold_count_aggregate; +mod rule_grouping_sets_to_union; mod rule_push_down_filter_aggregate; mod rule_push_down_limit_aggregate; mod rule_split_aggregate; @@ -22,6 +23,7 @@ mod rule_try_apply_agg_index; pub use rule_eager_aggregation::RuleEagerAggregation; pub use rule_fold_count_aggregate::RuleFoldCountAggregate; +pub use rule_grouping_sets_to_union::RuleGroupingSetsToUnion; pub use rule_push_down_filter_aggregate::RulePushDownFilterAggregate; pub use rule_push_down_limit_aggregate::RulePushDownRankLimitAggregate; pub use rule_split_aggregate::RuleSplitAggregate; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs new file mode 100644 index 0000000000000..2f24aa273b288 --- /dev/null +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs @@ -0,0 +1,216 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::Result; +use databend_common_expression::Scalar; + +use crate::optimizer::ir::Matcher; +use crate::optimizer::ir::SExpr; +use crate::optimizer::optimizers::rule::Rule; +use crate::optimizer::optimizers::rule::RuleID; +use crate::optimizer::optimizers::rule::TransformResult; +use crate::plans::walk_expr_mut; +use crate::plans::Aggregate; +use crate::plans::AggregateMode; +use crate::plans::CastExpr; +use crate::plans::ConstantExpr; +use crate::plans::EvalScalar; +use crate::plans::RelOp; +use crate::plans::UnionAll; +use crate::plans::VisitorMut; +use crate::IndexType; +use crate::ScalarExpr; + +// TODO +const ID: RuleID = RuleID::RuleGroupingSetsToUnion; +// Split `Grouping Sets` into `Union All` of `Group by` +// Eg: +// select number % 10 AS a, number % 3 AS b, number % 4 AS c +// from numbers(100000000) +// group by grouping sets((a,b),(a,c)); + +// INTO: + +// select number % 10 AS a, number % 3 AS b, number % 4 AS c +// from numbers(100000000) +// group by a,b +// union all +// select number % 10 AS a, number % 3 AS b, number % 4 AS c +// from numbers(100000000) +// group by a,c +// +pub struct RuleGroupingSetsToUnion { + id: RuleID, + matchers: Vec, +} + +impl RuleGroupingSetsToUnion { + pub fn new() -> Self { + Self { + id: ID, + // Aggregate + // \ + // * + matchers: vec![Matcher::MatchOp { + op_type: RelOp::EvalScalar, + children: vec![Matcher::MatchOp { + op_type: RelOp::Aggregate, + children: vec![Matcher::Leaf], + }], + }], + } + } +} + +// Must go before `RuleSplitAggregate` +impl Rule for RuleGroupingSetsToUnion { + fn id(&self) -> RuleID { + self.id + } + + fn apply(&self, s_expr: &SExpr, state: &mut TransformResult) -> Result<()> { + let eval_scalar: EvalScalar = s_expr.plan().clone().try_into()?; + let agg: Aggregate = s_expr.child(0)?.plan().clone().try_into()?; + if agg.mode != AggregateMode::Initial { + return Ok(()); + } + + let agg_input = s_expr.child(0)?.child(0)?; + + if let Some(grouping_sets) = &agg.grouping_sets { + if grouping_sets.sets.len() > 1 { + let mut children = Vec::with_capacity(grouping_sets.sets.len()); + for set in &grouping_sets.sets { + let mut eval_scalar = eval_scalar.clone(); + let mut agg = agg.clone(); + agg.grouping_sets = None; + + let null_group_ids: Vec = agg + .group_items + .iter() + .map(|i| i.index) + .filter(|index| !set.contains(&index)) + .clone() + .collect(); + + agg.group_items.retain(|x| set.contains(&x.index)); + + let group_ids: Vec = + agg.group_items.iter().map(|i| i.index).collect(); + + let mut none_match_visitor = ReplaceColumnBeBullVisitor { + indexes: null_group_ids, + kind: SetKind::Null, + }; + + let mut match_visitor = ReplaceColumnBeBullVisitor { + indexes: group_ids, + kind: SetKind::Nullable, + }; + + // Replace the index to be null in aggregate functions + for func in agg.aggregate_functions.iter_mut() { + if let ScalarExpr::AggregateFunction(func) = &mut func.scalar { + for arg in func.args.iter_mut() { + none_match_visitor.visit(arg)?; + } + + for arg in func.sort_descs.iter_mut() { + none_match_visitor.visit(&mut arg.expr)?; + } + } + } + + for scalar in eval_scalar.items.iter_mut() { + none_match_visitor.visit(&mut scalar.scalar)?; + match_visitor.visit(&mut scalar.scalar)?; + } + + let agg_plan = SExpr::create_unary(agg, agg_input.clone()); + let eval_plan = SExpr::create_unary(eval_scalar, agg_plan); + children.push(eval_plan); + } + + // fold children into result + let mut result = children.first().unwrap().clone(); + for other in children.into_iter().skip(1) { + let union_plan = UnionAll { + left_outputs: eval_scalar.items.iter().map(|x| (x.index, None)).collect(), + right_outputs: eval_scalar.items.iter().map(|x| (x.index, None)).collect(), + cte_scan_names: vec![], + output_indexes: eval_scalar.items.iter().map(|x| x.index).collect(), + }; + result = SExpr::create_binary(Arc::new(union_plan.into()), result, other); + } + state.add_result(result); + return Ok(()); + } + } + Ok(()) + } + + fn matchers(&self) -> &[Matcher] { + &self.matchers + } +} + +impl Default for RuleGroupingSetsToUnion { + fn default() -> Self { + Self::new() + } +} + +enum SetKind { + Null, + Nullable, +} + +struct ReplaceColumnBeBullVisitor { + indexes: Vec, + kind: SetKind, +} + +impl VisitorMut<'_> for ReplaceColumnBeBullVisitor { + fn visit(&mut self, expr: &mut ScalarExpr) -> Result<()> { + let old = expr.clone(); + + if let ScalarExpr::BoundColumnRef(col) = expr { + if self.indexes.contains(&col.column.index) { + match self.kind { + SetKind::Null => { + *expr = ScalarExpr::TypedConstantExpr( + ConstantExpr { + value: Scalar::Null, + span: col.span, + }, + col.column.data_type.wrap_nullable(), + ); + } + SetKind::Nullable => { + *expr = ScalarExpr::CastExpr(CastExpr { + argument: Box::new(old), + is_try: true, + target_type: Box::new(col.column.data_type.wrap_nullable()), + span: col.span, + }); + } + } + } + return Ok(()); + } + walk_expr_mut(self, expr) + } +} diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs index f941c88dc4d9b..fcb5359692c70 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs @@ -25,6 +25,7 @@ use crate::optimizer::optimizers::rule::RuleEliminateSort; use crate::optimizer::optimizers::rule::RuleEliminateUnion; use crate::optimizer::optimizers::rule::RuleFilterNulls; use crate::optimizer::optimizers::rule::RuleFoldCountAggregate; +use crate::optimizer::optimizers::rule::RuleGroupingSetsToUnion; use crate::optimizer::optimizers::rule::RuleID; use crate::optimizer::optimizers::rule::RuleLeftExchangeJoin; use crate::optimizer::optimizers::rule::RuleMergeEvalScalar; @@ -101,6 +102,7 @@ impl RuleFactory { RuleID::MergeEvalScalar => Ok(Box::new(RuleMergeEvalScalar::new())), RuleID::MergeFilter => Ok(Box::new(RuleMergeFilter::new())), RuleID::NormalizeScalarFilter => Ok(Box::new(RuleNormalizeScalarFilter::new())), + RuleID::RuleGroupingSetsToUnion => Ok(Box::new(RuleGroupingSetsToUnion::new())), RuleID::SplitAggregate => Ok(Box::new(RuleSplitAggregate::new())), RuleID::FoldCountAggregate => Ok(Box::new(RuleFoldCountAggregate::new())), RuleID::CommuteJoin => Ok(Box::new(RuleCommuteJoin::new())), diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/rule.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/rule.rs index f6af7ad45b861..957ff98d08087 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/rule.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/rule.rs @@ -58,6 +58,7 @@ pub static DEFAULT_REWRITE_RULES: LazyLock> = LazyLock::new(|| { RuleID::PushDownFilterScan, RuleID::PushDownPrewhere, /* PushDownPrwhere should be after all rules except PushDownFilterScan */ RuleID::PushDownSortScan, // PushDownSortScan should be after PushDownPrewhere + RuleID::RuleGroupingSetsToUnion, ] }); @@ -112,6 +113,7 @@ pub enum RuleID { EliminateSort, MergeEvalScalar, MergeFilter, + RuleGroupingSetsToUnion, SplitAggregate, FoldCountAggregate, PushDownPrewhere, @@ -156,6 +158,7 @@ impl Display for RuleID { RuleID::MergeEvalScalar => write!(f, "MergeEvalScalar"), RuleID::MergeFilter => write!(f, "MergeFilter"), RuleID::NormalizeScalarFilter => write!(f, "NormalizeScalarFilter"), + RuleID::RuleGroupingSetsToUnion => write!(f, "RuleGroupingSetsToUnion"), RuleID::SplitAggregate => write!(f, "SplitAggregate"), RuleID::FoldCountAggregate => write!(f, "FoldCountAggregate"), RuleID::PushDownPrewhere => write!(f, "PushDownPrewhere"), From c7b9d955e8ad5c866efc876e314c7167a3cb7792 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Thu, 24 Jul 2025 16:26:04 +0800 Subject: [PATCH 62/80] feat(query): add rule_grouping_sets_to_union --- src/query/settings/src/settings_default.rs | 7 +++++++ .../settings/src/settings_getter_setter.rs | 4 ++++ .../src/planner/optimizer/optimizer_context.rs | 8 ++++++++ .../agg_rules/rule_grouping_sets_to_union.rs | 17 ++--------------- .../agg_rules/rule_push_down_limit_aggregate.rs | 2 +- .../optimizer/optimizers/rule/factory.rs | 4 ++-- .../planner/optimizer/optimizers/rule/rule.rs | 12 ++++++------ 7 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index 9b32bbbc233fd..bbeafaa178e88 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -506,6 +506,13 @@ impl DefaultSettings { scope: SettingScope::Both, range: Some(SettingRange::Numeric(0..=1)), }), + ("grouping_sets_to_union", DefaultSettingValue { + value: UserSettingValue::UInt64(0), + desc: "Enables grouping sets to union.", + mode: SettingMode::Both, + scope: SettingScope::Both, + range: Some(SettingRange::Numeric(0..=1)), + }), ("storage_fetch_part_num", DefaultSettingValue { value: UserSettingValue::UInt64(2), desc: "Sets the number of partitions that are fetched in parallel from storage during query execution.", diff --git a/src/query/settings/src/settings_getter_setter.rs b/src/query/settings/src/settings_getter_setter.rs index 0e9a7617a1ac9..3a088c18cbfbd 100644 --- a/src/query/settings/src/settings_getter_setter.rs +++ b/src/query/settings/src/settings_getter_setter.rs @@ -516,6 +516,10 @@ impl Settings { self.try_get_string("group_by_shuffle_mode") } + pub fn get_grouping_sets_to_union(&self) -> Result { + Ok(self.try_get_u64("grouping_sets_to_union")? == 1) + } + pub fn get_efficiently_memory_group_by(&self) -> Result { Ok(self.try_get_u64("efficiently_memory_group_by")? == 1) } diff --git a/src/query/sql/src/planner/optimizer/optimizer_context.rs b/src/query/sql/src/planner/optimizer/optimizer_context.rs index 35d682b55901f..82acdb2ccca1e 100644 --- a/src/query/sql/src/planner/optimizer/optimizer_context.rs +++ b/src/query/sql/src/planner/optimizer/optimizer_context.rs @@ -19,6 +19,7 @@ use databend_common_catalog::table_context::TableContext; use educe::Educe; use parking_lot::RwLock; +use crate::optimizer::optimizers::rule::RuleID; use crate::planner::QueryExecutor; use crate::MetadataRef; @@ -152,6 +153,13 @@ impl OptimizerContext { /// Check if an optimizer or rule is disabled based on optimizer_skip_list setting pub fn is_optimizer_disabled(self: &Arc, name: &str) -> bool { let settings = self.get_table_ctx().get_settings(); + + if !settings.get_grouping_sets_to_union().unwrap_or_default() { + if name == RuleID::GroupingSetsToUnion.to_string() { + return true; + } + } + match settings.get_optimizer_skip_list() { Ok(skip_list) if !skip_list.is_empty() => { let name_lower = name.to_lowercase(); diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs index 2f24aa273b288..cf79313f8817e 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs @@ -35,7 +35,7 @@ use crate::IndexType; use crate::ScalarExpr; // TODO -const ID: RuleID = RuleID::RuleGroupingSetsToUnion; +const ID: RuleID = RuleID::GroupingSetsToUnion; // Split `Grouping Sets` into `Union All` of `Group by` // Eg: // select number % 10 AS a, number % 3 AS b, number % 4 AS c @@ -102,7 +102,7 @@ impl Rule for RuleGroupingSetsToUnion { .group_items .iter() .map(|i| i.index) - .filter(|index| !set.contains(&index)) + .filter(|index| !set.contains(index)) .clone() .collect(); @@ -121,19 +121,6 @@ impl Rule for RuleGroupingSetsToUnion { kind: SetKind::Nullable, }; - // Replace the index to be null in aggregate functions - for func in agg.aggregate_functions.iter_mut() { - if let ScalarExpr::AggregateFunction(func) = &mut func.scalar { - for arg in func.args.iter_mut() { - none_match_visitor.visit(arg)?; - } - - for arg in func.sort_descs.iter_mut() { - none_match_visitor.visit(&mut arg.expr)?; - } - } - } - for scalar in eval_scalar.items.iter_mut() { none_match_visitor.visit(&mut scalar.scalar)?; match_visitor.visit(&mut scalar.scalar)?; diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_push_down_limit_aggregate.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_push_down_limit_aggregate.rs index bf2f1c1eb05ab..4917f45285697 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_push_down_limit_aggregate.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_push_down_limit_aggregate.rs @@ -47,7 +47,7 @@ pub struct RulePushDownRankLimitAggregate { impl RulePushDownRankLimitAggregate { pub fn new(max_limit: usize) -> Self { Self { - id: RuleID::RulePushDownRankLimitAggregate, + id: RuleID::PushDownRankLimitAggregate, matchers: vec![ Matcher::MatchOp { op_type: RelOp::Limit, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs index fcb5359692c70..913befc8e8002 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/factory.rs @@ -90,7 +90,7 @@ impl RuleFactory { RuleID::PushDownLimitWindow => Ok(Box::new(RulePushDownLimitWindow::new( ctx.get_max_push_down_limit(), ))), - RuleID::RulePushDownRankLimitAggregate => Ok(Box::new( + RuleID::PushDownRankLimitAggregate => Ok(Box::new( RulePushDownRankLimitAggregate::new(ctx.get_max_push_down_limit()), )), RuleID::PushDownFilterAggregate => Ok(Box::new(RulePushDownFilterAggregate::new())), @@ -102,7 +102,7 @@ impl RuleFactory { RuleID::MergeEvalScalar => Ok(Box::new(RuleMergeEvalScalar::new())), RuleID::MergeFilter => Ok(Box::new(RuleMergeFilter::new())), RuleID::NormalizeScalarFilter => Ok(Box::new(RuleNormalizeScalarFilter::new())), - RuleID::RuleGroupingSetsToUnion => Ok(Box::new(RuleGroupingSetsToUnion::new())), + RuleID::GroupingSetsToUnion => Ok(Box::new(RuleGroupingSetsToUnion::new())), RuleID::SplitAggregate => Ok(Box::new(RuleSplitAggregate::new())), RuleID::FoldCountAggregate => Ok(Box::new(RuleFoldCountAggregate::new())), RuleID::CommuteJoin => Ok(Box::new(RuleCommuteJoin::new())), diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/rule.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/rule.rs index 957ff98d08087..eabd76d1e3402 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/rule.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/rule.rs @@ -49,7 +49,7 @@ pub static DEFAULT_REWRITE_RULES: LazyLock> = LazyLock::new(|| { RuleID::PushDownLimitEvalScalar, RuleID::PushDownLimitSort, RuleID::PushDownLimitWindow, - RuleID::RulePushDownRankLimitAggregate, + RuleID::PushDownRankLimitAggregate, RuleID::PushDownLimitOuterJoin, RuleID::PushDownLimitScan, RuleID::SemiToInnerJoin, @@ -58,7 +58,7 @@ pub static DEFAULT_REWRITE_RULES: LazyLock> = LazyLock::new(|| { RuleID::PushDownFilterScan, RuleID::PushDownPrewhere, /* PushDownPrwhere should be after all rules except PushDownFilterScan */ RuleID::PushDownSortScan, // PushDownSortScan should be after PushDownPrewhere - RuleID::RuleGroupingSetsToUnion, + RuleID::GroupingSetsToUnion, ] }); @@ -103,7 +103,7 @@ pub enum RuleID { PushDownLimitEvalScalar, PushDownLimitSort, PushDownLimitWindow, - RulePushDownRankLimitAggregate, + PushDownRankLimitAggregate, PushDownLimitScan, PushDownSortEvalScalar, PushDownSortScan, @@ -113,7 +113,7 @@ pub enum RuleID { EliminateSort, MergeEvalScalar, MergeFilter, - RuleGroupingSetsToUnion, + GroupingSetsToUnion, SplitAggregate, FoldCountAggregate, PushDownPrewhere, @@ -144,7 +144,7 @@ impl Display for RuleID { RuleID::PushDownLimitOuterJoin => write!(f, "PushDownLimitOuterJoin"), RuleID::PushDownLimitEvalScalar => write!(f, "PushDownLimitEvalScalar"), RuleID::PushDownLimitSort => write!(f, "PushDownLimitSort"), - RuleID::RulePushDownRankLimitAggregate => write!(f, "RulePushDownRankLimitAggregate"), + RuleID::PushDownRankLimitAggregate => write!(f, "PushDownRankLimitAggregate"), RuleID::PushDownFilterAggregate => write!(f, "PushDownFilterAggregate"), RuleID::PushDownLimitScan => write!(f, "PushDownLimitScan"), RuleID::PushDownSortScan => write!(f, "PushDownSortScan"), @@ -158,7 +158,7 @@ impl Display for RuleID { RuleID::MergeEvalScalar => write!(f, "MergeEvalScalar"), RuleID::MergeFilter => write!(f, "MergeFilter"), RuleID::NormalizeScalarFilter => write!(f, "NormalizeScalarFilter"), - RuleID::RuleGroupingSetsToUnion => write!(f, "RuleGroupingSetsToUnion"), + RuleID::GroupingSetsToUnion => write!(f, "GroupingSetsToUnion"), RuleID::SplitAggregate => write!(f, "SplitAggregate"), RuleID::FoldCountAggregate => write!(f, "FoldCountAggregate"), RuleID::PushDownPrewhere => write!(f, "PushDownPrewhere"), From 1f8df6c925e2b0c7c213a4787589bf713117e1e4 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 24 Jul 2025 16:50:08 +0800 Subject: [PATCH 63/80] simplify --- .../src/planner/binder/bind_table_reference/bind_cte.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index 17dee5dfcbd78..a90584c66abe5 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -184,14 +184,9 @@ impl Binder { let materialized_cte = MaterializedCTE::new(cte_name, bind_context.columns, *ref_count); - let materialized_cte = - SExpr::create_unary(Arc::new(materialized_cte.into()), Arc::new(s_expr)); + let materialized_cte = SExpr::create_unary(materialized_cte, s_expr); let sequence = Sequence {}; - current_expr = SExpr::create_binary( - Arc::new(sequence.into()), - materialized_cte, - Arc::new(current_expr), - ); + current_expr = SExpr::create_binary(sequence, materialized_cte, current_expr); } } From a1178bff2339d9e1543e2062d4865c1bf5b1f4f4 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 24 Jul 2025 17:12:06 +0800 Subject: [PATCH 64/80] ref_count calculation is not required when constructing MaterializedCTE --- .../sql/src/planner/binder/bind_query/bind.rs | 8 +-- .../binder/bind_table_reference/bind_cte.rs | 7 +-- .../operator/cte/cleanup_unused_cte.rs | 54 ++++++++++++------- .../sql/src/planner/plans/materialized_cte.rs | 5 +- 4 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/query/sql/src/planner/binder/bind_query/bind.rs b/src/query/sql/src/planner/binder/bind_query/bind.rs index ec7190bbce4ae..fbc5ec7e44c95 100644 --- a/src/query/sql/src/planner/binder/bind_query/bind.rs +++ b/src/query/sql/src/planner/binder/bind_query/bind.rs @@ -85,13 +85,7 @@ impl Binder { s_expr = self.bind_query_limit(query, s_expr, limit, offset); if let Some(with) = &with { - let cte_ref_count = self.compute_cte_ref_count(with, query)?; - s_expr = self.bind_materialized_cte( - with, - s_expr, - bind_context.cte_context.clone(), - &cte_ref_count, - )?; + s_expr = self.bind_materialized_cte(with, s_expr, bind_context.cte_context.clone())?; } Ok((s_expr, bind_context)) diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index a90584c66abe5..f345b89cb6343 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::sync::Arc; use databend_common_ast::ast::Query; @@ -170,7 +169,6 @@ impl Binder { with: &With, main_query_expr: SExpr, cte_context: CteContext, - cte_ref_count: &HashMap, ) -> Result { let mut current_expr = main_query_expr; @@ -180,10 +178,7 @@ impl Binder { let (s_expr, bind_context) = self.bind_cte_definition(&cte_name, &cte_context.cte_map, &cte.query)?; - let ref_count = cte_ref_count.get(&cte_name).unwrap(); - - let materialized_cte = - MaterializedCTE::new(cte_name, bind_context.columns, *ref_count); + let materialized_cte = MaterializedCTE::new(cte_name, bind_context.columns); let materialized_cte = SExpr::create_unary(materialized_cte, s_expr); let sequence = Sequence {}; current_expr = SExpr::create_binary(sequence, materialized_cte, current_expr); diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs index cab43b979205e..28fa2fe908cbe 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs @@ -12,14 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashSet; +use std::collections::HashMap; use std::sync::Arc; use databend_common_exception::Result; use crate::optimizer::ir::SExpr; use crate::optimizer::Optimizer; -use crate::plans::MaterializedCTE; use crate::plans::RelOperator; /// Optimizer that removes unused CTEs from the query plan. @@ -28,21 +27,23 @@ use crate::plans::RelOperator; pub struct CleanupUnusedCTEOptimizer; impl CleanupUnusedCTEOptimizer { - /// Collect all CTE names that are referenced by CTEConsumer nodes - fn collect_referenced_ctes(s_expr: &SExpr) -> Result> { - let mut referenced_ctes = HashSet::new(); + /// Collect all CTE names that are referenced by CTEConsumer nodes and count their references + fn collect_referenced_ctes(s_expr: &SExpr) -> Result> { + let mut referenced_ctes = HashMap::new(); Self::collect_referenced_ctes_recursive(s_expr, &mut referenced_ctes)?; Ok(referenced_ctes) } - /// Recursively traverse the expression tree to find CTEConsumer nodes + /// Recursively traverse the expression tree to find CTEConsumer nodes and count references fn collect_referenced_ctes_recursive( s_expr: &SExpr, - referenced_ctes: &mut HashSet, + referenced_ctes: &mut HashMap, ) -> Result<()> { // Check if current node is a CTEConsumer if let RelOperator::CTEConsumer(consumer) = s_expr.plan() { - referenced_ctes.insert(consumer.cte_name.clone()); + *referenced_ctes + .entry(consumer.cte_name.clone()) + .or_insert(0) += 1; } // Recursively process children @@ -53,16 +54,33 @@ impl CleanupUnusedCTEOptimizer { Ok(()) } - /// Remove unused CTEs from the expression tree - fn remove_unused_ctes(s_expr: &SExpr, referenced_ctes: &HashSet) -> Result { + /// Remove unused CTEs from the expression tree and update ref_count + fn remove_unused_ctes( + s_expr: &SExpr, + referenced_ctes: &HashMap, + ) -> Result { if let RelOperator::Sequence(_) = s_expr.plan() { let left_child = s_expr.child(0)?.plan().clone(); - let cte: MaterializedCTE = left_child.try_into()?; - // If this CTE is not referenced, remove it by returning the right child - if !referenced_ctes.contains(&cte.cte_name) { - // Return the right child (main query) and skip the left child (CTE definition) - let right_child = s_expr.child(1)?; - return Self::remove_unused_ctes(right_child, referenced_ctes); + if let RelOperator::MaterializedCTE(mut cte) = left_child { + let ref_count = referenced_ctes.get(&cte.cte_name).cloned().unwrap_or(0); + if ref_count == 0 { + // Return the right child (main query) and skip the left child (CTE definition) + let right_child = s_expr.child(1)?; + return Self::remove_unused_ctes(right_child, referenced_ctes); + } else { + cte.ref_count = ref_count; + // Rebuild the left child with updated ref_count + let left_child_expr = + SExpr::create_unary(cte, Arc::new(s_expr.child(0)?.child(0)?.clone())); + let right_child_expr = + Self::remove_unused_ctes(s_expr.child(1)?, referenced_ctes)?; + let new_expr = SExpr::create_binary( + s_expr.plan().clone(), + Arc::new(left_child_expr), + Arc::new(right_child_expr), + ); + return Ok(new_expr); + } } } @@ -87,10 +105,10 @@ impl Optimizer for CleanupUnusedCTEOptimizer { } async fn optimize(&mut self, s_expr: &SExpr) -> Result { - // Collect all referenced CTEs + // Collect all referenced CTEs and their ref_count let referenced_ctes = Self::collect_referenced_ctes(s_expr)?; - // Remove unused CTEs + // Remove unused CTEs and update ref_count Self::remove_unused_ctes(s_expr, &referenced_ctes) } } diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index d2bc704fa93cf..4d2a613c8c172 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -35,11 +35,12 @@ pub struct MaterializedCTE { } impl MaterializedCTE { - pub fn new(cte_name: String, output_columns: Vec, ref_count: usize) -> Self { + pub fn new(cte_name: String, output_columns: Vec) -> Self { Self { cte_name, cte_output_columns: output_columns, - ref_count, + // ref_count is set to 0 by default, will be updated by CleanupUnusedCTEOptimizer + ref_count: 0, } } } From 934a2d424194c7068bb7277d1abe5477ebbe34bf Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Thu, 24 Jul 2025 17:22:13 +0800 Subject: [PATCH 65/80] Merge fb --- .../agg_rules/rule_grouping_sets_to_union.rs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs index cf79313f8817e..69bae10e1b791 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs @@ -25,10 +25,14 @@ use crate::optimizer::optimizers::rule::TransformResult; use crate::plans::walk_expr_mut; use crate::plans::Aggregate; use crate::plans::AggregateMode; +use crate::plans::CTEConsumer; use crate::plans::CastExpr; use crate::plans::ConstantExpr; use crate::plans::EvalScalar; +use crate::plans::MaterializedCTE; +use crate::plans::Operator; use crate::plans::RelOp; +use crate::plans::Sequence; use crate::plans::UnionAll; use crate::plans::VisitorMut; use crate::IndexType; @@ -133,6 +137,27 @@ impl Rule for RuleGroupingSetsToUnion { // fold children into result let mut result = children.first().unwrap().clone(); + + // Let's build a temporary CTE PLAN + let temp_cte_name = "cte".to_string(); + + let cte_materialized_plan = SExpr::create_unary( + MaterializedCTE { + cte_name: temp_cte_name, + cte_output_columns: vec![], + ref_count: children.len(), + }, + agg_input.clone(), + ); + + let cte_consumer = SExpr::create_leaf(CTEConsumer { + cte_name: temp_cte_name, + cte_schema: todo!(), + def: agg_input.clone(), + }); + + result = SExpr::create_unary(result.plan, cte_materialized_plan.clone()); + for other in children.into_iter().skip(1) { let union_plan = UnionAll { left_outputs: eval_scalar.items.iter().map(|x| (x.index, None)).collect(), @@ -142,6 +167,8 @@ impl Rule for RuleGroupingSetsToUnion { }; result = SExpr::create_binary(Arc::new(union_plan.into()), result, other); } + result = SExpr::create_binary(Sequence, cte_materialized_plan, result); + state.add_result(result); return Ok(()); } From 9a91ebb05857509d710bffd6890861f22f073376 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 24 Jul 2025 18:02:05 +0800 Subject: [PATCH 66/80] simplify MaterializedCTE --- .../builders/builder_materialized_cte.rs | 16 ++++++++------ .../physical_materialized_cte.rs | 22 +++++++++---------- .../binder/bind_table_reference/bind_cte.rs | 2 +- .../sql/src/planner/plans/materialized_cte.rs | 4 ++-- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 387516c7c224c..0800594ded0ab 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -21,13 +21,15 @@ impl PipelineBuilder { pub(crate) fn build_materialized_cte(&mut self, cte: &MaterializedCTE) -> Result<()> { self.build_pipeline(&cte.input)?; let input_schema = cte.input.output_schema()?; - Self::build_result_projection( - &self.func_ctx, - input_schema, - &cte.cte_output_columns, - &mut self.main_pipeline, - false, - )?; + if let Some(output_columns) = &cte.cte_output_columns { + Self::build_result_projection( + &self.func_ctx, + input_schema, + output_columns, + &mut self.main_pipeline, + false, + )?; + } self.main_pipeline.try_resize(1)?; let tx = self .ctx diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index 7833d3a022ff2..83064f4fdd88f 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -18,6 +18,7 @@ use databend_common_expression::DataSchemaRef; use crate::executor::explain::PlanStatsInfo; use crate::executor::PhysicalPlan; use crate::executor::PhysicalPlanBuilder; +use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::ColumnBinding; @@ -29,7 +30,7 @@ pub struct MaterializedCTE { pub stat_info: Option, pub input: Box, pub cte_name: String, - pub cte_output_columns: Vec, + pub cte_output_columns: Option>, pub ref_count: usize, } @@ -46,17 +47,14 @@ impl PhysicalPlanBuilder { materialized_cte: &crate::plans::MaterializedCTE, stat_info: PlanStatsInfo, ) -> Result { - let input = Box::new( - self.build( - s_expr.child(0)?, - materialized_cte - .cte_output_columns - .iter() - .map(|c| c.index) - .collect(), - ) - .await?, - ); + let required = match &materialized_cte.cte_output_columns { + Some(o) => o.iter().map(|c| c.index).collect(), + None => RelExpr::with_s_expr(s_expr.child(0)?) + .derive_relational_prop()? + .output_columns + .clone(), + }; + let input = Box::new(self.build(s_expr.child(0)?, required).await?); Ok(PhysicalPlan::MaterializedCTE(Box::new(MaterializedCTE { plan_id: 0, stat_info: Some(stat_info), diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index f345b89cb6343..bd77d9801d7ab 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -178,7 +178,7 @@ impl Binder { let (s_expr, bind_context) = self.bind_cte_definition(&cte_name, &cte_context.cte_map, &cte.query)?; - let materialized_cte = MaterializedCTE::new(cte_name, bind_context.columns); + let materialized_cte = MaterializedCTE::new(cte_name, Some(bind_context.columns)); let materialized_cte = SExpr::create_unary(materialized_cte, s_expr); let sequence = Sequence {}; current_expr = SExpr::create_binary(sequence, materialized_cte, current_expr); diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index 4d2a613c8c172..002fa24c96110 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -30,12 +30,12 @@ use crate::ColumnBinding; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct MaterializedCTE { pub cte_name: String, - pub cte_output_columns: Vec, + pub cte_output_columns: Option>, pub ref_count: usize, } impl MaterializedCTE { - pub fn new(cte_name: String, output_columns: Vec) -> Self { + pub fn new(cte_name: String, output_columns: Option>) -> Self { Self { cte_name, cte_output_columns: output_columns, From b79eb4131c568c4f9b578a05735bdb731d2d4972 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Thu, 24 Jul 2025 18:30:16 +0800 Subject: [PATCH 67/80] simplify CTEConsumer --- .../physical_plans/physical_cte_consumer.rs | 11 ++++++++++- .../binder/bind_table_reference/bind_cte.rs | 15 ++------------- src/query/sql/src/planner/plans/cte_consumer.rs | 3 +-- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs index cfb418c43fae2..53b7f94ef7730 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs @@ -13,7 +13,9 @@ // limitations under the License. use databend_common_exception::Result; +use databend_common_expression::DataField; use databend_common_expression::DataSchemaRef; +use databend_common_expression::DataSchemaRefExt; use crate::executor::explain::PlanStatsInfo; use crate::executor::PhysicalPlan; @@ -42,11 +44,18 @@ impl PhysicalPlanBuilder { cte_consumer: &crate::plans::CTEConsumer, stat_info: PlanStatsInfo, ) -> Result { + let mut fields = Vec::new(); + let metadata = self.metadata.read(); + for index in &cte_consumer.output_columns { + let column = metadata.column(*index); + fields.push(DataField::new(&index.to_string(), column.data_type())); + } + let cte_schema = DataSchemaRefExt::create(fields); Ok(PhysicalPlan::CTEConsumer(Box::new(CTEConsumer { plan_id: 0, stat_info: Some(stat_info), cte_name: cte_consumer.cte_name.clone(), - cte_schema: cte_consumer.cte_schema.clone(), + cte_schema, }))) } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index bd77d9801d7ab..a84ad45583b87 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -19,8 +19,6 @@ use databend_common_ast::ast::TableAlias; use databend_common_ast::ast::With; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::DataField; -use databend_common_expression::DataSchemaRefExt; use indexmap::IndexMap; use crate::binder::BindContext; @@ -116,16 +114,7 @@ impl Binder { cte_output_columns[index].column_name = column_name.clone(); } - let fields = cte_output_columns - .iter() - .map(|column_binding| { - DataField::new( - &column_binding.index.to_string(), - *column_binding.data_type.clone(), - ) - }) - .collect(); - let cte_schema = DataSchemaRefExt::create(fields); + let output_columns = cte_output_columns.iter().map(|c| c.index).collect(); let mut new_bind_context = bind_context.clone(); for column in cte_output_columns { @@ -134,7 +123,7 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { cte_name: table_name.to_string(), - cte_schema, + output_columns, def: s_expr, }))); Ok((s_expr, new_bind_context)) diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index fe40ee8070215..2365ea094c321 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -19,7 +19,6 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::DataSchemaRef; use crate::optimizer::ir::Distribution; use crate::optimizer::ir::PhysicalProperty; @@ -34,7 +33,7 @@ use crate::plans::RelOp; #[derive(Clone, Debug, PartialEq, Eq)] pub struct CTEConsumer { pub cte_name: String, - pub cte_schema: DataSchemaRef, + pub output_columns: Vec, pub def: SExpr, } From e38ca13ae40eb10c2c2e71285ba67ce3ddba2541 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Fri, 25 Jul 2025 11:12:56 +0800 Subject: [PATCH 68/80] Merge --- Cargo.lock | 1 + src/query/sql/Cargo.toml | 1 + .../agg_rules/rule_grouping_sets_to_union.rs | 47 +++++++++---------- .../sql/src/planner/plans/materialized_cte.rs | 4 +- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e8fb5735e302..3a025ce2541f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4223,6 +4223,7 @@ dependencies = [ "tokio", "unicase", "url", + "uuid", ] [[package]] diff --git a/src/query/sql/Cargo.toml b/src/query/sql/Cargo.toml index e8fdebbbde820..450c92e506029 100644 --- a/src/query/sql/Cargo.toml +++ b/src/query/sql/Cargo.toml @@ -45,6 +45,7 @@ async-trait = { workspace = true } chrono = { workspace = true } chrono-tz = { workspace = true } cidr = { workspace = true } +uuid = { workspace = true } cron = { workspace = true } ctor = { workspace = true } dashmap = { workspace = true } diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs index 69bae10e1b791..7f5dc44eac9ba 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs @@ -18,6 +18,7 @@ use databend_common_exception::Result; use databend_common_expression::Scalar; use crate::optimizer::ir::Matcher; +use crate::optimizer::ir::RelExpr; use crate::optimizer::ir::SExpr; use crate::optimizer::optimizers::rule::Rule; use crate::optimizer::optimizers::rule::RuleID; @@ -30,7 +31,6 @@ use crate::plans::CastExpr; use crate::plans::ConstantExpr; use crate::plans::EvalScalar; use crate::plans::MaterializedCTE; -use crate::plans::Operator; use crate::plans::RelOp; use crate::plans::Sequence; use crate::plans::UnionAll; @@ -93,10 +93,30 @@ impl Rule for RuleGroupingSetsToUnion { } let agg_input = s_expr.child(0)?.child(0)?; + let agg_input_columns: Vec = RelExpr::with_s_expr(&agg_input) + .derive_relational_prop()? + .output_columns + .iter() + .cloned() + .collect(); if let Some(grouping_sets) = &agg.grouping_sets { if grouping_sets.sets.len() > 1 { let mut children = Vec::with_capacity(grouping_sets.sets.len()); + + // Let's build a temporary CTE PLAN + let temp_cte_name = format!("cte_{}", uuid::Uuid::new_v4().simple().to_string()); + let cte_materialized_sexpr = SExpr::create_unary( + MaterializedCTE::new(temp_cte_name.clone(), None), + agg_input.clone(), + ); + + let cte_consumer = SExpr::create_leaf(CTEConsumer { + cte_name: temp_cte_name, + output_columns: agg_input_columns.clone(), + def: agg_input.clone(), + }); + for set in &grouping_sets.sets { let mut eval_scalar = eval_scalar.clone(); let mut agg = agg.clone(); @@ -130,34 +150,13 @@ impl Rule for RuleGroupingSetsToUnion { match_visitor.visit(&mut scalar.scalar)?; } - let agg_plan = SExpr::create_unary(agg, agg_input.clone()); + let agg_plan = SExpr::create_unary(agg, cte_consumer.clone()); let eval_plan = SExpr::create_unary(eval_scalar, agg_plan); children.push(eval_plan); } // fold children into result let mut result = children.first().unwrap().clone(); - - // Let's build a temporary CTE PLAN - let temp_cte_name = "cte".to_string(); - - let cte_materialized_plan = SExpr::create_unary( - MaterializedCTE { - cte_name: temp_cte_name, - cte_output_columns: vec![], - ref_count: children.len(), - }, - agg_input.clone(), - ); - - let cte_consumer = SExpr::create_leaf(CTEConsumer { - cte_name: temp_cte_name, - cte_schema: todo!(), - def: agg_input.clone(), - }); - - result = SExpr::create_unary(result.plan, cte_materialized_plan.clone()); - for other in children.into_iter().skip(1) { let union_plan = UnionAll { left_outputs: eval_scalar.items.iter().map(|x| (x.index, None)).collect(), @@ -167,7 +166,7 @@ impl Rule for RuleGroupingSetsToUnion { }; result = SExpr::create_binary(Arc::new(union_plan.into()), result, other); } - result = SExpr::create_binary(Sequence, cte_materialized_plan, result); + result = SExpr::create_binary(Sequence, cte_materialized_sexpr, result); state.add_result(result); return Ok(()); diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index 002fa24c96110..71d4dc676bf83 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -35,9 +35,9 @@ pub struct MaterializedCTE { } impl MaterializedCTE { - pub fn new(cte_name: String, output_columns: Option>) -> Self { + pub fn new(cte_name: impl Into, output_columns: Option>) -> Self { Self { - cte_name, + cte_name: cte_name.into(), cte_output_columns: output_columns, // ref_count is set to 0 by default, will be updated by CleanupUnusedCTEOptimizer ref_count: 0, From 576f98a3bb92af89dd8c022e62efd68c26e9a550 Mon Sep 17 00:00:00 2001 From: Sky Fan <3374614481@qq.com> Date: Fri, 25 Jul 2025 11:31:56 +0800 Subject: [PATCH 69/80] Update src/query/service/src/pipelines/builders/builder_sequence.rs Co-authored-by: Winter Zhang --- src/query/service/src/pipelines/builders/builder_sequence.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/service/src/pipelines/builders/builder_sequence.rs b/src/query/service/src/pipelines/builders/builder_sequence.rs index 7f0c3f1c3bc9d..7bcf722acccc4 100644 --- a/src/query/service/src/pipelines/builders/builder_sequence.rs +++ b/src/query/service/src/pipelines/builders/builder_sequence.rs @@ -32,7 +32,7 @@ impl PipelineBuilder { self.pipelines.extend(build_res.sources_pipelines); // build main pipeline - self.build_pipeline(&sequence.right)?; + self.build_pipeline(&sequence.right) Ok(()) } } From df268f786b41b2e6ee5a64512fd021bb5e73adfd Mon Sep 17 00:00:00 2001 From: Sky Fan <3374614481@qq.com> Date: Fri, 25 Jul 2025 11:32:19 +0800 Subject: [PATCH 70/80] Update src/query/service/src/pipelines/builders/builder_materialized_cte.rs Co-authored-by: Winter Zhang --- .../service/src/pipelines/builders/builder_materialized_cte.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 0800594ded0ab..8de775f0ad68b 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -35,7 +35,7 @@ impl PipelineBuilder { .ctx .get_materialized_cte_senders(&cte.cte_name, cte.ref_count); self.main_pipeline - .add_sink(|input| MaterializedCteSink::create(input, tx.clone()))?; + .add_sink(|input| MaterializedCteSink::create(input, tx.clone())) Ok(()) } } From a05383418b632d2e4ab547ca1b2ad07a5949b519 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Fri, 25 Jul 2025 11:49:11 +0800 Subject: [PATCH 71/80] make lint --- .../service/src/pipelines/builders/builder_materialized_cte.rs | 1 - src/query/service/src/pipelines/builders/builder_sequence.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 8de775f0ad68b..8f64bb253770e 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -36,6 +36,5 @@ impl PipelineBuilder { .get_materialized_cte_senders(&cte.cte_name, cte.ref_count); self.main_pipeline .add_sink(|input| MaterializedCteSink::create(input, tx.clone())) - Ok(()) } } diff --git a/src/query/service/src/pipelines/builders/builder_sequence.rs b/src/query/service/src/pipelines/builders/builder_sequence.rs index 7bcf722acccc4..6a8164404cfc9 100644 --- a/src/query/service/src/pipelines/builders/builder_sequence.rs +++ b/src/query/service/src/pipelines/builders/builder_sequence.rs @@ -33,6 +33,5 @@ impl PipelineBuilder { // build main pipeline self.build_pipeline(&sequence.right) - Ok(()) } } From 0a01f288f0bdc1a9848b0af3ff069d7f33ee0af2 Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Fri, 25 Jul 2025 12:03:43 +0800 Subject: [PATCH 72/80] rename CTEConsumer to MaterializeCTERef --- .../builders/builder_cte_consumer.rs | 4 ++-- .../service/src/pipelines/pipeline_builder.rs | 2 +- .../processors/transforms/materialized_cte.rs | 2 +- .../transform_recursive_cte_source.rs | 2 +- .../src/schedulers/fragments/fragmenter.rs | 6 +++--- src/query/sql/src/executor/format.rs | 8 ++++---- src/query/sql/src/executor/physical_plan.rs | 18 ++++++++--------- .../sql/src/executor/physical_plan_builder.rs | 2 +- .../sql/src/executor/physical_plan_visitor.rs | 10 +++++----- .../physical_plans/physical_cte_consumer.rs | 20 ++++++++++--------- .../binder/bind_table_reference/bind_cte.rs | 14 +++++++------ .../sql/src/planner/optimizer/ir/format.rs | 2 +- .../optimizer/optimizers/hyper_dp/dphyp.rs | 6 +++--- .../hyper_dp/dynamic_sample/dynamic_sample.rs | 2 +- .../operator/cte/cleanup_unused_cte.rs | 10 +++++----- .../decorrelate/subquery_decorrelator.rs | 2 +- .../join_rules/rule_semi_to_inner_join.rs | 2 +- .../sql/src/planner/plans/cte_consumer.rs | 8 ++++---- src/query/sql/src/planner/plans/operator.rs | 8 ++++---- .../sql/src/planner/plans/operator_macros.rs | 2 +- .../standalone/explain/materialized_cte.test | 10 +++++----- .../push_down_filter_self_join.test | 6 +++--- .../sqllogictests/suites/tpch/join_order.test | 4 ++-- 23 files changed, 77 insertions(+), 73 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs index 7eba2117cad13..7d81a51bc3f52 100644 --- a/src/query/service/src/pipelines/builders/builder_cte_consumer.rs +++ b/src/query/service/src/pipelines/builders/builder_cte_consumer.rs @@ -13,14 +13,14 @@ // limitations under the License. use databend_common_exception::Result; -use databend_common_sql::executor::physical_plans::CTEConsumer; +use databend_common_sql::executor::physical_plans::MaterializeCTERef; use databend_common_storages_fuse::TableContext; use crate::pipelines::processors::transforms::CTESource; use crate::pipelines::PipelineBuilder; impl PipelineBuilder { - pub(crate) fn build_cte_consumer(&mut self, cte: &CTEConsumer) -> Result<()> { + pub(crate) fn build_cte_consumer(&mut self, cte: &MaterializeCTERef) -> Result<()> { let receiver = self.ctx.get_materialized_cte_receiver(&cte.cte_name); self.main_pipeline.add_source( |output_port| { diff --git a/src/query/service/src/pipelines/pipeline_builder.rs b/src/query/service/src/pipelines/pipeline_builder.rs index 64dba326e941f..3d3937aaaf8cd 100644 --- a/src/query/service/src/pipelines/pipeline_builder.rs +++ b/src/query/service/src/pipelines/pipeline_builder.rs @@ -297,7 +297,7 @@ impl PipelineBuilder { )), PhysicalPlan::MaterializedCTE(cte) => self.build_materialized_cte(cte), - PhysicalPlan::CTEConsumer(cte_consumer) => self.build_cte_consumer(cte_consumer), + PhysicalPlan::MaterializeCTERef(cte_consumer) => self.build_cte_consumer(cte_consumer), PhysicalPlan::Sequence(sequence) => self.build_sequence(sequence), }?; diff --git a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs index 4a7317863de5d..46efb557eaab3 100644 --- a/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs +++ b/src/query/service/src/pipelines/processors/transforms/materialized_cte.rs @@ -77,7 +77,7 @@ impl CTESource { #[async_trait::async_trait] impl AsyncSource for CTESource { - const NAME: &'static str = "CTEConsumerSource"; + const NAME: &'static str = "MaterializeCTESource"; #[async_backtrace::framed] async fn generate(&mut self) -> Result> { diff --git a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs index 4e515788c27da..83d8747d615b9 100644 --- a/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs +++ b/src/query/service/src/pipelines/processors/transforms/transform_recursive_cte_source.rs @@ -368,7 +368,7 @@ async fn create_memory_table_for_cte_scan( | PhysicalPlan::ChunkMerge(_) | PhysicalPlan::ChunkCommitInsert(_) | PhysicalPlan::BroadcastSource(_) - | PhysicalPlan::CTEConsumer(_) + | PhysicalPlan::MaterializeCTERef(_) | PhysicalPlan::BroadcastSink(_) => {} } Ok(()) diff --git a/src/query/service/src/schedulers/fragments/fragmenter.rs b/src/query/service/src/schedulers/fragments/fragmenter.rs index ac5dfda0c8645..458b441725a66 100644 --- a/src/query/service/src/schedulers/fragments/fragmenter.rs +++ b/src/query/service/src/schedulers/fragments/fragmenter.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use databend_common_catalog::table_context::TableContext; use databend_common_exception::Result; use databend_common_meta_types::NodeInfo; -use databend_common_sql::executor::physical_plans::CTEConsumer; use databend_common_sql::executor::physical_plans::CompactSource; use databend_common_sql::executor::physical_plans::ConstantTableScan; use databend_common_sql::executor::physical_plans::CopyIntoTable; @@ -27,6 +26,7 @@ use databend_common_sql::executor::physical_plans::ExchangeSink; use databend_common_sql::executor::physical_plans::ExchangeSource; use databend_common_sql::executor::physical_plans::FragmentKind; use databend_common_sql::executor::physical_plans::HashJoin; +use databend_common_sql::executor::physical_plans::MaterializeCTERef; use databend_common_sql::executor::physical_plans::MaterializedCTE; use databend_common_sql::executor::physical_plans::MutationSource; use databend_common_sql::executor::physical_plans::RangeJoin; @@ -164,9 +164,9 @@ impl PhysicalPlanReplacer for Fragmenter { Ok(PhysicalPlan::TableScan(plan.clone())) } - fn replace_cte_consumer(&mut self, plan: &CTEConsumer) -> Result { + fn replace_cte_consumer(&mut self, plan: &MaterializeCTERef) -> Result { self.state = State::Other; - Ok(PhysicalPlan::CTEConsumer(Box::new(plan.clone()))) + Ok(PhysicalPlan::MaterializeCTERef(Box::new(plan.clone()))) } fn replace_constant_table_scan(&mut self, plan: &ConstantTableScan) -> Result { diff --git a/src/query/sql/src/executor/format.rs b/src/query/sql/src/executor/format.rs index cf13c2aeb2591..4b1839c136d1a 100644 --- a/src/query/sql/src/executor/format.rs +++ b/src/query/sql/src/executor/format.rs @@ -188,7 +188,7 @@ impl PhysicalPlan { children, )) } - PhysicalPlan::CTEConsumer(plan) => { + PhysicalPlan::MaterializeCTERef(plan) => { let children = vec![ FormatTreeNode::new(format!("cte_name: {}", plan.cte_name)), FormatTreeNode::new(format!( @@ -197,7 +197,7 @@ impl PhysicalPlan { )), ]; Ok(FormatTreeNode::with_children( - "CTEConsumer".to_string(), + "MaterializeCTERef".to_string(), children, )) } @@ -570,7 +570,7 @@ fn to_format_tree( children, )) } - PhysicalPlan::CTEConsumer(plan) => { + PhysicalPlan::MaterializeCTERef(plan) => { let mut children = Vec::new(); children.push(FormatTreeNode::new(format!( "cte_name: {}", @@ -582,7 +582,7 @@ fn to_format_tree( ))); append_profile_info(&mut children, profs, plan.plan_id); Ok(FormatTreeNode::with_children( - "CTEConsumer".to_string(), + "MaterializeCTERef".to_string(), children, )) } diff --git a/src/query/sql/src/executor/physical_plan.rs b/src/query/sql/src/executor/physical_plan.rs index 37899282ab246..1d9d93dd375c2 100644 --- a/src/query/sql/src/executor/physical_plan.rs +++ b/src/query/sql/src/executor/physical_plan.rs @@ -37,7 +37,6 @@ use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFinal; use crate::executor::physical_plans::AggregatePartial; use crate::executor::physical_plans::AsyncFunction; -use crate::executor::physical_plans::CTEConsumer; use crate::executor::physical_plans::CacheScan; use crate::executor::physical_plans::ChunkAppendData; use crate::executor::physical_plans::ChunkCastSchema; @@ -63,6 +62,7 @@ use crate::executor::physical_plans::ExpressionScan; use crate::executor::physical_plans::Filter; use crate::executor::physical_plans::HashJoin; use crate::executor::physical_plans::Limit; +use crate::executor::physical_plans::MaterializeCTERef; use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Mutation; use crate::executor::physical_plans::ProjectSet; @@ -167,7 +167,7 @@ pub enum PhysicalPlan { // CTE MaterializedCTE(Box), - CTEConsumer(Box), + MaterializeCTERef(Box), Sequence(Box), } @@ -435,7 +435,7 @@ impl PhysicalPlan { *next_id += 1; plan.input.adjust_plan_id(next_id); } - PhysicalPlan::CTEConsumer(plan) => { + PhysicalPlan::MaterializeCTERef(plan) => { plan.plan_id = *next_id; *next_id += 1; } @@ -504,7 +504,7 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSource(v) => v.plan_id, PhysicalPlan::BroadcastSink(v) => v.plan_id, PhysicalPlan::MaterializedCTE(v) => v.plan_id, - PhysicalPlan::CTEConsumer(v) => v.plan_id, + PhysicalPlan::MaterializeCTERef(v) => v.plan_id, PhysicalPlan::Sequence(v) => v.plan_id, } } @@ -564,7 +564,7 @@ impl PhysicalPlan { PhysicalPlan::ChunkMerge(_) => todo!(), PhysicalPlan::ChunkCommitInsert(_) => todo!(), PhysicalPlan::MaterializedCTE(plan) => plan.output_schema(), - PhysicalPlan::CTEConsumer(plan) => plan.output_schema(), + PhysicalPlan::MaterializeCTERef(plan) => plan.output_schema(), PhysicalPlan::Sequence(plan) => plan.output_schema(), } } @@ -630,7 +630,7 @@ impl PhysicalPlan { PhysicalPlan::BroadcastSource(_) => "RuntimeFilterSource".to_string(), PhysicalPlan::BroadcastSink(_) => "RuntimeFilterSink".to_string(), PhysicalPlan::MaterializedCTE(_) => "MaterializedCTE".to_string(), - PhysicalPlan::CTEConsumer(_) => "CTEConsumer".to_string(), + PhysicalPlan::MaterializeCTERef(_) => "MaterializeCTERef".to_string(), PhysicalPlan::Sequence(_) => "Sequence".to_string(), } } @@ -645,7 +645,7 @@ impl PhysicalPlan { | PhysicalPlan::ReplaceAsyncSourcer(_) | PhysicalPlan::Recluster(_) | PhysicalPlan::RecursiveCteScan(_) - | PhysicalPlan::CTEConsumer(_) + | PhysicalPlan::MaterializeCTERef(_) | PhysicalPlan::BroadcastSource(_) => Box::new(std::iter::empty()), PhysicalPlan::HilbertPartition(plan) => Box::new(std::iter::once(plan.input.as_ref())), PhysicalPlan::Filter(plan) => Box::new(std::iter::once(plan.input.as_ref())), @@ -724,7 +724,7 @@ impl PhysicalPlan { | PhysicalPlan::ReplaceAsyncSourcer(_) | PhysicalPlan::Recluster(_) | PhysicalPlan::BroadcastSource(_) - | PhysicalPlan::CTEConsumer(_) + | PhysicalPlan::MaterializeCTERef(_) | PhysicalPlan::RecursiveCteScan(_) => Box::new(std::iter::empty()), PhysicalPlan::HilbertPartition(plan) => Box::new(std::iter::once(plan.input.as_mut())), PhysicalPlan::Filter(plan) => Box::new(std::iter::once(plan.input.as_mut())), @@ -849,7 +849,7 @@ impl PhysicalPlan { | PhysicalPlan::BroadcastSource(_) | PhysicalPlan::BroadcastSink(_) | PhysicalPlan::MaterializedCTE(_) - | PhysicalPlan::CTEConsumer(_) + | PhysicalPlan::MaterializeCTERef(_) | PhysicalPlan::Sequence(_) => None, } } diff --git a/src/query/sql/src/executor/physical_plan_builder.rs b/src/query/sql/src/executor/physical_plan_builder.rs index c97ff00113cc1..fda4b7808b0ea 100644 --- a/src/query/sql/src/executor/physical_plan_builder.rs +++ b/src/query/sql/src/executor/physical_plan_builder.rs @@ -133,7 +133,7 @@ impl PhysicalPlanBuilder { self.build_materialized_cte(s_expr, materialized_cte, stat_info) .await } - RelOperator::CTEConsumer(cte_consumer) => { + RelOperator::MaterializeCTERef(cte_consumer) => { self.build_cte_consumer(cte_consumer, stat_info).await } RelOperator::Sequence(sequence) => { diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index a4de69cee8f36..8fa7bec68ab73 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -29,7 +29,6 @@ use crate::executor::physical_plans::AggregateExpand; use crate::executor::physical_plans::AggregateFinal; use crate::executor::physical_plans::AggregatePartial; use crate::executor::physical_plans::AsyncFunction; -use crate::executor::physical_plans::CTEConsumer; use crate::executor::physical_plans::ChunkAppendData; use crate::executor::physical_plans::ChunkCastSchema; use crate::executor::physical_plans::ChunkCommitInsert; @@ -53,6 +52,7 @@ use crate::executor::physical_plans::ExchangeSource; use crate::executor::physical_plans::Filter; use crate::executor::physical_plans::HashJoin; use crate::executor::physical_plans::Limit; +use crate::executor::physical_plans::MaterializeCTERef; use crate::executor::physical_plans::MaterializedCTE; use crate::executor::physical_plans::Mutation; use crate::executor::physical_plans::MutationSource; @@ -128,7 +128,7 @@ pub trait PhysicalPlanReplacer { PhysicalPlan::BroadcastSource(plan) => self.replace_runtime_filter_source(plan), PhysicalPlan::BroadcastSink(plan) => self.replace_runtime_filter_sink(plan), PhysicalPlan::MaterializedCTE(plan) => self.replace_materialized_cte(plan), - PhysicalPlan::CTEConsumer(plan) => self.replace_cte_consumer(plan), + PhysicalPlan::MaterializeCTERef(plan) => self.replace_cte_consumer(plan), PhysicalPlan::Sequence(plan) => self.replace_sequence(plan), } } @@ -662,8 +662,8 @@ pub trait PhysicalPlanReplacer { }))) } - fn replace_cte_consumer(&mut self, plan: &CTEConsumer) -> Result { - Ok(PhysicalPlan::CTEConsumer(Box::new(plan.clone()))) + fn replace_cte_consumer(&mut self, plan: &MaterializeCTERef) -> Result { + Ok(PhysicalPlan::MaterializeCTERef(Box::new(plan.clone()))) } fn replace_sequence(&mut self, plan: &Sequence) -> Result { @@ -699,7 +699,7 @@ impl PhysicalPlan { | PhysicalPlan::ExchangeSource(_) | PhysicalPlan::CompactSource(_) | PhysicalPlan::MutationSource(_) - | PhysicalPlan::CTEConsumer(_) + | PhysicalPlan::MaterializeCTERef(_) | PhysicalPlan::BroadcastSource(_) => {} PhysicalPlan::Filter(plan) => { Self::traverse(&plan.input, pre_visit, visit, post_visit); diff --git a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs index 53b7f94ef7730..18a205a9f8809 100644 --- a/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs +++ b/src/query/sql/src/executor/physical_plans/physical_cte_consumer.rs @@ -23,7 +23,7 @@ use crate::executor::PhysicalPlanBuilder; /// This is a leaf operator that consumes the result of a materialized CTE. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -pub struct CTEConsumer { +pub struct MaterializeCTERef { // A unique id of operator in a `PhysicalPlan` tree, only used for display. pub plan_id: u32, // Only used for explain @@ -32,7 +32,7 @@ pub struct CTEConsumer { pub cte_schema: DataSchemaRef, } -impl CTEConsumer { +impl MaterializeCTERef { pub fn output_schema(&self) -> Result { Ok(self.cte_schema.clone()) } @@ -41,7 +41,7 @@ impl CTEConsumer { impl PhysicalPlanBuilder { pub(crate) async fn build_cte_consumer( &mut self, - cte_consumer: &crate::plans::CTEConsumer, + cte_consumer: &crate::plans::MaterializeCTERef, stat_info: PlanStatsInfo, ) -> Result { let mut fields = Vec::new(); @@ -51,11 +51,13 @@ impl PhysicalPlanBuilder { fields.push(DataField::new(&index.to_string(), column.data_type())); } let cte_schema = DataSchemaRefExt::create(fields); - Ok(PhysicalPlan::CTEConsumer(Box::new(CTEConsumer { - plan_id: 0, - stat_info: Some(stat_info), - cte_name: cte_consumer.cte_name.clone(), - cte_schema, - }))) + Ok(PhysicalPlan::MaterializeCTERef(Box::new( + MaterializeCTERef { + plan_id: 0, + stat_info: Some(stat_info), + cte_name: cte_consumer.cte_name.clone(), + cte_schema, + }, + ))) } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index a84ad45583b87..e9845aa9b6a60 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -27,7 +27,7 @@ use crate::binder::CteContext; use crate::binder::CteInfo; use crate::normalize_identifier; use crate::optimizer::ir::SExpr; -use crate::plans::CTEConsumer; +use crate::plans::MaterializeCTERef; use crate::plans::MaterializedCTE; use crate::plans::RelOperator; use crate::plans::Sequence; @@ -121,11 +121,13 @@ impl Binder { new_bind_context.add_column_binding(column); } - let s_expr = SExpr::create_leaf(Arc::new(RelOperator::CTEConsumer(CTEConsumer { - cte_name: table_name.to_string(), - output_columns, - def: s_expr, - }))); + let s_expr = SExpr::create_leaf(Arc::new(RelOperator::MaterializeCTERef( + MaterializeCTERef { + cte_name: table_name.to_string(), + output_columns, + def: s_expr, + }, + ))); Ok((s_expr, new_bind_context)) } diff --git a/src/query/sql/src/planner/optimizer/ir/format.rs b/src/query/sql/src/planner/optimizer/ir/format.rs index 7e7ec8b63bec4..957e4c77c35b7 100644 --- a/src/query/sql/src/planner/optimizer/ir/format.rs +++ b/src/query/sql/src/planner/optimizer/ir/format.rs @@ -81,7 +81,7 @@ fn display_rel_op(rel_op: &RelOperator) -> String { RelOperator::MutationSource(_) => "MutationSource".to_string(), RelOperator::CompactBlock(_) => "CompactBlock".to_string(), RelOperator::MaterializedCTE(_) => "MaterializedCTE".to_string(), - RelOperator::CTEConsumer(_) => "CTEConsumer".to_string(), + RelOperator::MaterializeCTERef(_) => "MaterializeCTERef".to_string(), RelOperator::Sequence(_) => "Sequence".to_string(), } } diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs index a31376ae5b045..fd90d73eaa962 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dphyp.rs @@ -297,7 +297,7 @@ impl DPhpyOptimizer { join_relation: Option<&SExpr>, ) -> Result<(Arc, bool)> { let cte_consumer = match s_expr.plan() { - RelOperator::CTEConsumer(consumer) => consumer, + RelOperator::MaterializeCTERef(consumer) => consumer, _ => unreachable!(), }; @@ -332,7 +332,7 @@ impl DPhpyOptimizer { table_indexes.push(scan.table_index); } - if let RelOperator::CTEConsumer(cte_consumer) = s_expr.plan() { + if let RelOperator::MaterializeCTERef(cte_consumer) = s_expr.plan() { Self::collect_table_indexes_recursive(&cte_consumer.def, table_indexes); } @@ -404,7 +404,7 @@ impl DPhpyOptimizer { RelOperator::Join(_) => self.process_join_node(s_expr, join_conditions).await, RelOperator::Sequence(_) => self.process_sequence_node(s_expr).await, - RelOperator::CTEConsumer(_) => { + RelOperator::MaterializeCTERef(_) => { self.process_cte_consumer_node(s_expr, join_relation).await } diff --git a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs index 3f1d2ace543f7..be3032562a04b 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/hyper_dp/dynamic_sample/dynamic_sample.rs @@ -93,7 +93,7 @@ pub async fn dynamic_sample( | RelOperator::Mutation(_) | RelOperator::CompactBlock(_) | RelOperator::MaterializedCTE(_) - | RelOperator::CTEConsumer(_) + | RelOperator::MaterializeCTERef(_) | RelOperator::Sequence(_) | RelOperator::MutationSource(_) => { s_expr.plan().derive_stats(&RelExpr::with_s_expr(s_expr)) diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs index 28fa2fe908cbe..87f6f922b87c7 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs @@ -23,24 +23,24 @@ use crate::plans::RelOperator; /// Optimizer that removes unused CTEs from the query plan. /// This optimizer should be applied at the end of the optimization pipeline -/// to clean up any CTEs that are not referenced by any CTEConsumer. +/// to clean up any CTEs that are not referenced by any MaterializeCTERef. pub struct CleanupUnusedCTEOptimizer; impl CleanupUnusedCTEOptimizer { - /// Collect all CTE names that are referenced by CTEConsumer nodes and count their references + /// Collect all CTE names that are referenced by MaterializeCTERef nodes and count their references fn collect_referenced_ctes(s_expr: &SExpr) -> Result> { let mut referenced_ctes = HashMap::new(); Self::collect_referenced_ctes_recursive(s_expr, &mut referenced_ctes)?; Ok(referenced_ctes) } - /// Recursively traverse the expression tree to find CTEConsumer nodes and count references + /// Recursively traverse the expression tree to find MaterializeCTERef nodes and count references fn collect_referenced_ctes_recursive( s_expr: &SExpr, referenced_ctes: &mut HashMap, ) -> Result<()> { - // Check if current node is a CTEConsumer - if let RelOperator::CTEConsumer(consumer) = s_expr.plan() { + // Check if current node is a MaterializeCTERef + if let RelOperator::MaterializeCTERef(consumer) = s_expr.plan() { *referenced_ctes .entry(consumer.cte_name.clone()) .or_insert(0) += 1; diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs index bbcd120ee9fd7..fa6e7eeee9d76 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/decorrelate/subquery_decorrelator.rs @@ -335,7 +335,7 @@ impl SubqueryDecorrelatorOptimizer { | RelOperator::RecursiveCteScan(_) | RelOperator::Mutation(_) | RelOperator::MutationSource(_) - | RelOperator::CTEConsumer(_) + | RelOperator::MaterializeCTERef(_) | RelOperator::CompactBlock(_) => Ok(s_expr.clone()), } } diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs index 7ec7e8742ec94..00449c81fb277 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/join_rules/rule_semi_to_inner_join.rs @@ -153,7 +153,7 @@ fn find_group_by_keys( | RelOperator::Mutation(_) | RelOperator::MutationSource(_) | RelOperator::MaterializedCTE(_) - | RelOperator::CTEConsumer(_) + | RelOperator::MaterializeCTERef(_) | RelOperator::CompactBlock(_) | RelOperator::Sequence(_) => {} } diff --git a/src/query/sql/src/planner/plans/cte_consumer.rs b/src/query/sql/src/planner/plans/cte_consumer.rs index 2365ea094c321..7b2f68db4c2ef 100644 --- a/src/query/sql/src/planner/plans/cte_consumer.rs +++ b/src/query/sql/src/planner/plans/cte_consumer.rs @@ -31,21 +31,21 @@ use crate::plans::Operator; use crate::plans::RelOp; #[derive(Clone, Debug, PartialEq, Eq)] -pub struct CTEConsumer { +pub struct MaterializeCTERef { pub cte_name: String, pub output_columns: Vec, pub def: SExpr, } -impl Hash for CTEConsumer { +impl Hash for MaterializeCTERef { fn hash(&self, state: &mut H) { self.cte_name.hash(state); } } -impl Operator for CTEConsumer { +impl Operator for MaterializeCTERef { fn rel_op(&self) -> RelOp { - RelOp::CTEConsumer + RelOp::MaterializeCTERef } /// Get arity of this operator diff --git a/src/query/sql/src/planner/plans/operator.rs b/src/query/sql/src/planner/plans/operator.rs index 189bdedd18336..ae0cf09b42ffe 100644 --- a/src/query/sql/src/planner/plans/operator.rs +++ b/src/query/sql/src/planner/plans/operator.rs @@ -34,7 +34,6 @@ use crate::plans::r_cte_scan::RecursiveCteScan; use crate::plans::sequence::Sequence; use crate::plans::Aggregate; use crate::plans::AsyncFunction; -use crate::plans::CTEConsumer; use crate::plans::CacheScan; use crate::plans::ConstantTableScan; use crate::plans::DummyTableScan; @@ -44,6 +43,7 @@ use crate::plans::ExpressionScan; use crate::plans::Filter; use crate::plans::Join; use crate::plans::Limit; +use crate::plans::MaterializeCTERef; use crate::plans::MaterializedCTE; use crate::plans::Mutation; use crate::plans::OptimizeCompactBlock as CompactBlock; @@ -131,7 +131,7 @@ pub enum RelOp { CompactBlock, MutationSource, MaterializedCTE, - CTEConsumer, + MaterializeCTERef, Sequence, } @@ -167,7 +167,7 @@ pub enum RelOperator { CompactBlock(CompactBlock), MutationSource(MutationSource), MaterializedCTE(MaterializedCTE), - CTEConsumer(CTEConsumer), + MaterializeCTERef(MaterializeCTERef), Sequence(Sequence), } @@ -260,6 +260,6 @@ impl_try_from_rel_operator! { CompactBlock, MutationSource, MaterializedCTE, - CTEConsumer, + MaterializeCTERef, Sequence } diff --git a/src/query/sql/src/planner/plans/operator_macros.rs b/src/query/sql/src/planner/plans/operator_macros.rs index a81779d997c58..d3a2f176f33a4 100644 --- a/src/query/sql/src/planner/plans/operator_macros.rs +++ b/src/query/sql/src/planner/plans/operator_macros.rs @@ -111,7 +111,7 @@ macro_rules! impl_match_rel_op { RelOperator::CompactBlock($rel_op) => $rel_op.$method($($arg),*), RelOperator::MutationSource($rel_op) => $rel_op.$method($($arg),*), RelOperator::MaterializedCTE($rel_op) => $rel_op.$method($($arg),*), - RelOperator::CTEConsumer($rel_op) => $rel_op.$method($($arg),*), + RelOperator::MaterializeCTERef($rel_op) => $rel_op.$method($($arg),*), RelOperator::Sequence($rel_op) => $rel_op.$method($($arg),*), } } diff --git a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test index 354c1275c5443..8c43c5e79a377 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/materialized_cte.test @@ -22,10 +22,10 @@ Sequence ├── build join filters: │ └── filter id:0, build key:t2.b (#1), probe key:t1.a (#0), filter type:bloom,inlist,min_max ├── estimated rows: 0.00 - ├── CTEConsumer(Build) + ├── MaterializeCTERef(Build) │ ├── cte_name: t1 │ └── cte_schema: [number (#1)] - └── CTEConsumer(Probe) + └── MaterializeCTERef(Probe) ├── cte_name: t1 └── cte_schema: [number (#0)] @@ -45,7 +45,7 @@ Sequence │ └── estimated rows: 10.00 └── Sequence ├── MaterializedCTE: t2 - │ └── CTEConsumer + │ └── MaterializeCTERef │ ├── cte_name: t1 │ └── cte_schema: [number (#2)] └── HashJoin @@ -58,9 +58,9 @@ Sequence ├── build join filters: │ └── filter id:0, build key:t2.b (#1), probe key:t1.a (#0), filter type:bloom,inlist,min_max ├── estimated rows: 0.00 - ├── CTEConsumer(Build) + ├── MaterializeCTERef(Build) │ ├── cte_name: t2 │ └── cte_schema: [number (#1)] - └── CTEConsumer(Probe) + └── MaterializeCTERef(Probe) ├── cte_name: t1 └── cte_schema: [number (#0)] diff --git a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test index 09b64a79562cc..42673b5481a1f 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_join/push_down_filter_self_join.test @@ -38,7 +38,7 @@ Sequence │ └── estimated rows: 10.00 └── Sequence ├── MaterializedCTE: b - │ └── CTEConsumer + │ └── MaterializeCTERef │ ├── cte_name: a │ └── cte_schema: [a (#16), b (#17)] └── HashJoin @@ -49,9 +49,9 @@ Sequence ├── keys is null equal: [false] ├── filters: [d.b (#5) < d.b (#11)] ├── estimated rows: 0.00 - ├── CTEConsumer(Build) + ├── MaterializeCTERef(Build) │ ├── cte_name: b │ └── cte_schema: [a (#10), b (#11)] - └── CTEConsumer(Probe) + └── MaterializeCTERef(Probe) ├── cte_name: b └── cte_schema: [a (#4), b (#5)] diff --git a/tests/sqllogictests/suites/tpch/join_order.test b/tests/sqllogictests/suites/tpch/join_order.test index 21b3b5e4a4786..4b0805da6d422 100644 --- a/tests/sqllogictests/suites/tpch/join_order.test +++ b/tests/sqllogictests/suites/tpch/join_order.test @@ -733,13 +733,13 @@ Sequence ├── Build │ └── HashJoin: INNER │ ├── Build - │ │ └── CTEConsumer + │ │ └── MaterializeCTERef │ │ ├── cte_name: revenue │ │ └── cte_schema: [l_suppkey (#9), total_revenue (#25)] │ └── Probe │ └── Scan: default.tpch_test.supplier (#0) (read rows: 10000) └── Probe - └── CTEConsumer + └── MaterializeCTERef ├── cte_name: revenue └── cte_schema: [l_suppkey (#28), total_revenue (#44)] From 4513b15c25dcc9952df4efc25bdb0f5f632b572c Mon Sep 17 00:00:00 2001 From: Sky Fan <3374614481@qq.com> Date: Fri, 25 Jul 2025 12:05:06 +0800 Subject: [PATCH 73/80] Update src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs Co-authored-by: Winter Zhang --- .../optimizer/optimizers/operator/cte/cleanup_unused_cte.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs index 87f6f922b87c7..c4a508d274e6c 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/operator/cte/cleanup_unused_cte.rs @@ -35,6 +35,7 @@ impl CleanupUnusedCTEOptimizer { } /// Recursively traverse the expression tree to find MaterializeCTERef nodes and count references + #[recursive::recursive] fn collect_referenced_ctes_recursive( s_expr: &SExpr, referenced_ctes: &mut HashMap, From 5f951f00a154c8aba3dfa1ab85e2055edb66b1c9 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Fri, 25 Jul 2025 22:55:26 +0800 Subject: [PATCH 74/80] Merge --- Cargo.lock | 1 - src/query/sql/Cargo.toml | 1 - .../physical_aggregate_final.rs | 15 +- src/query/sql/src/planner/binder/aggregate.rs | 19 ++- .../agg_rules/rule_grouping_sets_to_union.rs | 120 ++++++++----- .../rule_eliminate_eval_scalar.rs | 26 +++ .../group_by_grouping_sets_union_all.test | 7 + .../explain/explain_grouping_sets.test | 161 ++++++++++++++++++ 8 files changed, 286 insertions(+), 64 deletions(-) create mode 100644 tests/sqllogictests/suites/duckdb/sql/aggregate/group/group_by_grouping_sets_union_all.test diff --git a/Cargo.lock b/Cargo.lock index 3a025ce2541f4..6e8fb5735e302 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4223,7 +4223,6 @@ dependencies = [ "tokio", "unicase", "url", - "uuid", ] [[package]] diff --git a/src/query/sql/Cargo.toml b/src/query/sql/Cargo.toml index 450c92e506029..e8fdebbbde820 100644 --- a/src/query/sql/Cargo.toml +++ b/src/query/sql/Cargo.toml @@ -45,7 +45,6 @@ async-trait = { workspace = true } chrono = { workspace = true } chrono-tz = { workspace = true } cidr = { workspace = true } -uuid = { workspace = true } cron = { workspace = true } ctor = { workspace = true } dashmap = { workspace = true } diff --git a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs index d3511fbb8cdaa..1c62800a40e6d 100644 --- a/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs +++ b/src/query/sql/src/executor/physical_plans/physical_aggregate_final.rs @@ -232,13 +232,13 @@ impl PhysicalPlanBuilder { settings.get_enable_experimental_aggregate_hashtable()?; if let Some(grouping_sets) = agg.grouping_sets.as_ref() { - assert_eq!(grouping_sets.dup_group_items.len(), group_items.len() - 1); // ignore `_grouping_id`. - // If the aggregation function argument if a group item, - // we cannot use the group item directly. - // It's because the group item will be wrapped with nullable and fill dummy NULLs (in `AggregateExpand` plan), - // which will cause panic while executing aggregation function. - // To avoid the panic, we will duplicate (`Arc::clone`) original group item columns in `AggregateExpand`, - // we should use these columns instead. + // ignore `_grouping_id`. + // If the aggregation function argument if a group item, + // we cannot use the group item directly. + // It's because the group item will be wrapped with nullable and fill dummy NULLs (in `AggregateExpand` plan), + // which will cause panic while executing aggregation function. + // To avoid the panic, we will duplicate (`Arc::clone`) original group item columns in `AggregateExpand`, + // we should use these columns instead. for func in agg_funcs.iter_mut() { for arg in func.arg_indices.iter_mut() { if let Some(pos) = group_items.iter().position(|g| g == arg) { @@ -480,7 +480,6 @@ impl PhysicalPlanBuilder { if let Some(grouping_sets) = agg.grouping_sets.as_ref() { // The argument types are wrapped nullable due to `AggregateExpand` plan. We should recover them to original types. - assert_eq!(grouping_sets.dup_group_items.len(), group_items.len() - 1); // ignore `_grouping_id`. for func in agg_funcs.iter_mut() { for (arg, ty) in func.arg_indices.iter_mut().zip(func.sig.args.iter_mut()) { if let Some(pos) = group_items.iter().position(|g| g == arg) { diff --git a/src/query/sql/src/planner/binder/aggregate.rs b/src/query/sql/src/planner/binder/aggregate.rs index 8f4bac849bb4a..fea0d22d07d2a 100644 --- a/src/query/sql/src/planner/binder/aggregate.rs +++ b/src/query/sql/src/planner/binder/aggregate.rs @@ -771,6 +771,7 @@ impl Binder { ); dup_group_items.push((dummy.index, *dummy.data_type)); } + // Add a virtual column `_grouping_id` to group items. let grouping_id_column = self.create_derived_column_binding( "_grouping_id".to_string(), @@ -783,14 +784,16 @@ impl Binder { column: grouping_id_column.clone(), }; - agg_info.group_items_map.insert( - bound_grouping_id_col.clone().into(), - agg_info.group_items.len(), - ); - agg_info.group_items.push(ScalarItem { - index: grouping_id_column.index, - scalar: bound_grouping_id_col.into(), - }); + if !self.ctx.get_settings().get_grouping_sets_to_union()? { + agg_info.group_items_map.insert( + bound_grouping_id_col.clone().into(), + agg_info.group_items.len(), + ); + agg_info.group_items.push(ScalarItem { + index: grouping_id_column.index, + scalar: bound_grouping_id_col.into(), + }); + } let grouping_sets_info = GroupingSetsInfo { grouping_id_column, diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs index 7f5dc44eac9ba..3f77cb857ec33 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs @@ -12,9 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::hash::DefaultHasher; +use std::hash::Hash; +use std::hash::Hasher; use std::sync::Arc; use databend_common_exception::Result; +use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; use crate::optimizer::ir::Matcher; @@ -101,11 +105,13 @@ impl Rule for RuleGroupingSetsToUnion { .collect(); if let Some(grouping_sets) = &agg.grouping_sets { - if grouping_sets.sets.len() > 1 { + if grouping_sets.sets.len() >= 1 { let mut children = Vec::with_capacity(grouping_sets.sets.len()); - // Let's build a temporary CTE PLAN - let temp_cte_name = format!("cte_{}", uuid::Uuid::new_v4().simple().to_string()); + let mut hasher = DefaultHasher::new(); + agg.grouping_sets.hash(&mut hasher); + let hash = hasher.finish(); + let temp_cte_name = format!("cte_groupingsets_{hash}"); let cte_materialized_sexpr = SExpr::create_unary( MaterializedCTE::new(temp_cte_name.clone(), None), agg_input.clone(), @@ -117,7 +123,34 @@ impl Rule for RuleGroupingSetsToUnion { def: agg_input.clone(), }); + let mask = (1 << grouping_sets.dup_group_items.len()) - 1; + let group_bys = agg + .group_items + .iter() + .map(|i| { + agg_input_columns + .iter() + .position(|t| *t == i.index) + .unwrap() + }) + .collect::>(); + for set in &grouping_sets.sets { + let mut id = 0; + + // For element in `group_bys`, + // if it is in current grouping set: set 0, else: set 1. (1 represents it will be NULL in grouping) + // Example: GROUP BY GROUPING SETS ((a, b), (a), (b), ()) + // group_bys: [a, b] + // grouping_sets: [[0, 1], [0], [1], []] + // grouping_ids: 00, 01, 10, 11 + + for g in set { + let i = group_bys.iter().position(|t| *t == *g).unwrap(); + id |= 1 << i; + } + let grouping_id = !id & mask; + let mut eval_scalar = eval_scalar.clone(); let mut agg = agg.clone(); agg.grouping_sets = None; @@ -131,23 +164,18 @@ impl Rule for RuleGroupingSetsToUnion { .collect(); agg.group_items.retain(|x| set.contains(&x.index)); - let group_ids: Vec = agg.group_items.iter().map(|i| i.index).collect(); - let mut none_match_visitor = ReplaceColumnBeBullVisitor { - indexes: null_group_ids, - kind: SetKind::Null, - }; - - let mut match_visitor = ReplaceColumnBeBullVisitor { - indexes: group_ids, - kind: SetKind::Nullable, + let mut visitor = ReplaceColumnForGroupingSetsVisitor { + group_indexes: group_ids, + exclude_group_indexes: null_group_ids, + grouping_id_index: grouping_sets.grouping_id_index, + grouping_id_value: grouping_id, }; for scalar in eval_scalar.items.iter_mut() { - none_match_visitor.visit(&mut scalar.scalar)?; - match_visitor.visit(&mut scalar.scalar)?; + visitor.visit(&mut scalar.scalar)?; } let agg_plan = SExpr::create_unary(agg, cte_consumer.clone()); @@ -158,16 +186,19 @@ impl Rule for RuleGroupingSetsToUnion { // fold children into result let mut result = children.first().unwrap().clone(); for other in children.into_iter().skip(1) { + let left_outputs: Vec<(IndexType, Option)> = + eval_scalar.items.iter().map(|x| (x.index, None)).collect(); + let right_outputs = left_outputs.clone(); + let union_plan = UnionAll { - left_outputs: eval_scalar.items.iter().map(|x| (x.index, None)).collect(), - right_outputs: eval_scalar.items.iter().map(|x| (x.index, None)).collect(), + left_outputs, + right_outputs, cte_scan_names: vec![], output_indexes: eval_scalar.items.iter().map(|x| x.index).collect(), }; result = SExpr::create_binary(Arc::new(union_plan.into()), result, other); } result = SExpr::create_binary(Sequence, cte_materialized_sexpr, result); - state.add_result(result); return Ok(()); } @@ -186,41 +217,38 @@ impl Default for RuleGroupingSetsToUnion { } } -enum SetKind { - Null, - Nullable, -} - -struct ReplaceColumnBeBullVisitor { - indexes: Vec, - kind: SetKind, +struct ReplaceColumnForGroupingSetsVisitor { + group_indexes: Vec, + exclude_group_indexes: Vec, + grouping_id_index: IndexType, + grouping_id_value: u32, } -impl VisitorMut<'_> for ReplaceColumnBeBullVisitor { +impl VisitorMut<'_> for ReplaceColumnForGroupingSetsVisitor { fn visit(&mut self, expr: &mut ScalarExpr) -> Result<()> { let old = expr.clone(); if let ScalarExpr::BoundColumnRef(col) = expr { - if self.indexes.contains(&col.column.index) { - match self.kind { - SetKind::Null => { - *expr = ScalarExpr::TypedConstantExpr( - ConstantExpr { - value: Scalar::Null, - span: col.span, - }, - col.column.data_type.wrap_nullable(), - ); - } - SetKind::Nullable => { - *expr = ScalarExpr::CastExpr(CastExpr { - argument: Box::new(old), - is_try: true, - target_type: Box::new(col.column.data_type.wrap_nullable()), - span: col.span, - }); - } - } + if self.group_indexes.contains(&col.column.index) { + *expr = ScalarExpr::CastExpr(CastExpr { + argument: Box::new(old), + is_try: true, + target_type: Box::new(col.column.data_type.wrap_nullable()), + span: col.span, + }); + } else if self.exclude_group_indexes.contains(&col.column.index) { + *expr = ScalarExpr::TypedConstantExpr( + ConstantExpr { + value: Scalar::Null, + span: col.span, + }, + col.column.data_type.wrap_nullable(), + ); + } else if self.grouping_id_index == col.column.index { + *expr = ScalarExpr::ConstantExpr(ConstantExpr { + value: Scalar::Number(NumberScalar::UInt32(self.grouping_id_value)), + span: col.span, + }); } return Ok(()); } diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs index e04f2058f70ce..1acb7d127ccb2 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/scalar_rules/rule_eliminate_eval_scalar.rs @@ -25,6 +25,7 @@ use crate::plans::Operator; use crate::plans::RelOp; use crate::ColumnSet; use crate::MetadataRef; +use crate::ScalarExpr; pub struct RuleEliminateEvalScalar { id: RuleID, @@ -73,7 +74,32 @@ impl Rule for RuleEliminateEvalScalar { .clone(); let eval_scalar_output_cols: ColumnSet = eval_scalar.items.iter().map(|x| x.index).collect(); + if eval_scalar_output_cols.is_subset(&child_output_cols) { + // check if there's f(#x) as #x, if so we can't eliminate the eval scalar + for item in eval_scalar.items { + match item.scalar { + ScalarExpr::FunctionCall(func) => { + if func.arguments.len() == 1 { + if let ScalarExpr::BoundColumnRef(bound_column_ref) = &func.arguments[0] + { + if bound_column_ref.column.index == item.index { + return Ok(()); + } + } + } + } + ScalarExpr::CastExpr(cast) => { + if let ScalarExpr::BoundColumnRef(bound_column_ref) = cast.argument.as_ref() + { + if bound_column_ref.column.index == item.index { + return Ok(()); + } + } + } + _ => {} + } + } state.add_result(s_expr.child(0)?.clone()); return Ok(()); } diff --git a/tests/sqllogictests/suites/duckdb/sql/aggregate/group/group_by_grouping_sets_union_all.test b/tests/sqllogictests/suites/duckdb/sql/aggregate/group/group_by_grouping_sets_union_all.test new file mode 100644 index 0000000000000..c815d16d84465 --- /dev/null +++ b/tests/sqllogictests/suites/duckdb/sql/aggregate/group/group_by_grouping_sets_union_all.test @@ -0,0 +1,7 @@ +statement ok +set grouping_sets_to_union = 1; + +include ./group_by_grouping_sets.test + +statement ok +unset grouping_sets_to_union; diff --git a/tests/sqllogictests/suites/mode/standalone/explain/explain_grouping_sets.test b/tests/sqllogictests/suites/mode/standalone/explain/explain_grouping_sets.test index 1573e610ea0f1..2ab12c830786d 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/explain_grouping_sets.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/explain_grouping_sets.test @@ -65,3 +65,164 @@ EvalScalar ├── partitions scanned: 1 ├── push downs: [filters: [], limit: NONE] └── estimated rows: 1.00 + + +statement ok +set grouping_sets_to_union = 1; + +query T +explain select number % 2 as a, number % 3 as b, number % 5 as c from numbers(1) group by cube(a, b, c); +---- +Sequence +├── MaterializedCTE: cte_groupingsets_16366510952463710337 +│ └── EvalScalar +│ ├── output columns: [numbers.number (#0), a (#1), b (#2), c (#3)] +│ ├── expressions: [numbers.number (#0) % 2, numbers.number (#0) % 3, numbers.number (#0) % 5] +│ ├── estimated rows: 1.00 +│ └── TableScan +│ ├── table: default.system.numbers +│ ├── output columns: [number (#0)] +│ ├── read rows: 1 +│ ├── read size: < 1 KiB +│ ├── partitions total: 1 +│ ├── partitions scanned: 1 +│ ├── push downs: [filters: [], limit: NONE] +│ └── estimated rows: 1.00 +└── UnionAll + ├── output columns: [a (#8), b (#9), c (#10)] + ├── estimated rows: 8.00 + ├── UnionAll + │ ├── output columns: [a (#8), b (#9), c (#10)] + │ ├── estimated rows: 7.00 + │ ├── UnionAll + │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ ├── estimated rows: 6.00 + │ │ ├── UnionAll + │ │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ │ ├── estimated rows: 5.00 + │ │ │ ├── UnionAll + │ │ │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ │ │ ├── estimated rows: 4.00 + │ │ │ │ ├── UnionAll + │ │ │ │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ │ │ │ ├── estimated rows: 3.00 + │ │ │ │ │ ├── UnionAll + │ │ │ │ │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ │ │ │ │ ├── estimated rows: 2.00 + │ │ │ │ │ │ ├── EvalScalar + │ │ │ │ │ │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ │ │ │ │ │ ├── expressions: [NULL, NULL, NULL] + │ │ │ │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ │ │ │ └── DummyTableScan + │ │ │ │ │ │ └── EvalScalar + │ │ │ │ │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ │ │ │ │ ├── expressions: [TRY_CAST(group_item (#1) AS UInt8 NULL), NULL, NULL] + │ │ │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ │ │ └── AggregateFinal + │ │ │ │ │ │ ├── output columns: [a (#1)] + │ │ │ │ │ │ ├── group by: [a] + │ │ │ │ │ │ ├── aggregate functions: [] + │ │ │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ │ │ └── AggregatePartial + │ │ │ │ │ │ ├── group by: [a] + │ │ │ │ │ │ ├── aggregate functions: [] + │ │ │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ │ │ └── CTEConsumer + │ │ │ │ │ │ ├── cte_name: cte_groupingsets_16366510952463710337 + │ │ │ │ │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] + │ │ │ │ │ └── EvalScalar + │ │ │ │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ │ │ │ ├── expressions: [NULL, TRY_CAST(group_item (#2) AS UInt8 NULL), NULL] + │ │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ │ └── AggregateFinal + │ │ │ │ │ ├── output columns: [b (#2)] + │ │ │ │ │ ├── group by: [b] + │ │ │ │ │ ├── aggregate functions: [] + │ │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ │ └── AggregatePartial + │ │ │ │ │ ├── group by: [b] + │ │ │ │ │ ├── aggregate functions: [] + │ │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ │ └── CTEConsumer + │ │ │ │ │ ├── cte_name: cte_groupingsets_16366510952463710337 + │ │ │ │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] + │ │ │ │ └── EvalScalar + │ │ │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ │ │ ├── expressions: [NULL, NULL, TRY_CAST(group_item (#3) AS UInt8 NULL)] + │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ └── AggregateFinal + │ │ │ │ ├── output columns: [c (#3)] + │ │ │ │ ├── group by: [c] + │ │ │ │ ├── aggregate functions: [] + │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ └── AggregatePartial + │ │ │ │ ├── group by: [c] + │ │ │ │ ├── aggregate functions: [] + │ │ │ │ ├── estimated rows: 1.00 + │ │ │ │ └── CTEConsumer + │ │ │ │ ├── cte_name: cte_groupingsets_16366510952463710337 + │ │ │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] + │ │ │ └── EvalScalar + │ │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ │ ├── expressions: [TRY_CAST(group_item (#1) AS UInt8 NULL), TRY_CAST(group_item (#2) AS UInt8 NULL), NULL] + │ │ │ ├── estimated rows: 1.00 + │ │ │ └── AggregateFinal + │ │ │ ├── output columns: [a (#1), b (#2)] + │ │ │ ├── group by: [a, b] + │ │ │ ├── aggregate functions: [] + │ │ │ ├── estimated rows: 1.00 + │ │ │ └── AggregatePartial + │ │ │ ├── group by: [a, b] + │ │ │ ├── aggregate functions: [] + │ │ │ ├── estimated rows: 1.00 + │ │ │ └── CTEConsumer + │ │ │ ├── cte_name: cte_groupingsets_16366510952463710337 + │ │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] + │ │ └── EvalScalar + │ │ ├── output columns: [a (#8), b (#9), c (#10)] + │ │ ├── expressions: [TRY_CAST(group_item (#1) AS UInt8 NULL), NULL, TRY_CAST(group_item (#3) AS UInt8 NULL)] + │ │ ├── estimated rows: 1.00 + │ │ └── AggregateFinal + │ │ ├── output columns: [a (#1), c (#3)] + │ │ ├── group by: [a, c] + │ │ ├── aggregate functions: [] + │ │ ├── estimated rows: 1.00 + │ │ └── AggregatePartial + │ │ ├── group by: [a, c] + │ │ ├── aggregate functions: [] + │ │ ├── estimated rows: 1.00 + │ │ └── CTEConsumer + │ │ ├── cte_name: cte_groupingsets_16366510952463710337 + │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] + │ └── EvalScalar + │ ├── output columns: [a (#8), b (#9), c (#10)] + │ ├── expressions: [NULL, TRY_CAST(group_item (#2) AS UInt8 NULL), TRY_CAST(group_item (#3) AS UInt8 NULL)] + │ ├── estimated rows: 1.00 + │ └── AggregateFinal + │ ├── output columns: [b (#2), c (#3)] + │ ├── group by: [b, c] + │ ├── aggregate functions: [] + │ ├── estimated rows: 1.00 + │ └── AggregatePartial + │ ├── group by: [b, c] + │ ├── aggregate functions: [] + │ ├── estimated rows: 1.00 + │ └── CTEConsumer + │ ├── cte_name: cte_groupingsets_16366510952463710337 + │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] + └── EvalScalar + ├── output columns: [a (#8), b (#9), c (#10)] + ├── expressions: [TRY_CAST(group_item (#1) AS UInt8 NULL), TRY_CAST(group_item (#2) AS UInt8 NULL), TRY_CAST(group_item (#3) AS UInt8 NULL)] + ├── estimated rows: 1.00 + └── AggregateFinal + ├── output columns: [a (#1), b (#2), c (#3)] + ├── group by: [a, b, c] + ├── aggregate functions: [] + ├── estimated rows: 1.00 + └── AggregatePartial + ├── group by: [a, b, c] + ├── aggregate functions: [] + ├── estimated rows: 1.00 + └── CTEConsumer + ├── cte_name: cte_groupingsets_16366510952463710337 + └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] From 2c6399afbe65a84321c8e07208f23f0c0c1309bd Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Sat, 26 Jul 2025 07:34:36 +0800 Subject: [PATCH 75/80] Merge --- src/query/sql/src/planner/optimizer/optimizer_context.rs | 8 ++++---- .../rule/agg_rules/rule_grouping_sets_to_union.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizer_context.rs b/src/query/sql/src/planner/optimizer/optimizer_context.rs index 82acdb2ccca1e..28e0e9071c26d 100644 --- a/src/query/sql/src/planner/optimizer/optimizer_context.rs +++ b/src/query/sql/src/planner/optimizer/optimizer_context.rs @@ -154,10 +154,10 @@ impl OptimizerContext { pub fn is_optimizer_disabled(self: &Arc, name: &str) -> bool { let settings = self.get_table_ctx().get_settings(); - if !settings.get_grouping_sets_to_union().unwrap_or_default() { - if name == RuleID::GroupingSetsToUnion.to_string() { - return true; - } + if !settings.get_grouping_sets_to_union().unwrap_or_default() + && name == RuleID::GroupingSetsToUnion.to_string() + { + return true; } match settings.get_optimizer_skip_list() { diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs index 3f77cb857ec33..68471cc55bfe8 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs @@ -97,7 +97,7 @@ impl Rule for RuleGroupingSetsToUnion { } let agg_input = s_expr.child(0)?.child(0)?; - let agg_input_columns: Vec = RelExpr::with_s_expr(&agg_input) + let agg_input_columns: Vec = RelExpr::with_s_expr(agg_input) .derive_relational_prop()? .output_columns .iter() @@ -105,7 +105,7 @@ impl Rule for RuleGroupingSetsToUnion { .collect(); if let Some(grouping_sets) = &agg.grouping_sets { - if grouping_sets.sets.len() >= 1 { + if !grouping_sets.sets.is_empty() { let mut children = Vec::with_capacity(grouping_sets.sets.len()); let mut hasher = DefaultHasher::new(); From 8ff3944371a446864cfbc92b144db429f843873c Mon Sep 17 00:00:00 2001 From: sky <3374614481@qq.com> Date: Sun, 27 Jul 2025 21:11:47 +0800 Subject: [PATCH 76/80] add channel size config --- .../src/pipelines/builders/builder_materialized_cte.rs | 6 +++--- src/query/service/src/sessions/query_ctx.rs | 7 ++++++- src/query/sql/src/executor/physical_plan_visitor.rs | 1 + .../executor/physical_plans/physical_materialized_cte.rs | 2 ++ .../src/planner/binder/bind_table_reference/bind_cte.rs | 3 ++- src/query/sql/src/planner/plans/materialized_cte.rs | 8 +++++++- 6 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs index 8f64bb253770e..3f54dd3bc8354 100644 --- a/src/query/service/src/pipelines/builders/builder_materialized_cte.rs +++ b/src/query/service/src/pipelines/builders/builder_materialized_cte.rs @@ -31,9 +31,9 @@ impl PipelineBuilder { )?; } self.main_pipeline.try_resize(1)?; - let tx = self - .ctx - .get_materialized_cte_senders(&cte.cte_name, cte.ref_count); + let tx = + self.ctx + .get_materialized_cte_senders(&cte.cte_name, cte.ref_count, cte.channel_size); self.main_pipeline .add_sink(|input| MaterializedCteSink::create(input, tx.clone())) } diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index eb3432e4bfbbc..5dba92497904c 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -590,11 +590,16 @@ impl QueryContext { &self, cte_name: &str, cte_ref_count: usize, + channel_size: Option, ) -> Vec> { let mut senders = vec![]; let mut receivers = vec![]; for _ in 0..cte_ref_count { - let (sender, receiver) = async_channel::unbounded(); + let (sender, receiver) = if let Some(channel_size) = channel_size { + async_channel::bounded(channel_size) + } else { + async_channel::unbounded() + }; senders.push(sender); receivers.push(receiver); } diff --git a/src/query/sql/src/executor/physical_plan_visitor.rs b/src/query/sql/src/executor/physical_plan_visitor.rs index 8fa7bec68ab73..adc5861971286 100644 --- a/src/query/sql/src/executor/physical_plan_visitor.rs +++ b/src/query/sql/src/executor/physical_plan_visitor.rs @@ -659,6 +659,7 @@ pub trait PhysicalPlanReplacer { cte_name: plan.cte_name.clone(), cte_output_columns: plan.cte_output_columns.clone(), ref_count: plan.ref_count, + channel_size: plan.channel_size, }))) } diff --git a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs index 83064f4fdd88f..6cb3049444cbb 100644 --- a/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs +++ b/src/query/sql/src/executor/physical_plans/physical_materialized_cte.rs @@ -32,6 +32,7 @@ pub struct MaterializedCTE { pub cte_name: String, pub cte_output_columns: Option>, pub ref_count: usize, + pub channel_size: Option, } impl MaterializedCTE { @@ -62,6 +63,7 @@ impl PhysicalPlanBuilder { cte_name: materialized_cte.cte_name.clone(), cte_output_columns: materialized_cte.cte_output_columns.clone(), ref_count: materialized_cte.ref_count, + channel_size: materialized_cte.channel_size, }))) } } diff --git a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs index e9845aa9b6a60..224bcb5dcf72a 100644 --- a/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs +++ b/src/query/sql/src/planner/binder/bind_table_reference/bind_cte.rs @@ -169,7 +169,8 @@ impl Binder { let (s_expr, bind_context) = self.bind_cte_definition(&cte_name, &cte_context.cte_map, &cte.query)?; - let materialized_cte = MaterializedCTE::new(cte_name, Some(bind_context.columns)); + let materialized_cte = + MaterializedCTE::new(cte_name, Some(bind_context.columns), None); let materialized_cte = SExpr::create_unary(materialized_cte, s_expr); let sequence = Sequence {}; current_expr = SExpr::create_binary(sequence, materialized_cte, current_expr); diff --git a/src/query/sql/src/planner/plans/materialized_cte.rs b/src/query/sql/src/planner/plans/materialized_cte.rs index 002fa24c96110..a1cec441397a5 100644 --- a/src/query/sql/src/planner/plans/materialized_cte.rs +++ b/src/query/sql/src/planner/plans/materialized_cte.rs @@ -32,15 +32,21 @@ pub struct MaterializedCTE { pub cte_name: String, pub cte_output_columns: Option>, pub ref_count: usize, + pub channel_size: Option, } impl MaterializedCTE { - pub fn new(cte_name: String, output_columns: Option>) -> Self { + pub fn new( + cte_name: String, + output_columns: Option>, + channel_size: Option, + ) -> Self { Self { cte_name, cte_output_columns: output_columns, // ref_count is set to 0 by default, will be updated by CleanupUnusedCTEOptimizer ref_count: 0, + channel_size, } } } From c2eacfae34604b3572de2b44cce54aaf0382759b Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Sun, 27 Jul 2025 21:35:24 +0800 Subject: [PATCH 77/80] Merge --- .../rule/agg_rules/rule_grouping_sets_to_union.rs | 4 ++-- .../standalone/explain/explain_grouping_sets.test | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs index 5cd2e087ce4fa..47a789d7cf2b9 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs @@ -30,10 +30,10 @@ use crate::optimizer::optimizers::rule::TransformResult; use crate::plans::walk_expr_mut; use crate::plans::Aggregate; use crate::plans::AggregateMode; -use crate::plans::CTEConsumer; use crate::plans::CastExpr; use crate::plans::ConstantExpr; use crate::plans::EvalScalar; +use crate::plans::MaterializeCTERef; use crate::plans::MaterializedCTE; use crate::plans::RelOp; use crate::plans::Sequence; @@ -117,7 +117,7 @@ impl Rule for RuleGroupingSetsToUnion { agg_input.clone(), ); - let cte_consumer = SExpr::create_leaf(CTEConsumer { + let cte_consumer = SExpr::create_leaf(MaterializeCTERef { cte_name: temp_cte_name, output_columns: agg_input_columns.clone(), def: agg_input.clone(), diff --git a/tests/sqllogictests/suites/mode/standalone/explain/explain_grouping_sets.test b/tests/sqllogictests/suites/mode/standalone/explain/explain_grouping_sets.test index 2ab12c830786d..6961b77298dfd 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/explain_grouping_sets.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/explain_grouping_sets.test @@ -127,7 +127,7 @@ Sequence │ │ │ │ │ │ ├── group by: [a] │ │ │ │ │ │ ├── aggregate functions: [] │ │ │ │ │ │ ├── estimated rows: 1.00 - │ │ │ │ │ │ └── CTEConsumer + │ │ │ │ │ │ └── MaterializeCTERef │ │ │ │ │ │ ├── cte_name: cte_groupingsets_16366510952463710337 │ │ │ │ │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] │ │ │ │ │ └── EvalScalar @@ -143,7 +143,7 @@ Sequence │ │ │ │ │ ├── group by: [b] │ │ │ │ │ ├── aggregate functions: [] │ │ │ │ │ ├── estimated rows: 1.00 - │ │ │ │ │ └── CTEConsumer + │ │ │ │ │ └── MaterializeCTERef │ │ │ │ │ ├── cte_name: cte_groupingsets_16366510952463710337 │ │ │ │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] │ │ │ │ └── EvalScalar @@ -159,7 +159,7 @@ Sequence │ │ │ │ ├── group by: [c] │ │ │ │ ├── aggregate functions: [] │ │ │ │ ├── estimated rows: 1.00 - │ │ │ │ └── CTEConsumer + │ │ │ │ └── MaterializeCTERef │ │ │ │ ├── cte_name: cte_groupingsets_16366510952463710337 │ │ │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] │ │ │ └── EvalScalar @@ -175,7 +175,7 @@ Sequence │ │ │ ├── group by: [a, b] │ │ │ ├── aggregate functions: [] │ │ │ ├── estimated rows: 1.00 - │ │ │ └── CTEConsumer + │ │ │ └── MaterializeCTERef │ │ │ ├── cte_name: cte_groupingsets_16366510952463710337 │ │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] │ │ └── EvalScalar @@ -191,7 +191,7 @@ Sequence │ │ ├── group by: [a, c] │ │ ├── aggregate functions: [] │ │ ├── estimated rows: 1.00 - │ │ └── CTEConsumer + │ │ └── MaterializeCTERef │ │ ├── cte_name: cte_groupingsets_16366510952463710337 │ │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] │ └── EvalScalar @@ -207,7 +207,7 @@ Sequence │ ├── group by: [b, c] │ ├── aggregate functions: [] │ ├── estimated rows: 1.00 - │ └── CTEConsumer + │ └── MaterializeCTERef │ ├── cte_name: cte_groupingsets_16366510952463710337 │ └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] └── EvalScalar @@ -223,6 +223,6 @@ Sequence ├── group by: [a, b, c] ├── aggregate functions: [] ├── estimated rows: 1.00 - └── CTEConsumer + └── MaterializeCTERef ├── cte_name: cte_groupingsets_16366510952463710337 └── cte_schema: [number (#0), a (#1), b (#2), c (#3)] From a71c994505158ffc79ec287a95f4dd902bcf344c Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Sun, 27 Jul 2025 22:20:07 +0800 Subject: [PATCH 78/80] Merge --- .../service/src/pipelines/builders/builder_union_all.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/service/src/pipelines/builders/builder_union_all.rs b/src/query/service/src/pipelines/builders/builder_union_all.rs index 757347d1f7c70..5c5b09fdaea01 100644 --- a/src/query/service/src/pipelines/builders/builder_union_all.rs +++ b/src/query/service/src/pipelines/builders/builder_union_all.rs @@ -42,7 +42,10 @@ impl PipelineBuilder { self.main_pipeline.extend_sinks(left_sinks); self.main_pipeline.extend_sinks(right_sinks); - match self.ctx.get_settings().get_enable_parallel_union_all()? { + let enable_parallel_union_all = self.ctx.get_settings().get_enable_parallel_union_all()? + || self.ctx.get_settings().get_grouping_sets_to_union()?; + + match enable_parallel_union_all { true => self.main_pipeline.resize(outputs, false), false => self.main_pipeline.sequence_group(sequence_groups, outputs), } From da58283d31d722cb9890228500a75b12d1a9a177 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Sun, 27 Jul 2025 23:19:17 +0800 Subject: [PATCH 79/80] update --- .../service/src/pipelines/builders/builder_union_all.rs | 5 ++--- .../optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_union_all.rs b/src/query/service/src/pipelines/builders/builder_union_all.rs index 5c5b09fdaea01..9b2b7568945a4 100644 --- a/src/query/service/src/pipelines/builders/builder_union_all.rs +++ b/src/query/service/src/pipelines/builders/builder_union_all.rs @@ -42,9 +42,8 @@ impl PipelineBuilder { self.main_pipeline.extend_sinks(left_sinks); self.main_pipeline.extend_sinks(right_sinks); - let enable_parallel_union_all = self.ctx.get_settings().get_enable_parallel_union_all()? - || self.ctx.get_settings().get_grouping_sets_to_union()?; - + let enable_parallel_union_all = self.ctx.get_settings().get_enable_parallel_union_all()?; + || self.ctx.get_settings().get_grouping_sets_to_union()?; match enable_parallel_union_all { true => self.main_pipeline.resize(outputs, false), false => self.main_pipeline.sequence_group(sequence_groups, outputs), diff --git a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs index 47a789d7cf2b9..301b5823907eb 100644 --- a/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs +++ b/src/query/sql/src/planner/optimizer/optimizers/rule/agg_rules/rule_grouping_sets_to_union.rs @@ -112,6 +112,7 @@ impl Rule for RuleGroupingSetsToUnion { agg.grouping_sets.hash(&mut hasher); let hash = hasher.finish(); let temp_cte_name = format!("cte_groupingsets_{hash}"); + let cte_materialized_sexpr = SExpr::create_unary( MaterializedCTE::new(temp_cte_name.clone(), None, Some(1)), agg_input.clone(), From 16bb07790f1e3149bb95b6427e27d2607526b017 Mon Sep 17 00:00:00 2001 From: sundyli <543950155@qq.com> Date: Mon, 28 Jul 2025 09:15:28 +0800 Subject: [PATCH 80/80] update --- src/query/service/src/pipelines/builders/builder_union_all.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/service/src/pipelines/builders/builder_union_all.rs b/src/query/service/src/pipelines/builders/builder_union_all.rs index 9b2b7568945a4..ba2fee9af90c6 100644 --- a/src/query/service/src/pipelines/builders/builder_union_all.rs +++ b/src/query/service/src/pipelines/builders/builder_union_all.rs @@ -42,8 +42,8 @@ impl PipelineBuilder { self.main_pipeline.extend_sinks(left_sinks); self.main_pipeline.extend_sinks(right_sinks); - let enable_parallel_union_all = self.ctx.get_settings().get_enable_parallel_union_all()?; - || self.ctx.get_settings().get_grouping_sets_to_union()?; + let enable_parallel_union_all = self.ctx.get_settings().get_enable_parallel_union_all()? + || self.ctx.get_settings().get_grouping_sets_to_union()?; match enable_parallel_union_all { true => self.main_pipeline.resize(outputs, false), false => self.main_pipeline.sequence_group(sequence_groups, outputs),