Skip to content

Commit 018b40d

Browse files
julian-elasticjulian-elastic
committed
Switch to storing the filter in Join
2 parents 6e6e28e + a59d0de commit 018b40d

File tree

4 files changed

+42
-30
lines changed

4 files changed

+42
-30
lines changed

‎docs/changelog/132889.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132889
2+
summary: Improve Expanding Lookup Join performance by pushing a filter to the lookup
3+
join
4+
area: "ES|QL, Performance"
5+
type: enhancement
6+
issues: []

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/InlineJoin.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@
77

88
package org.elasticsearch.xpack.esql.plan.logical.join;
99

10+
import org.elasticsearch.TransportVersions;
1011
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1112
import org.elasticsearch.common.io.stream.StreamInput;
1213
import org.elasticsearch.compute.data.Block;
1314
import org.elasticsearch.compute.data.BlockUtils;
1415
import org.elasticsearch.xpack.esql.core.expression.Alias;
1516
import org.elasticsearch.xpack.esql.core.expression.Attribute;
17+
import org.elasticsearch.xpack.esql.core.expression.Expression;
1618
import org.elasticsearch.xpack.esql.core.expression.Literal;
1719
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
1820
import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -152,7 +154,7 @@ public InlineJoin(
152154
List<Attribute> leftFields,
153155
List<Attribute> rightFields
154156
) {
155-
super(source, left, right, type, matchFields, leftFields, rightFields);
157+
super(source, left, right, type, matchFields, leftFields, rightFields, false, null);
156158
}
157159

158160
private static InlineJoin readFrom(StreamInput in) throws IOException {
@@ -161,6 +163,9 @@ private static InlineJoin readFrom(StreamInput in) throws IOException {
161163
LogicalPlan left = in.readNamedWriteable(LogicalPlan.class);
162164
LogicalPlan right = in.readNamedWriteable(LogicalPlan.class);
163165
JoinConfig config = new JoinConfig(in);
166+
if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_LOOKUP_JOIN_PRE_JOIN_FILTER)) {
167+
Expression ignored = in.readOptionalNamedWriteable(Expression.class);
168+
}
164169
return new InlineJoin(source, left, replaceStub(left, right), config);
165170
}
166171

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,10 +120,14 @@ public Join(
120120
JoinType type,
121121
List<Attribute> matchFields,
122122
List<Attribute> leftFields,
123-
List<Attribute> rightFields
123+
List<Attribute> rightFields,
124+
boolean isRemote,
125+
Expression optionalRightHandFilters
124126
) {
125127
super(source, left, right);
126128
this.config = new JoinConfig(type, matchFields, leftFields, rightFields);
129+
this.isRemote = isRemote;
130+
this.optionalRightHandFilters = optionalRightHandFilters;
127131
}
128132

129133
public Join(StreamInput in) throws IOException {
@@ -170,7 +174,9 @@ protected NodeInfo<Join> info() {
170174
config.type(),
171175
config.matchFields(),
172176
config.leftFields(),
173-
config.rightFields()
177+
config.rightFields(),
178+
isRemote(),
179+
optionalRightHandFilters()
174180
);
175181
}
176182

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java

Lines changed: 22 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1923,10 +1923,9 @@ public void testDoNotPushDownIsNullFilterPastLookupJoin() {
19231923
* uages{f}#10 AS language_code#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]]
19241924
* \_Limit[1000[INTEGER],false]
19251925
* \_Filter[language_name{f}#19 > a[KEYWORD]]
1926-
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18]]
1926+
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18],false,language_name{f}#19 > a[KEYWORD]]
19271927
* |_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
1928-
* \_Filter[language_name{f}#19 > a[KEYWORD]]
1929-
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
1928+
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
19301929
*/
19311930
public void testPushDownGreaterThanFilterPastLookupJoin() {
19321931
var plan = plan("""
@@ -1940,8 +1939,8 @@ public void testPushDownGreaterThanFilterPastLookupJoin() {
19401939
var limit = as(project.child(), Limit.class);
19411940
var filter = as(limit.child(), Filter.class);
19421941
var join = as(filter.child(), Join.class);
1943-
var right = as(join.right(), Filter.class);
1944-
assertThat(right.condition().toString(), is("language_name > \"a\""));
1942+
var right = as(join.right(), EsRelation.class);
1943+
assertThat(join.optionalRightHandFilters().toString(), is("language_name > \"a\""));
19451944
}
19461945

19471946
/**
@@ -1970,13 +1969,12 @@ public void testDoNotPushDownCoalesceFilterPastLookupJoin() {
19701969

19711970
/**
19721971
* Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang
1973-
* uages{f}#10 AS language_code#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]]
1972+
*uages{f}#10 AS language_code#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]]
19741973
* \_Limit[1000[INTEGER],false]
19751974
* \_Filter[ISNOTNULL(language_name{f}#19)]
1976-
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18]]
1975+
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18],false,ISNOTNULL(language_name{f}#19)]
19771976
* |_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
1978-
* \_Filter[ISNOTNULL(language_name{f}#19)]
1979-
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
1977+
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
19801978
*/
19811979
public void testPushDownIsNotNullFilterPastLookupJoin() {
19821980
var plan = plan("""
@@ -1990,8 +1988,8 @@ public void testPushDownIsNotNullFilterPastLookupJoin() {
19901988
var limit = as(project.child(), Limit.class);
19911989
var filter = as(limit.child(), Filter.class);
19921990
var join = as(filter.child(), Join.class);
1993-
var right = as(join.right(), Filter.class);
1994-
assertThat(right.condition().toString(), is("language_name IS NOT NULL"));
1991+
var right = as(join.right(), EsRelation.class);
1992+
assertThat(join.optionalRightHandFilters().toString(), is("language_name IS NOT NULL"));
19951993
}
19961994

19971995
/**
@@ -7150,14 +7148,13 @@ public void testLookupJoinPushDownFilterOnLeftSideField() {
71507148
* Expects
71517149
*
71527150
* <pre>{@code
7153-
* Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16,
7154-
* languages{f}#10 AS language_code#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]]
7151+
* Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang
7152+
* uages{f}#10 AS language_code#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]]
71557153
* \_Limit[1000[INTEGER],false]
7156-
* \_Filter[language_name{f}#19 == [45 6e 67 6c 69 73 68][KEYWORD]]
7157-
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18]]
7154+
* \_Filter[language_name{f}#19 == English[KEYWORD]]
7155+
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18],false,language_name{f}#19 == English[KEYWORD]]
71587156
* |_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
7159-
* \_Filter[language_name{f}#19 == English[KEYWORD]]
7160-
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
7157+
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
71617158
* }</pre>
71627159
*/
71637160
public void testLookupJoinPushDownDisabledForLookupField() {
@@ -7186,9 +7183,8 @@ public void testLookupJoinPushDownDisabledForLookupField() {
71867183
assertThat(join.config().type(), equalTo(JoinTypes.LEFT));
71877184

71887185
var leftRel = as(join.left(), EsRelation.class);
7189-
var filterRight = as(join.right(), Filter.class);
7190-
assertEquals("language_name == \"English\"", filterRight.condition().toString());
7191-
var joinRightEsRelation = as(filterRight.child(), EsRelation.class);
7186+
assertEquals("language_name == \"English\"", join.optionalRightHandFilters().toString());
7187+
var joinRightEsRelation = as(join.right(), EsRelation.class);
71927188

71937189
}
71947190

@@ -7198,11 +7194,11 @@ public void testLookupJoinPushDownDisabledForLookupField() {
71987194
* Expects
71997195
*
72007196
* <pre>{@code
7201-
* Project[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17,
7202-
* languages{f}#11 AS language_code#4, last_name{f}#12, long_noidx{f}#18, salary{f}#13, language_name{f}#20]]
7197+
* Project[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan
7198+
* guages{f}#11 AS language_code#4, last_name{f}#12, long_noidx{f}#18, salary{f}#13, language_name{f}#20]]
72037199
* \_Limit[1000[INTEGER],false]
7204-
* \_Filter[language_name{f}#20 == [45 6e 67 6c 69 73 68][KEYWORD]]
7205-
* \_Join[LEFT,[languages{f}#11],[languages{f}#11],[language_code{f}#19]]
7200+
* \_Filter[language_name{f}#20 == English[KEYWORD]]
7201+
* \_Join[LEFT,[languages{f}#11],[languages{f}#11],[language_code{f}#19],false,language_name{f}#20 == English[KEYWORD]]
72067202
* |_Filter[emp_no{f}#8 > 1[INTEGER]]
72077203
* | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..]
72087204
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20]
@@ -7242,9 +7238,8 @@ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightFiel
72427238
assertThat(literal.value(), equalTo(1));
72437239

72447240
var leftRel = as(filter.child(), EsRelation.class);
7245-
var filterRight = as(join.right(), Filter.class);
7246-
assertEquals("language_name == \"English\"", filterRight.condition().toString());
7247-
var rightRel = as(filterRight.child(), EsRelation.class);
7241+
assertEquals("language_name == \"English\"", join.optionalRightHandFilters().toString());
7242+
var rightRel = as(join.right(), EsRelation.class);
72487243
}
72497244

72507245
/**

0 commit comments

Comments
 (0)