Skip to content

Commit ee11e41

Browse files
julian-elasticjulian-elastic
committed
Get basic case with translatable filters to work
1 parent 6d71404 commit ee11e41

File tree

11 files changed

+428
-87
lines changed

11 files changed

+428
-87
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5050,3 +5050,181 @@ id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:int
50505050
15 | null | bar2 | null | null
50515051
null | null | plugh | null | null
50525052
;
5053+
5054+
5055+
lookupJoinWithPushableFilterOnLeft
5056+
required_capability: join_lookup_v12
5057+
required_capability: lookup_join_on_multiple_fields
5058+
5059+
FROM multi_column_joinable
5060+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5061+
| WHERE other2 > 5000
5062+
| KEEP id_int, name_str, extra1, other1, other2
5063+
| SORT id_int, name_str, extra1, other1, other2
5064+
| LIMIT 20
5065+
;
5066+
5067+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5068+
4 | David | qux | zeta | 6000
5069+
5 | Eve | quux | eta | 7000
5070+
5 | Eve | quux | theta | 8000
5071+
6 | null | corge | iota | 9000
5072+
7 | Grace | grault | kappa | 10000
5073+
8 | Hank | garply | lambda | 11000
5074+
12 | Liam | xyzzy | nu | 13000
5075+
13 | Mia | thud | xi | 14000
5076+
14 | Nina | foo2 | omicron | 15000
5077+
;
5078+
5079+
lookupJoinWithTwoPushableFiltersOnLeft
5080+
required_capability: join_lookup_v12
5081+
required_capability: lookup_join_on_multiple_fields
5082+
5083+
FROM multi_column_joinable
5084+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5085+
| WHERE other2 > 5000
5086+
| WHERE other1 like "*ta"
5087+
| KEEP id_int, name_str, extra1, other1, other2
5088+
| SORT id_int, name_str, extra1, other1, other2
5089+
| LIMIT 20
5090+
;
5091+
5092+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5093+
4 | David | qux | zeta | 6000
5094+
5 | Eve | quux | eta | 7000
5095+
5 | Eve | quux | theta | 8000
5096+
6 | null | corge | iota | 9000
5097+
;
5098+
5099+
lookupJoinWithMixLeftAndRightFilters
5100+
required_capability: join_lookup_v12
5101+
required_capability: lookup_join_on_multiple_fields
5102+
5103+
FROM multi_column_joinable
5104+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5105+
| WHERE other2 > 5000 AND (extra1 == "qux" OR extra1 == "foo2") AND other1 like ("*ta", "*ron")
5106+
| KEEP id_int, name_str, extra1, other1, other2
5107+
| SORT id_int, name_str, extra1, other1, other2
5108+
| LIMIT 20
5109+
;
5110+
5111+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5112+
4 | David | qux | zeta | 6000
5113+
14 | Nina | foo2 | omicron | 15000
5114+
;
5115+
5116+
lookupJoinWithMixLeftAndRightFiltersNotPushableToLucene
5117+
required_capability: join_lookup_v12
5118+
required_capability: lookup_join_on_multiple_fields
5119+
5120+
FROM multi_column_joinable
5121+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5122+
| WHERE ABS(other2) > 5000 AND (extra1 == "qux" OR extra1 == "foo2") AND other1 like ("*ta", "*ron")
5123+
| KEEP id_int, name_str, extra1, other1, other2
5124+
| SORT id_int, name_str, extra1, other1, other2
5125+
| LIMIT 20
5126+
;
5127+
5128+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5129+
4 | David | qux | zeta | 6000
5130+
14 | Nina | foo2 | omicron | 15000
5131+
;
5132+
5133+
5134+
lookupJoinWithMixJoinAndNonJoinColumnsNotPushable
5135+
required_capability: join_lookup_v12
5136+
required_capability: lookup_join_on_multiple_fields
5137+
5138+
FROM multi_column_joinable
5139+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5140+
| WHERE ABS(other2) > id_int + 5000
5141+
| KEEP id_int, name_str, extra1, other1, other2
5142+
| SORT id_int, name_str, extra1, other1, other2
5143+
| LIMIT 20
5144+
;
5145+
5146+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5147+
4 | David | qux | zeta | 6000
5148+
5 | Eve | quux | eta | 7000
5149+
5 | Eve | quux | theta | 8000
5150+
6 | null | corge | iota | 9000
5151+
7 | Grace | grault | kappa | 10000
5152+
8 | Hank | garply | lambda | 11000
5153+
12 | Liam | xyzzy | nu | 13000
5154+
13 | Mia | thud | xi | 14000
5155+
14 | Nina | foo2 | omicron | 15000
5156+
;
5157+
5158+
5159+
lookupJoinWithMixJoinAndNonJoinColumnsPushable
5160+
required_capability: join_lookup_v12
5161+
required_capability: lookup_join_on_multiple_fields
5162+
5163+
FROM multi_column_joinable
5164+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5165+
| WHERE other2 > id_int + 5000
5166+
| KEEP id_int, name_str, extra1, other1, other2
5167+
| SORT id_int, name_str, extra1, other1, other2
5168+
| LIMIT 20
5169+
;
5170+
5171+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5172+
4 | David | qux | zeta | 6000
5173+
5 | Eve | quux | eta | 7000
5174+
5 | Eve | quux | theta | 8000
5175+
6 | null | corge | iota | 9000
5176+
7 | Grace | grault | kappa | 10000
5177+
8 | Hank | garply | lambda | 11000
5178+
12 | Liam | xyzzy | nu | 13000
5179+
13 | Mia | thud | xi | 14000
5180+
14 | Nina | foo2 | omicron | 15000
5181+
;
5182+
5183+
lookupJoinWithJoinAttrFilter
5184+
required_capability: join_lookup_v12
5185+
required_capability: lookup_join_on_multiple_fields
5186+
5187+
FROM multi_column_joinable
5188+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5189+
| WHERE id_int > 7
5190+
| KEEP id_int, name_str, extra1, other1, other2
5191+
| SORT id_int, name_str, extra1, other1, other2
5192+
| LIMIT 20
5193+
;
5194+
5195+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5196+
8 | Hank | garply | lambda | 11000
5197+
9 | null | waldo | null | null
5198+
10 | null | fred | null | null
5199+
12 | Liam | xyzzy | nu | 13000
5200+
13 | Mia | thud | xi | 14000
5201+
14 | Nina | foo2 | omicron | 15000
5202+
15 | null | bar2 | null | null
5203+
;
5204+
5205+
5206+
lookupJoinWithExpressionOfOtherFields
5207+
required_capability: join_lookup_v12
5208+
required_capability: lookup_join_on_multiple_fields
5209+
5210+
FROM multi_column_joinable
5211+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5212+
| WHERE ABS(other2) > LENGTH(other1)*1000 + 2000
5213+
| KEEP id_int, name_str, extra1, other1, other2
5214+
| SORT id_int, name_str, extra1, other1, other2
5215+
| LIMIT 20
5216+
/*
5217+
NEED TO DEBUG WHY THE FILTER IS NOT PUSHED TO THE RIGHT SIDE OF THE LOOKUP JOIN
5218+
*/
5219+
;
5220+
5221+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5222+
5 | Eve | quux | eta | 7000
5223+
5 | Eve | quux | theta | 8000
5224+
6 | null | corge | iota | 9000
5225+
7 | Grace | grault | kappa | 10000
5226+
8 | Hank | garply | lambda | 11000
5227+
12 | Liam | xyzzy | nu | 13000
5228+
13 | Mia | thud | xi | 14000
5229+
14 | Nina | foo2 | omicron | 15000
5230+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator;
3939
import org.elasticsearch.compute.operator.Driver;
4040
import org.elasticsearch.compute.operator.DriverContext;
41+
import org.elasticsearch.compute.operator.FilterOperator;
4142
import org.elasticsearch.compute.operator.Operator;
4243
import org.elasticsearch.compute.operator.OutputOperator;
4344
import org.elasticsearch.compute.operator.ProjectOperator;
@@ -73,10 +74,15 @@
7374
import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
7475
import org.elasticsearch.xpack.esql.core.expression.Alias;
7576
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
77+
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
7678
import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
7779
import org.elasticsearch.xpack.esql.core.tree.Source;
7880
import org.elasticsearch.xpack.esql.core.type.DataType;
81+
import org.elasticsearch.xpack.esql.core.type.EsField;
82+
import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
83+
import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
7984
import org.elasticsearch.xpack.esql.planner.EsPhysicalOperationProviders;
85+
import org.elasticsearch.xpack.esql.planner.Layout;
8086
import org.elasticsearch.xpack.esql.planner.PlannerUtils;
8187
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
8288

@@ -355,13 +361,36 @@ private void doLookup(T request, CancellableTask task, ActionListener<List<Page>
355361
warnings
356362
);
357363
releasables.add(queryOperator);
358-
364+
Layout.Builder builder = new Layout.Builder();
365+
// append the docsIds and positions to the layout
366+
builder.append(
367+
// this looks wrong, what is the datatype for the Docs? It says DocVector but it is not a DataType
368+
new FieldAttribute(Source.EMPTY, "Docs", new EsField("Docs", DataType.DOC_DATA_TYPE, Collections.emptyMap(), false))
369+
);
370+
builder.append(
371+
new FieldAttribute(Source.EMPTY, "Positions", new EsField("Positions", DataType.INTEGER, Collections.emptyMap(), false))
372+
);
359373
List<Operator> operators = new ArrayList<>();
360374
if (request.extractFields.isEmpty() == false) {
361375
var extractFieldsOperator = extractFieldsOperator(shardContext.context, driverContext, request.extractFields);
376+
builder.append(request.extractFields);
362377
releasables.add(extractFieldsOperator);
363378
operators.add(extractFieldsOperator);
364379
}
380+
if (queryList instanceof PostJoinFilterable postJoinFilterable) {
381+
FilterExec filterExec = postJoinFilterable.getPostJoinFilter();
382+
Operator inputOperator;
383+
if (operators.isEmpty() == false) {
384+
inputOperator = operators.getLast();
385+
} else {
386+
inputOperator = queryOperator;
387+
}
388+
Operator postJoinFilter = filterExecOperator(filterExec, inputOperator, shardContext.context, driverContext, builder);
389+
if (postJoinFilter != null) {
390+
releasables.add(postJoinFilter);
391+
operators.add(postJoinFilter);
392+
}
393+
}
365394
operators.add(finishPages);
366395

367396
/*
@@ -423,6 +452,27 @@ public void onFailure(Exception e) {
423452
}
424453
}
425454

455+
private Operator filterExecOperator(
456+
FilterExec filterExec,
457+
Operator inputOperator, // not needed?
458+
EsPhysicalOperationProviders.ShardContext shardContext,
459+
DriverContext driverContext,
460+
Layout.Builder builder
461+
) {
462+
if (filterExec == null) {
463+
return null;
464+
}
465+
466+
var evaluatorFactory = EvalMapper.toEvaluator(
467+
FoldContext.small()/*is this correct*/,
468+
filterExec.condition(),
469+
builder.build(),
470+
List.of(shardContext)
471+
);
472+
var filterOperatorFactory = new FilterOperator.FilterOperatorFactory(evaluatorFactory);
473+
return filterOperatorFactory.get(driverContext);
474+
}
475+
426476
private static Operator extractFieldsOperator(
427477
EsPhysicalOperationProviders.ShardContext shardContext,
428478
DriverContext driverContext,

0 commit comments

Comments
 (0)