Skip to content

Commit 06acc78

Browse files
0aix0aix
authored
Support Multiple Lateral Views (#106)
1 parent 22ec609 commit 06acc78

File tree

7 files changed

+221
-35
lines changed

7 files changed

+221
-35
lines changed

‎src/main/java/com/yahoo/bullet/query/tablefunctions/LateralView.java

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import com.yahoo.bullet.querying.tablefunctors.TableFunctor;
1010
import lombok.Getter;
1111

12+
import java.util.Collections;
13+
import java.util.List;
1214
import java.util.Objects;
1315

1416
/**
@@ -18,16 +20,25 @@
1820
public class LateralView extends TableFunction {
1921
private static final long serialVersionUID = -8238108616312386350L;
2022

21-
private final TableFunction tableFunction;
23+
private final List<TableFunction> tableFunctions;
2224

2325
/**
24-
* Constructor that creates a LATERAL VIEW table function.
26+
* Constructor that creates a LATERAL VIEW from a {@link List} of {@link TableFunction}.
2527
*
26-
* @param tableFunction The non-null table function to take a lateral view of.
28+
* @param tableFunctions The non-null list of table functions to chain lateral views.
2729
*/
28-
public LateralView(TableFunction tableFunction) {
30+
public LateralView(List<TableFunction> tableFunctions) {
2931
super(TableFunctionType.LATERAL_VIEW);
30-
this.tableFunction = Objects.requireNonNull(tableFunction);
32+
this.tableFunctions = Objects.requireNonNull(tableFunctions);
33+
}
34+
35+
/**
36+
* Constructor that creates a LATERAL VIEW from a {@link TableFunction}.
37+
*
38+
* @param tableFunction The non-null table function to apply a lateral view.
39+
*/
40+
public LateralView(TableFunction tableFunction) {
41+
this(Collections.singletonList(Objects.requireNonNull(tableFunction)));
3142
}
3243

3344
@Override
@@ -37,6 +48,6 @@ public TableFunctor getTableFunctor() {
3748

3849
@Override
3950
public String toString() {
40-
return "{type: " + type + ", tableFunction: " + tableFunction + "}";
51+
return "{type: " + type + ", tableFunctions: " + tableFunctions + "}";
4152
}
4253
}

‎src/main/java/com/yahoo/bullet/querying/tablefunctors/LateralViewBulletRecord.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ public Serializable getAndRemove(String field) {
8484
@Override
8585
public BulletRecord remove(String field) {
8686
culledFields.add(field);
87-
return topRecord.remove(field);
87+
topRecord.remove(field);
88+
return this;
8889
}
8990

9091
@Override

‎src/main/java/com/yahoo/bullet/querying/tablefunctors/LateralViewFunctor.java

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,44 @@
66
package com.yahoo.bullet.querying.tablefunctors;
77

88
import com.yahoo.bullet.query.tablefunctions.LateralView;
9+
import com.yahoo.bullet.query.tablefunctions.TableFunction;
910
import com.yahoo.bullet.record.BulletRecord;
1011
import com.yahoo.bullet.record.BulletRecordProvider;
1112

1213
import java.util.List;
1314
import java.util.stream.Collectors;
15+
import java.util.stream.Stream;
1416

1517
/**
1618
* A table functor that joins the generated records of the nested table functor with the original input record.
1719
*/
1820
public class LateralViewFunctor extends TableFunctor {
1921
private static final long serialVersionUID = 1017033253024183470L;
2022

21-
final TableFunctor tableFunctor;
23+
final List<TableFunctor> tableFunctors;
2224

2325
/**
2426
* Constructor that creates a lateral view table functor from a {@link LateralView}.
2527
*
2628
* @param lateralView The lateral view table function to construct the table functor from.
2729
*/
2830
public LateralViewFunctor(LateralView lateralView) {
29-
tableFunctor = lateralView.getTableFunction().getTableFunctor();
31+
tableFunctors = lateralView.getTableFunctions().stream().map(TableFunction::getTableFunctor).collect(Collectors.toList());
3032
}
3133

3234
@Override
3335
public List<BulletRecord> apply(BulletRecord record, BulletRecordProvider provider) {
36+
if (tableFunctors.size() == 1) {
37+
return apply(record, provider, tableFunctors.get(0));
38+
}
39+
Stream<BulletRecord> recordsStream = Stream.of(record);
40+
for (TableFunctor tableFunctor : tableFunctors) {
41+
recordsStream = recordsStream.flatMap(r -> apply(r, provider, tableFunctor).stream());
42+
}
43+
return recordsStream.collect(Collectors.toList());
44+
}
45+
46+
private List<BulletRecord> apply(BulletRecord record, BulletRecordProvider provider, TableFunctor tableFunctor) {
3447
List<BulletRecord> records = tableFunctor.apply(record, provider);
3548
return records.stream().map(generated -> new LateralViewBulletRecord(record, generated)).collect(Collectors.toList());
3649
}

‎src/test/java/com/yahoo/bullet/query/tablefunctions/LateralViewTest.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,14 @@ public void testLateralViewTableFunction() {
1717
LateralView tableFunction = new LateralView(explode);
1818

1919
Assert.assertEquals(tableFunction.getType(), TableFunctionType.LATERAL_VIEW);
20-
Assert.assertEquals(tableFunction.getTableFunction(), explode);
20+
Assert.assertEquals(tableFunction.getTableFunctions().get(0), explode);
2121
Assert.assertTrue(tableFunction.getTableFunctor() instanceof LateralViewFunctor);
2222
}
2323

2424
@Test
2525
public void testToString() {
2626
LateralView tableFunction = new LateralView(new Explode(new FieldExpression("abc"), "foo", null, true));
2727

28-
Assert.assertEquals(tableFunction.toString(), "{type: LATERAL_VIEW, tableFunction: {outer: true, type: EXPLODE, field: {field: abc, type: null}, keyAlias: foo, valueAlias: null}}");
28+
Assert.assertEquals(tableFunction.toString(), "{type: LATERAL_VIEW, tableFunctions: [{outer: true, type: EXPLODE, field: {field: abc, type: null}, keyAlias: foo, valueAlias: null}]}");
2929
}
3030
}

‎src/test/java/com/yahoo/bullet/querying/evaluators/BinaryOperationsTest.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import java.util.ArrayList;
1717
import java.util.Arrays;
18+
import java.util.Collections;
1819

1920
import static com.yahoo.bullet.querying.evaluators.EvaluatorUtils.fieldEvaluator;
2021
import static com.yahoo.bullet.querying.evaluators.EvaluatorUtils.listEvaluator;
@@ -389,6 +390,17 @@ record = RecordBox.get().addMap("map", Pair.of("abc", 123), Pair.of("def", null)
389390
Assert.assertEquals(BinaryOperations.in(valueEvaluator(null), valueEvaluator(null), record), TypedObject.NULL);
390391
Assert.assertEquals(BinaryOperations.in(valueEvaluator(123), fieldEvaluator("map"), record), TypedObject.TRUE);
391392
Assert.assertEquals(BinaryOperations.in(valueEvaluator(456), fieldEvaluator("map"), record), TypedObject.NULL);
393+
394+
// Mismatched numeric types
395+
Assert.assertEquals(BinaryOperations.in(valueEvaluator(123), listEvaluator(456.0, 789.0), record), TypedObject.FALSE);
396+
Assert.assertEquals(BinaryOperations.in(valueEvaluator(456), listEvaluator(456.0, 789.0), record), TypedObject.TRUE);
397+
Assert.assertEquals(BinaryOperations.in(valueEvaluator(123.0f), listEvaluator(456L, 789L), record), TypedObject.FALSE);
398+
Assert.assertEquals(BinaryOperations.in(valueEvaluator(456.0f), listEvaluator(456L, 789L), record), TypedObject.TRUE);
399+
400+
record = RecordBox.get().addListOfMaps("list", Collections.singletonMap("abc", 456), Collections.singletonMap("def", 789)).getRecord();
401+
402+
Assert.assertEquals(BinaryOperations.in(valueEvaluator(123.0), fieldEvaluator("list"), record), TypedObject.FALSE);
403+
Assert.assertEquals(BinaryOperations.in(valueEvaluator(456.0), fieldEvaluator("list"), record), TypedObject.TRUE);
392404
}
393405

394406
@Test
@@ -407,6 +419,17 @@ record = RecordBox.get().addMap("map", Pair.of("abc", 123), Pair.of("def", null)
407419
Assert.assertEquals(BinaryOperations.notIn(valueEvaluator(null), valueEvaluator(null), record), TypedObject.NULL);
408420
Assert.assertEquals(BinaryOperations.notIn(valueEvaluator(123), fieldEvaluator("map"), record), TypedObject.FALSE);
409421
Assert.assertEquals(BinaryOperations.notIn(valueEvaluator(456), fieldEvaluator("map"), record), TypedObject.NULL);
422+
423+
// Mismatched numeric types
424+
Assert.assertEquals(BinaryOperations.notIn(valueEvaluator(123), listEvaluator(456.0, 789.0), record), TypedObject.TRUE);
425+
Assert.assertEquals(BinaryOperations.notIn(valueEvaluator(456), listEvaluator(456.0, 789.0), record), TypedObject.FALSE);
426+
Assert.assertEquals(BinaryOperations.notIn(valueEvaluator(123.0f), listEvaluator(456L, 789L), record), TypedObject.TRUE);
427+
Assert.assertEquals(BinaryOperations.notIn(valueEvaluator(456.0f), listEvaluator(456L, 789L), record), TypedObject.FALSE);
428+
429+
record = RecordBox.get().addListOfMaps("list", Collections.singletonMap("abc", 456), Collections.singletonMap("def", 789)).getRecord();
430+
431+
Assert.assertEquals(BinaryOperations.notIn(valueEvaluator(123.0), fieldEvaluator("list"), record), TypedObject.TRUE);
432+
Assert.assertEquals(BinaryOperations.notIn(valueEvaluator(456.0), fieldEvaluator("list"), record), TypedObject.FALSE);
410433
}
411434

412435
@Test

‎src/test/java/com/yahoo/bullet/querying/tablefunctors/LateralViewBulletRecordTest.java

Lines changed: 102 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,45 +8,133 @@
88
import com.yahoo.bullet.record.simple.UntypedSimpleBulletRecord;
99
import com.yahoo.bullet.typesystem.TypedObject;
1010
import org.junit.Assert;
11+
import org.testng.annotations.BeforeMethod;
1112
import org.testng.annotations.Test;
1213

1314
import java.io.Serializable;
1415
import java.util.Map;
1516

1617
public class LateralViewBulletRecordTest {
17-
private LateralViewBulletRecord record = new LateralViewBulletRecord(new UntypedSimpleBulletRecord(), new UntypedSimpleBulletRecord());
18+
private LateralViewBulletRecord record;
19+
20+
@BeforeMethod
21+
public void setup() {
22+
record = new LateralViewBulletRecord(new UntypedSimpleBulletRecord(), new UntypedSimpleBulletRecord());
23+
}
1824

1925
@Test
2026
public void testGetRawDataMap() {
27+
Map<String, Serializable> map = record.getRawDataMap();
28+
29+
Assert.assertEquals(map.size(), 0);
30+
2131
record.getBaseRecord().setString("abc", "def");
22-
record.getTopRecord().setString("abc", "ghi");
32+
record.getTopRecord().setString("foo", "bar");
33+
34+
map = record.getRawDataMap();
35+
36+
Assert.assertEquals(map.size(), 2);
37+
Assert.assertEquals(map.get("abc"), "def");
38+
Assert.assertEquals(map.get("foo"), "bar");
39+
}
40+
41+
@Test
42+
public void testGetRawDataMapShadowedFieldAndCulledField() {
43+
record.getBaseRecord().setString("abc", "def");
44+
record.getTopRecord().setString("abc", "bar");
2345

2446
Map<String, Serializable> map = record.getRawDataMap();
2547

2648
Assert.assertEquals(map.size(), 1);
27-
Assert.assertEquals(map.get("abc"), "ghi");
49+
Assert.assertEquals(map.get("abc"), "bar");
2850

29-
// "abc" removed from the top record and also added to culled
3051
record.remove("abc");
3152

32-
Assert.assertEquals(record.getCulledFields().size(), 1);
33-
Assert.assertTrue(record.getCulledFields().contains("abc"));
34-
Assert.assertEquals(record.typedGet("abc"), TypedObject.NULL);
35-
3653
map = record.getRawDataMap();
3754

3855
Assert.assertEquals(map.size(), 0);
56+
}
3957

40-
// "abc" set in top record and also removed from culled. New "abc" shadows the field in base record
41-
record.typedSet("abc", TypedObject.valueOf("123"));
58+
@Test
59+
public void testRemove() {
60+
record.getBaseRecord().setString("abc", "def");
61+
record.getTopRecord().setString("abc", "bar");
4262

63+
Assert.assertTrue(record.getBaseRecord().hasField("abc"));
64+
Assert.assertTrue(record.getTopRecord().hasField("abc"));
4365
Assert.assertEquals(record.getCulledFields().size(), 0);
44-
Assert.assertEquals(record.typedGet("abc").getValue(), "123");
4566

46-
map = record.getRawDataMap();
67+
// remove returns this for chaining
68+
Assert.assertSame(record.remove("abc"), record);
4769

48-
Assert.assertEquals(map.size(), 1);
49-
Assert.assertEquals(map.get("abc"), "123");
70+
Assert.assertTrue(record.getBaseRecord().hasField("abc"));
71+
Assert.assertFalse(record.getTopRecord().hasField("abc"));
72+
Assert.assertEquals(record.getCulledFields().size(), 1);
73+
Assert.assertTrue((record.getCulledFields().contains("abc")));
74+
75+
// remove only affects the top record
76+
record.remove("abc");
77+
78+
Assert.assertTrue(record.getBaseRecord().hasField("abc"));
79+
Assert.assertFalse(record.getTopRecord().hasField("abc"));
80+
Assert.assertEquals(record.getCulledFields().size(), 1);
81+
Assert.assertTrue((record.getCulledFields().contains("abc")));
82+
}
83+
84+
@Test
85+
public void testTypedGet() {
86+
record.getBaseRecord().setString("abc", "def");
87+
record.getTopRecord().setString("foo", "bar");
88+
89+
Assert.assertEquals(record.typedGet("abc").getValue(), "def");
90+
Assert.assertEquals(record.typedGet("foo").getValue(), "bar");
91+
Assert.assertNull(record.typedGet("123").getValue());
92+
}
93+
94+
@Test
95+
public void testTypedGetShadowedFieldAndCulledField() {
96+
record.getBaseRecord().setString("abc", "def");
97+
record.getTopRecord().setString("abc", "bar");
98+
99+
Assert.assertEquals(record.typedGet("abc").getValue(), "bar");
100+
101+
record.remove("abc");
102+
103+
Assert.assertNull(record.typedGet("abc").getValue());
104+
}
105+
106+
@Test
107+
public void testTypedSet() {
108+
record.getBaseRecord().setString("abc", "def");
109+
110+
Assert.assertTrue(record.getBaseRecord().hasField("abc"));
111+
Assert.assertFalse(record.getTopRecord().hasField("abc"));
112+
113+
record.typedSet("abc", TypedObject.valueOf("def"));
114+
115+
Assert.assertTrue(record.getBaseRecord().hasField("abc"));
116+
Assert.assertTrue(record.getTopRecord().hasField("abc"));
117+
}
118+
119+
@Test
120+
public void testTypedGetAndTypedSetCulledField() {
121+
record.getBaseRecord().setString("abc", "def");
122+
record.getTopRecord().setString("abc", "bar");
123+
record.remove("abc");
124+
125+
Assert.assertNull(record.typedGet("abc").getValue());
126+
Assert.assertEquals(record.getCulledFields().size(), 1);
127+
Assert.assertTrue(record.getCulledFields().contains("abc"));
128+
129+
record.typedSet("abc", TypedObject.valueOf("bar"));
130+
131+
Assert.assertEquals(record.typedGet("abc").getValue(), "bar");
132+
Assert.assertEquals(record.getCulledFields().size(), 0);
133+
}
134+
135+
@Test
136+
public void testCopy() {
137+
Assert.assertSame(record, record.copy());
50138
}
51139

52140
@Test(expectedExceptions = UnsupportedOperationException.class)
@@ -79,11 +167,6 @@ public void testGetAndRemove() {
79167
record.getAndRemove(null);
80168
}
81169

82-
@Test
83-
public void testCopy() {
84-
Assert.assertSame(record, record.copy());
85-
}
86-
87170
@Test(expectedExceptions = UnsupportedOperationException.class)
88171
public void testIterator() {
89172
record.iterator();

0 commit comments

Comments
 (0)