Skip to content

Commit 3360099

Browse files
wxl24lifemergify[bot]
wxl24life
authored and
mergify[bot]
committed
[BugFix] Fail to build MaterializedView relationship when do reload after FE image loaded (#58990)
Signed-off-by: drake_wang <[email protected]> (cherry picked from commit 89e6352)
1 parent 4e56fa4 commit 3360099

File tree

3 files changed

+129
-22
lines changed

3 files changed

+129
-22
lines changed

‎fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import com.starrocks.catalog.constraint.ForeignKeyConstraint;
4040
import com.starrocks.catalog.constraint.GlobalConstraintManager;
4141
import com.starrocks.catalog.mv.MVPlanValidationResult;
42+
import com.starrocks.common.Config;
4243
import com.starrocks.common.DdlException;
4344
import com.starrocks.common.FeConstants;
4445
import com.starrocks.common.MaterializedViewExceptions;
@@ -737,7 +738,7 @@ public void setQueryOutputIndices(List<Integer> queryOutputIndices) {
737738
this.queryOutputIndices = queryOutputIndices;
738739
}
739740

740-
public boolean isReloaded() {
741+
public boolean hasReloaded() {
741742
return reloaded;
742743
}
743744

@@ -1069,9 +1070,6 @@ public void onReload(boolean postLoadImage) {
10691070
LOG.error("reload mv failed: {}", this, e);
10701071
setInactiveAndReason("reload failed: " + e.getMessage());
10711072
}
1072-
if (postLoadImage) {
1073-
reloaded = true;
1074-
}
10751073
}
10761074

10771075
/**
@@ -1148,12 +1146,19 @@ private boolean onReloadImpl(boolean postLoadImage) {
11481146
continue;
11491147
} else if (table.isMaterializedView()) {
11501148
MaterializedView baseMV = (MaterializedView) table;
1151-
// skip reloading only when postLoadImage is true
1152-
if (postLoadImage && baseMV.isReloaded()) {
1153-
continue;
1149+
// only consider skipping reload when postLoadImage is true
1150+
if (Config.enable_mv_post_image_reload_cache && postLoadImage) {
1151+
if (!baseMV.hasReloaded()) {
1152+
// recursive reload MV, to guarantee the order of hierarchical MV
1153+
baseMV.onReload(postLoadImage);
1154+
baseMV.setReloaded(true);
1155+
} else {
1156+
LOG.info("baseMv: {} has reloaded before, skip reload it again", baseMV.getName());
1157+
}
1158+
} else {
1159+
// recursive reload MV, to guarantee the order of hierarchical MV
1160+
baseMV.onReload(postLoadImage);
11541161
}
1155-
// recursive reload MV, to guarantee the order of hierarchical MV
1156-
baseMV.onReload(postLoadImage);
11571162
if (!baseMV.isActive()) {
11581163
LOG.warn("tableName :{} is invalid. set materialized view:{} to invalid",
11591164
baseTableInfo.getTableName(), id);

‎fe/fe-core/src/main/java/com/starrocks/common/Config.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3291,6 +3291,10 @@ public class Config extends ConfigBase {
32913291
"query the rewritten mv directly rather than original base table to improve query performance.")
32923292
public static boolean enable_mv_refresh_query_rewrite = false;
32933293

3294+
@ConfField(mutable = true, comment = "Whether do reload flag check after FE's image loaded." +
3295+
" If one base mv has done reload, no need to do it again while other mv that related to it is reloading ")
3296+
public static boolean enable_mv_post_image_reload_cache = true;
3297+
32943298
/**
32953299
* Whether analyze the mv after refresh in async mode.
32963300
*/

‎fe/fe-core/src/test/java/com/starrocks/catalog/MaterializedViewTest.java

Lines changed: 111 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -895,7 +895,7 @@ public void testCreateMVWithCoolDownTime() throws Exception {
895895
}
896896

897897
@Test
898-
public void testMaterializedViewReload() throws Exception {
898+
public void testMaterializedViewReloadNotPostLoadImage() throws Exception {
899899
starRocksAssert.withDatabase("test").useDatabase("test")
900900
.withTable("CREATE TABLE base_table\n" +
901901
"(\n" +
@@ -926,6 +926,7 @@ public void testMaterializedViewReload() throws Exception {
926926
"REFRESH manual\n" +
927927
"as select k1,k2,v1 from base_mv;");
928928
Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test");
929+
Table baseTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "base_table");
929930
MaterializedView baseMv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore()
930931
.getTable(testDb.getFullName(), "base_mv"));
931932
MaterializedView mv1 = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore()
@@ -934,26 +935,123 @@ public void testMaterializedViewReload() throws Exception {
934935
.getTable(testDb.getFullName(), "mv2"));
935936

936937
{
937-
// before post image reload, all materialized views has `reloaded` flag set to false
938-
Assert.assertFalse(mv1.isReloaded());
939-
Assert.assertFalse(mv2.isReloaded());
940-
Assert.assertFalse(baseMv.isReloaded());
938+
Config.enable_mv_post_image_reload_cache = false;
939+
boolean postLoadImage = false;
941940

942-
GlobalStateMgr.getCurrentState().processMvRelatedMeta();
941+
baseMv.setReloaded(false);
942+
baseTable.removeRelatedMaterializedView(baseMv.getMvId());
943+
baseMv.removeRelatedMaterializedView(mv1.getMvId());
944+
baseMv.removeRelatedMaterializedView(mv2.getMvId());
943945

944-
// after post image reload, all materialized views should have `reloaded` flag reset to false
945-
Assert.assertFalse(mv1.isReloaded());
946-
Assert.assertFalse(mv2.isReloaded());
947-
Assert.assertFalse(baseMv.isReloaded());
946+
mv1.onReload(postLoadImage);
947+
mv2.onReload(postLoadImage);
948+
949+
Assert.assertFalse(baseMv.hasReloaded());
950+
Assert.assertEquals(1, baseTable.getRelatedMaterializedViews().size());
951+
Assert.assertEquals(2, baseMv.getRelatedMaterializedViews().size());
952+
Assert.assertTrue(baseMv.getRelatedMaterializedViews().contains(mv1.getMvId()));
953+
Assert.assertTrue(baseMv.getRelatedMaterializedViews().contains(mv2.getMvId()));
948954
}
949955

950956
{
957+
Config.enable_mv_post_image_reload_cache = true;
951958
boolean postLoadImage = true;
959+
960+
baseMv.setReloaded(false);
961+
baseTable.removeRelatedMaterializedView(baseMv.getMvId());
962+
baseMv.removeRelatedMaterializedView(mv1.getMvId());
963+
baseMv.removeRelatedMaterializedView(mv2.getMvId());
964+
965+
mv1.onReload(postLoadImage);
966+
mv2.onReload(postLoadImage);
967+
968+
Assert.assertTrue(baseMv.hasReloaded());
969+
Assert.assertEquals(1, baseTable.getRelatedMaterializedViews().size());
970+
Assert.assertEquals(2, baseMv.getRelatedMaterializedViews().size());
971+
Assert.assertTrue(baseMv.getRelatedMaterializedViews().contains(mv1.getMvId()));
972+
Assert.assertTrue(baseMv.getRelatedMaterializedViews().contains(mv2.getMvId()));
973+
}
974+
975+
{
976+
Config.enable_mv_post_image_reload_cache = true;
977+
boolean postLoadImage = false;
978+
979+
baseMv.setReloaded(false);
980+
baseTable.removeRelatedMaterializedView(baseMv.getMvId());
981+
baseMv.removeRelatedMaterializedView(mv1.getMvId());
982+
baseMv.removeRelatedMaterializedView(mv2.getMvId());
983+
952984
mv1.onReload(postLoadImage);
953985
mv2.onReload(postLoadImage);
954-
Assert.assertTrue(mv1.isReloaded());
955-
Assert.assertTrue(mv2.isReloaded());
956-
Assert.assertTrue(baseMv.isReloaded());
986+
987+
Assert.assertFalse(baseMv.hasReloaded());
988+
Assert.assertEquals(1, baseTable.getRelatedMaterializedViews().size());
989+
Assert.assertEquals(2, baseMv.getRelatedMaterializedViews().size());
990+
Assert.assertTrue(baseMv.getRelatedMaterializedViews().contains(mv1.getMvId()));
991+
Assert.assertTrue(baseMv.getRelatedMaterializedViews().contains(mv2.getMvId()));
957992
}
958993
}
994+
995+
996+
@Test
997+
public void testMaterializedViewReloadPostLoadImage() throws Exception {
998+
starRocksAssert.withDatabase("test").useDatabase("test")
999+
.withTable("CREATE TABLE base_table\n" +
1000+
"(\n" +
1001+
" k1 date,\n" +
1002+
" k2 int,\n" +
1003+
" v1 int sum\n" +
1004+
")\n" +
1005+
"PARTITION BY RANGE(k1)\n" +
1006+
"(\n" +
1007+
" PARTITION p1 values [('2022-02-01'),('2022-02-16')),\n" +
1008+
" PARTITION p2 values [('2022-02-16'),('2022-03-01'))\n" +
1009+
")\n" +
1010+
"DISTRIBUTED BY HASH(k2) BUCKETS 3\n" +
1011+
"PROPERTIES('replication_num' = '1');")
1012+
.withMaterializedView("CREATE MATERIALIZED VIEW base_mv\n" +
1013+
"PARTITION BY k1\n" +
1014+
"DISTRIBUTED BY HASH(k2) BUCKETS 3\n" +
1015+
"REFRESH manual\n" +
1016+
"as select k1,k2,v1 from base_table;")
1017+
.withMaterializedView("CREATE MATERIALIZED VIEW mv1\n" +
1018+
"PARTITION BY k1\n" +
1019+
"DISTRIBUTED BY HASH(k2) BUCKETS 3\n" +
1020+
"REFRESH manual\n" +
1021+
"as select k1,k2,v1 from base_mv;")
1022+
.withMaterializedView("CREATE MATERIALIZED VIEW mv2\n" +
1023+
"PARTITION BY k1\n" +
1024+
"DISTRIBUTED BY HASH(k2) BUCKETS 3\n" +
1025+
"REFRESH manual\n" +
1026+
"as select k1,k2,v1 from base_mv;");
1027+
Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test");
1028+
Table baseTable = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(testDb.getFullName(), "base_table");
1029+
MaterializedView baseMv = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore()
1030+
.getTable(testDb.getFullName(), "base_mv"));
1031+
MaterializedView mv1 = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore()
1032+
.getTable(testDb.getFullName(), "mv1"));
1033+
MaterializedView mv2 = ((MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore()
1034+
.getTable(testDb.getFullName(), "mv2"));
1035+
1036+
baseTable.removeRelatedMaterializedView(baseMv.getMvId());
1037+
baseMv.removeRelatedMaterializedView(mv1.getMvId());
1038+
baseMv.removeRelatedMaterializedView(mv2.getMvId());
1039+
1040+
Assert.assertFalse(mv1.hasReloaded());
1041+
Assert.assertFalse(mv2.hasReloaded());
1042+
Assert.assertFalse(baseMv.hasReloaded());
1043+
1044+
Config.enable_mv_post_image_reload_cache = true;
1045+
// do post image reload
1046+
GlobalStateMgr.getCurrentState().processMvRelatedMeta();
1047+
1048+
// after post image reload, all materialized views should have `reloaded` flag reset to false
1049+
Assert.assertFalse(mv1.hasReloaded());
1050+
Assert.assertFalse(mv2.hasReloaded());
1051+
Assert.assertFalse(baseMv.hasReloaded());
1052+
Assert.assertEquals(1, baseTable.getRelatedMaterializedViews().size());
1053+
Assert.assertEquals(2, baseMv.getRelatedMaterializedViews().size());
1054+
Assert.assertTrue(baseMv.getRelatedMaterializedViews().contains(mv1.getMvId()));
1055+
Assert.assertTrue(baseMv.getRelatedMaterializedViews().contains(mv2.getMvId()));
1056+
}
9591057
}

0 commit comments

Comments
 (0)