Merge pull request #12856 from SergeTupchiy/EMQX-11826-prevent-replicants-from-rejoining-a-left-core

prevent replicants from rejoining a left core

apps/emqx/rebar.config

@@ -28,7 +28,7 @@
     {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.1"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.2"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.3"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
     {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.1"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},

apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl

@@ -33,12 +33,12 @@ end_per_suite(_) ->
     emqx_mgmt_api_test_util:end_suite([emqx_management, emqx_conf]).
 
 init_per_testcase(t_autocluster_leave = TC, Config) ->
-    [Core1, Core2, Core3, Repl] =
+    [Core1, Core2, Repl1, Repl2] =
         Nodes = [
             t_autocluster_leave_core1,
             t_autocluster_leave_core2,
-            t_autocluster_leave_core3,
+            t_autocluster_leave_replicant1,
-            t_autocluster_leave_replicant
+            t_autocluster_leave_replicant2
         ],
 
     NodeNames = [emqx_cth_cluster:node_name(N) || N <- Nodes],
@@ -58,8 +58,8 @@ init_per_testcase(t_autocluster_leave = TC, Config) ->
         [
             {Core1, #{role => core, apps => AppSpec}},
             {Core2, #{role => core, apps => AppSpec}},
-            {Core3, #{role => core, apps => AppSpec}},
+            {Repl1, #{role => replicant, apps => AppSpec}},
-            {Repl, #{role => replicant, apps => AppSpec}}
+            {Repl2, #{role => replicant, apps => AppSpec}}
         ],
         #{work_dir => emqx_cth_suite:work_dir(TC, Config)}
     ),
@@ -307,7 +307,7 @@ t_admin(_Config) ->
     ok.
 
 t_autocluster_leave(Config) ->
-    [Core1, Core2, Core3, Repl] = Cluster = ?config(cluster, Config),
+    [Core1, Core2, Repl1, Repl2] = Cluster = ?config(cluster, Config),
     %% Mria membership updates are async, makes sense to wait a little
     timer:sleep(300),
     ClusterView = [lists:sort(rpc:call(N, emqx, running_nodes, [])) || N <- Cluster],
@@ -317,24 +317,24 @@ t_autocluster_leave(Config) ->
     ?assertEqual(View1, View3),
     ?assertEqual(View1, View4),
 
-    rpc:call(Core3, emqx_mgmt_cli, cluster, [["leave"]]),
+    rpc:call(Core2, emqx_mgmt_cli, cluster, [["leave"]]),
     timer:sleep(1000),
-    %% Replicant node may still discover and join Core3 which is now split from [Core1, Core2],
+    %% Replicant nodes can discover Core2 which is now split from [Core1, Core2],
-    %% but it's expected to choose a bigger cluster of [Core1, Core2]..
+    %% but they are  expected to ignore Core2,
-    ?assertMatch([Core3], rpc:call(Core3, emqx, running_nodes, [])),
+    %% since mria_lb must filter out core nodes that disabled discovery.
+    ?assertMatch([Core2], rpc:call(Core2, emqx, running_nodes, [])),
     ?assertEqual(undefined, rpc:call(Core1, erlang, whereis, [ekka_autocluster])),
-    ?assertEqual(lists:sort([Core1, Core2, Repl]), rpc:call(Core1, emqx, running_nodes, [])),
+    ?assertEqual(lists:sort([Core1, Repl1, Repl2]), rpc:call(Core1, emqx, running_nodes, [])),
-    ?assertEqual(lists:sort([Core1, Core2, Repl]), rpc:call(Core2, emqx, running_nodes, [])),
+    ?assertEqual(lists:sort([Core1, Repl1, Repl2]), rpc:call(Repl1, emqx, running_nodes, [])),
-    ?assertEqual(lists:sort([Core1, Core2, Repl]), rpc:call(Repl, emqx, running_nodes, [])),
+    ?assertEqual(lists:sort([Core1, Repl1, Repl2]), rpc:call(Repl2, emqx, running_nodes, [])),
 
-    rpc:call(Repl, emqx_mgmt_cli, cluster, [["leave"]]),
+    rpc:call(Repl1, emqx_mgmt_cli, cluster, [["leave"]]),
     timer:sleep(1000),
-    ?assertEqual(lists:sort([Core1, Core2]), rpc:call(Core1, emqx, running_nodes, [])),
+    ?assertEqual(lists:sort([Core1, Repl2]), rpc:call(Core1, emqx, running_nodes, [])),
-    ?assertEqual(lists:sort([Core1, Core2]), rpc:call(Core2, emqx, running_nodes, [])),
 
-    rpc:call(Core3, emqx_mgmt_cli, cluster, [["discovery", "enable"]]),
+    rpc:call(Core2, emqx_mgmt_cli, cluster, [["discovery", "enable"]]),
-    rpc:call(Repl, emqx_mgmt_cli, cluster, [["discovery", "enable"]]),
+    rpc:call(Repl1, emqx_mgmt_cli, cluster, [["discovery", "enable"]]),
-    %% core nodes will join and restart asyncly, may need more time to re-cluster
+    %% nodes will join and restart asyncly, may need more time to re-cluster
     ?assertEqual(
         ok,
         emqx_common_test_helpers:wait_for(

mix.exs

@@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do
       {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true},
       {:esockd, github: "emqx/esockd", tag: "5.11.1", override: true},
       {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-2", override: true},
-      {:ekka, github: "emqx/ekka", tag: "0.19.2", override: true},
+      {:ekka, github: "emqx/ekka", tag: "0.19.3", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true},
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.4.0", override: true},

rebar.config

@@ -83,7 +83,7 @@
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.1"}}},
     {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-2"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.2"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.3"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
     {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}},
     {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.4.0"}}},