diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 70a0f4c70..9e8816596 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -29,7 +29,7 @@
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.13"}}},
-    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
+    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.0"}}},
     {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
diff --git a/apps/emqx/src/emqx_cm_sup.erl b/apps/emqx/src/emqx_cm_sup.erl
index 221e78c40..9db73e8e4 100644
--- a/apps/emqx/src/emqx_cm_sup.erl
+++ b/apps/emqx/src/emqx_cm_sup.erl
@@ -22,6 +22,9 @@
 
 -export([init/1]).
 
+%% for test
+-export([restart_flapping/0]).
+
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
@@ -59,3 +62,8 @@ child_spec(Mod, Shutdown, Type) ->
         type => Type,
         modules => [Mod]
     }.
+
+restart_flapping() ->
+    ok = supervisor:terminate_child(?MODULE, emqx_flapping),
+    {ok, _} = supervisor:restart_child(?MODULE, emqx_flapping),
+    ok.
diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl
index 021eaddbf..e580e58e3 100644
--- a/apps/emqx/test/emqx_flapping_SUITE.erl
+++ b/apps/emqx/test/emqx_flapping_SUITE.erl
@@ -133,7 +133,9 @@ t_conf_update(_) ->
     ok.
 
 t_conf_update_timer(_Config) ->
-    _ = emqx_flapping:start_link(),
+    %% delete all zones
+    ?assertMatch({ok, _}, emqx:update_config([zones], #{})),
+    emqx_cm_sup:restart_flapping(),
     validate_timer([{default, true}]),
     %% change zones
     {ok, _} =
diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl
index 7a7729878..4b4535cea 100644
--- a/apps/emqx/test/emqx_shared_sub_SUITE.erl
+++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl
@@ -513,7 +513,7 @@ test_two_messages(Strategy, Group) ->
     ok.
 
 last_message(ExpectedPayload, Pids) ->
-    last_message(ExpectedPayload, Pids, 100).
+    last_message(ExpectedPayload, Pids, 1000).
 
 last_message(ExpectedPayload, Pids, Timeout) ->
     receive
@@ -1234,16 +1234,25 @@ setup_node(Node, Port) ->
             ok
         end,
 
-    %% Load env before doing anything
-    [ok = rpc:call(Node, application, load, [App]) || App <- [gen_rpc, emqx, ekka, mria]],
-
-    %% Needs to be set explicitly because ekka:start() (which calls `gen` is called without Handler
-    %% in emqx_common_test_helpers:start_apps(...)
-    ok = rpc:call(Node, application, set_env, [gen_rpc, tcp_server_port, Port - 1]),
-    ok = rpc:call(Node, application, set_env, [gen_rpc, port_discovery, manual]),
+    MyPort = Port - 1,
+    PeerPort = Port - 2,
+    ok = pair_gen_rpc(node(), MyPort, PeerPort),
+    ok = pair_gen_rpc(Node, PeerPort, MyPort),
 
     %% Here we start the node and make it join the cluster
     ok = rpc:call(Node, emqx_common_test_helpers, start_apps, [[], EnvHandler]),
-    rpc:call(Node, mria, join, [node()]),
 
+    %% warm it up, also assert the peer ndoe name
+    Node = emqx_rpc:call(Node, erlang, node, []),
+    rpc:call(Node, mria, join, [node()]),
+    ok.
+
+pair_gen_rpc(Node, LocalPort, RemotePort) ->
+    _ = rpc:call(Node, application, load, [gen_rpc]),
+    ok = rpc:call(Node, application, set_env, [gen_rpc, port_discovery, manual]),
+    ok = rpc:call(Node, application, set_env, [gen_rpc, tcp_server_port, LocalPort]),
+    ok = rpc:call(Node, application, set_env, [gen_rpc, tcp_client_port, RemotePort]),
+    ok = rpc:call(Node, application, set_env, [gen_rpc, default_client_driver, tcp]),
+    _ = rpc:call(Node, application, stop, [gen_rpc]),
+    {ok, _} = rpc:call(Node, application, ensure_all_started, [gen_rpc]),
     ok.
diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl
index 432ce8697..1dd5adab2 100644
--- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl
+++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl
@@ -29,7 +29,7 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("emqx/include/emqx.hrl").
--include("emqx_dashboard.hrl").
+-include_lib("emqx_dashboard/include/emqx_dashboard.hrl").
 
 -define(HOST, "http://127.0.0.1:18083").
 
diff --git a/changes/ce/fix-11622.en.md b/changes/ce/fix-11622.en.md
new file mode 100644
index 000000000..e601fd39a
--- /dev/null
+++ b/changes/ce/fix-11622.en.md
@@ -0,0 +1 @@
+Upgrade rpc library gen_rpc from 2.8.1 to 3.1.0
diff --git a/mix.exs b/mix.exs
index 272013943..f38ff7ed4 100644
--- a/mix.exs
+++ b/mix.exs
@@ -56,7 +56,7 @@ defmodule EMQXUmbrella.MixProject do
       {:esockd, github: "emqx/esockd", tag: "5.9.7", override: true},
       {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true},
       {:ekka, github: "emqx/ekka", tag: "0.15.13", override: true},
-      {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
+      {:gen_rpc, github: "emqx/gen_rpc", tag: "3.1.0", override: true},
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.3.12", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true},
diff --git a/rebar.config b/rebar.config
index c2c73274a..1a0103c74 100644
--- a/rebar.config
+++ b/rebar.config
@@ -63,7 +63,7 @@
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}}
     , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.13"}}}
-    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
+    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.0"}}}
     , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.12"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}}