From 9e2987034bc6aca851eff5474668c2667a786ca5 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 2 Aug 2022 10:36:43 +0800
Subject: [PATCH 01/52] chore(exproto): start idle timer for udp clients

---
 .../emqx_exproto/src/emqx_exproto_channel.erl | 32 ++++++++++++++++---
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl
index 80e5f4045..765dcf53e 100644
--- a/apps/emqx_exproto/src/emqx_exproto_channel.erl
+++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl
@@ -76,7 +76,8 @@
 
 -define(TIMER_TABLE, #{
           alive_timer => keepalive,
-          force_timer => force_close
+          force_timer => force_close,
+          idle_timer => force_close_idle
          }).
 
 -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]).
@@ -94,6 +95,8 @@
          awaiting_rel_max
         ]).
 
+-define(DEFAULT_IDLE_TIMEOUT, 30000).
+
 %%--------------------------------------------------------------------
 %% Info, Attrs and Caps
 %%--------------------------------------------------------------------
@@ -149,8 +152,12 @@ init(ConnInfo = #{socktype := Socktype,
     GRpcChann = proplists:get_value(handler, Options),
     NConnInfo = default_conninfo(ConnInfo),
     ClientInfo = default_clientinfo(ConnInfo),
+
+    IdleTimeout = proplists:get_value(idle_timeout, Options, ?DEFAULT_IDLE_TIMEOUT),
+
+    NConnInfo1 = NConnInfo#{idle_timeout => IdleTimeout},
     Channel = #channel{gcli = #{channel => GRpcChann},
-                       conninfo = NConnInfo,
+                       conninfo = NConnInfo1,
                        clientinfo = ClientInfo,
                        conn_state = accepted,
                        timers = #{}
@@ -165,7 +172,8 @@ init(ConnInfo = #{socktype := Socktype,
                              #{socktype => socktype(Socktype),
                                peername => address(Peername),
                                sockname => address(Sockname)})},
-            try_dispatch(on_socket_created, wrap(Req), Channel)
+            start_idle_checking_timer(
+              try_dispatch(on_socket_created, wrap(Req), Channel))
     end.
 
 register_the_anonymous_client(ClientInfo, ConnInfo) ->
@@ -184,6 +192,12 @@ register_the_anonymous_client(ClientInfo, ConnInfo) ->
 unregister_the_anonymous_client(ClientId) ->
     emqx_cm:unregister_channel(ClientId).
 
+start_idle_checking_timer(Channel = #channel{conninfo = #{socktype := udp}}) ->
+    ensure_timer(idle_timer, Channel);
+
+start_idle_checking_timer(Channel) ->
+    Channel.
+
 %% @private
 peercert(NoSsl, ConnInfo) when NoSsl == nossl;
                                NoSsl == undefined ->
@@ -267,6 +281,9 @@ handle_timeout(_TRef, {keepalive, StatVal},
 handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) ->
     {shutdown, Reason, Channel};
 
+handle_timeout(_TRef, force_close_idle, Channel) ->
+    {shutdown, idle_timeout, Channel};
+
 handle_timeout(_TRef, Msg, Channel) ->
     ?WARN("Unexpected timeout: ~p", [Msg]),
     {ok, Channel}.
@@ -328,7 +345,8 @@ handle_call({start_timer, keepalive, Interval},
     NConnInfo = ConnInfo#{keepalive => Interval},
     NClientInfo = ClientInfo#{keepalive => Interval},
     NChannel = Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo},
-    {reply, ok, [{event, updated}], ensure_keepalive(NChannel)};
+    {reply, ok, [{event, updated}],
+     ensure_keepalive(cancel_timer(idle_timer, NChannel))};
 
 handle_call({subscribe, TopicFilter, Qos},
             Channel = #channel{
@@ -561,6 +579,12 @@ reset_timer(Name, Channel) ->
 clean_timer(Name, Channel = #channel{timers = Timers}) ->
     Channel#channel{timers = maps:remove(Name, Timers)}.
 
+cancel_timer(Name, Channel = #channel{timers = Timers}) ->
+    emqx_misc:cancel_timer(maps:get(Name, Timers, undefined)),
+    clean_timer(Name, Channel).
+
+interval(idle_timer, #channel{conninfo = #{idle_timeout := IdleTimeout}}) ->
+    IdleTimeout;
 interval(force_timer, _) ->
     15000;
 interval(alive_timer, #channel{keepalive = Keepalive}) ->

From ec5c0816f70258822b0d4e3517e3e9e5138ab168 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 3 Aug 2022 13:59:49 +0800
Subject: [PATCH 02/52] chore: update app.src & appup.src

---
 CHANGES-4.3.md                               | 6 ++++++
 apps/emqx_exproto/src/emqx_exproto.app.src   | 2 +-
 apps/emqx_exproto/src/emqx_exproto.appup.src | 8 ++++++--
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index db82c8eb1..d67f8e838 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -10,6 +10,12 @@ File format:
 - One list item per change topic
   Change log ends with a list of github PRs
 
+## v4.3.19
+
+### Bug fixes
+
+- Add a idle timer for ExProto UDP client to avoid client leaking [#8628]
+
 ## v4.3.18
 
 ### Enhancements
diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src
index 76bc9fa85..a267d1daf 100644
--- a/apps/emqx_exproto/src/emqx_exproto.app.src
+++ b/apps/emqx_exproto/src/emqx_exproto.app.src
@@ -1,6 +1,6 @@
 {application, emqx_exproto,
  [{description, "EMQ X Extension for Protocol"},
-  {vsn, "4.3.9"}, %% 4.3.3 is used by ee
+  {vsn, "4.3.10"}, %% 4.3.3 is used by ee
   {modules, []},
   {registered, []},
   {mod, {emqx_exproto_app, []}},
diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src
index d6406b239..8e0380089 100644
--- a/apps/emqx_exproto/src/emqx_exproto.appup.src
+++ b/apps/emqx_exproto/src/emqx_exproto.appup.src
@@ -1,7 +1,9 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{<<"4\\.3\\.[2-8]">>,
+  [{"4.3.9",
+    [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[2-8]">>,
     [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[0-1]">>,
@@ -10,7 +12,9 @@
      {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{<<"4\\.3\\.[2-8]">>,
+  [{"4.3.9",
+    [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[2-8]">>,
     [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]},
      {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[0-1]">>,

From 616a2235f28d11647256873193db70e7e3662383 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 4 Aug 2022 19:09:03 +0200
Subject: [PATCH 03/52] fix(lwm2m): improve error logging for bad lwm2m
 requests

---
 apps/emqx_lwm2m/src/emqx_lwm2m.app.src        |  2 +-
 apps/emqx_lwm2m/src/emqx_lwm2m.appup.src      | 74 +++++++++++++++----
 .../emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl |  6 +-
 apps/emqx_lwm2m/src/emqx_lwm2m_json.erl       |  4 +-
 apps/emqx_lwm2m/src/emqx_lwm2m_message.erl    |  4 +-
 apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl | 11 ++-
 .../src/emqx_lwm2m_xml_object_db.erl          |  7 +-
 apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl     | 54 ++++++++++++++
 8 files changed, 133 insertions(+), 29 deletions(-)

diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src
index 18c85faad..6b0e6bd92 100644
--- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src
+++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src
@@ -1,6 +1,6 @@
 {application,emqx_lwm2m,
  [{description,"EMQ X LwM2M Gateway"},
-  {vsn, "4.3.7"}, % strict semver, bump manually!
+  {vsn, "4.3.8"}, % strict semver, bump manually!
   {modules,[]},
   {registered,[emqx_lwm2m_sup]},
   {applications,[kernel,stdlib,lwm2m_coap]},
diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src
index 1d2dd58fc..793feecd1 100644
--- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src
+++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src
@@ -1,29 +1,75 @@
 %% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{<<"4\\.3\\.[0-1]">>,
-    [{restart_application,emqx_lwm2m}]},
+  [{"4.3.7",
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,[{restart_application,emqx_lwm2m}]},
    {"4.3.2",
-    [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[3-4]">>,
-    [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
-    [{load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
-    [ %% There are only changes to the schema file, so we don't need any
-      %% commands here
-    ]}],
-  [{<<"4\\.3\\.[0-1]">>,
-    [{restart_application,emqx_lwm2m}]},
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}]}],
+  [{"4.3.7",
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,[{restart_application,emqx_lwm2m}]},
    {"4.3.2",
-    [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[3-4]">>,
-    [{load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_protocol,brutal_purge,soft_purge,[]},
      {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
-    [{load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
-   {"4.3.6", []}]}.
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_api,brutal_purge,soft_purge,[]}]},
+   {"4.3.6",
+    [{load_module,emqx_lwm2m_xml_object_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_xml_object,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_message,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_json,brutal_purge,soft_purge,[]},
+     {load_module,emqx_lwm2m_cmd_handler,brutal_purge,soft_purge,[]}]}]}.
diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl
index b0c881f8b..3203cf070 100644
--- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl
+++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl
@@ -106,9 +106,11 @@ coap_read_to_mqtt({ok, SuccessCode}, CoapPayload, Format, Ref) ->
         Result = coap_content_to_mqtt_payload(CoapPayload, Format, Ref),
         make_response(SuccessCode, Ref, Format, Result)
     catch
-        error:not_implemented -> make_response(not_implemented, Ref);
+        throw : {bad_request, Reason} ->
+            ?LOG(error, "bad_request, reason=~p, payload=~p", [Reason, CoapPayload]),
+            make_response(bad_request, Ref);
         C:R:Stack ->
-            ?LOG(error, "~p, bad payload format: ~p, stacktrace: ~p", [{C, R}, CoapPayload, Stack]),
+            ?LOG(error, "bad_request, error=~p, stacktrace=~p~npayload=~p", [{C, R}, Stack, CoapPayload]),
             make_response(bad_request, Ref)
     end.
 
diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl
index 2a10f11f4..d2356424e 100644
--- a/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl
+++ b/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl
@@ -29,7 +29,7 @@
 tlv_to_json(BaseName, TlvData) ->
     DecodedTlv = emqx_lwm2m_tlv:parse(TlvData),
     ObjectId = object_id(BaseName),
-    ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true),
+    ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true),
     case DecodedTlv of
         [#{tlv_resource_with_value:=Id, value:=Value}] ->
             TrueBaseName = basename(BaseName, undefined, undefined, Id, 3),
@@ -315,7 +315,7 @@ encode_int(Int) -> binary:encode_unsigned(Int).
 
 text_to_json(BaseName, Text) ->
     {ObjectId, ResourceId} = object_resource_id(BaseName),
-    ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true),
+    ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true),
     {K, V} = text_value(Text, ResourceId, ObjDefinition),
     #{bn=>BaseName, e=>[#{K=>V}]}.
 
diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl
index 9a05e1663..0fb6c0e9c 100644
--- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl
+++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl
@@ -33,7 +33,7 @@
 tlv_to_json(BaseName, TlvData) ->
     DecodedTlv = emqx_lwm2m_tlv:parse(TlvData),
     ObjectId = object_id(BaseName),
-    ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true),
+    ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true),
     case DecodedTlv of
         [#{tlv_resource_with_value:=Id, value:=Value}] ->
             TrueBaseName = basename(BaseName, undefined, undefined, Id, 3),
@@ -289,7 +289,7 @@ path([H|T], Acc) ->
 
 text_to_json(BaseName, Text) ->
     {ObjectId, ResourceId} = object_resource_id(BaseName),
-    ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true),
+    ObjDefinition = emqx_lwm2m_xml_object:get_obj_def_assertive(ObjectId, true),
     Val = text_value(Text, ResourceId, ObjDefinition),
     [#{path => BaseName, value => Val}].
 
diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl
index 6ac29b3db..9046f6425 100644
--- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl
+++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl
@@ -20,6 +20,7 @@
 -include_lib("xmerl/include/xmerl.hrl").
 
 -export([ get_obj_def/2
+        , get_obj_def_assertive/2
         , get_object_id/1
         , get_object_name/1
         , get_object_and_resource_id/2
@@ -31,15 +32,19 @@
 -define(LOG(Level, Format, Args),
     logger:Level("LWM2M-OBJ: " ++ Format, Args)).
 
-% This module is for future use. Disabled now.
+get_obj_def_assertive(ObjectId, IsInt) ->
+    case get_obj_def(ObjectId, IsInt) of
+        {error, no_xml_definition} ->
+            erlang:throw({bad_request, {unknown_object_id, ObjectId}});
+        Xml ->
+            Xml
+    end.
 
 get_obj_def(ObjectIdInt, true) ->
     emqx_lwm2m_xml_object_db:find_objectid(ObjectIdInt);
 get_obj_def(ObjectNameStr, false) ->
     emqx_lwm2m_xml_object_db:find_name(ObjectNameStr).
 
-
-
 get_object_id(ObjDefinition) ->
     [#xmlText{value=ObjectId}] = xmerl_xpath:string("ObjectID/text()", ObjDefinition),
     ObjectId.
diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl
index 0f5fd7df9..df16d6f7c 100644
--- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl
+++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl
@@ -69,12 +69,9 @@ find_name(Name) ->
                  end,
     case ets:lookup(?LWM2M_OBJECT_NAME_TO_ID_TAB, NameBinary) of
         [] ->
-            undefined;
+            {error, no_xml_definition};
         [{NameBinary, ObjectId}] ->
-            case ets:lookup(?LWM2M_OBJECT_DEF_TAB, ObjectId) of
-                [] -> undefined;
-                [{ObjectId, Xml}] -> Xml
-            end
+            find_objectid(ObjectId)
     end.
 
 stop() ->
diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl
index 27cde6d08..31ee16745 100644
--- a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl
+++ b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl
@@ -689,6 +689,60 @@ case10_read(Config) ->
                              }),
     ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
 
+case10_read_bad_request(Config) ->
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId1 = 15,
+    RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"),
+    emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0),
+    timer:sleep(200),
+    % step 1, device register ...
+    test_send_coap_request( UdpSock,
+                            post,
+                            sprintf("coap://127.0.0.1:~b/rd?ep=~s&lt=345&lwm2m=1", [?PORT, Epn]),
+                            #coap_content{content_format = <<"text/plain">>,
+                            payload = <<"</lwm2m>;rt=\"oma.lwm2m\";ct=11543,</lwm2m/1/0>,</lwm2m/2/0>,</lwm2m/3/0>">>},
+                            [],
+                            MsgId1),
+    #coap_message{method = Method1} = test_recv_coap_response(UdpSock),
+    ?assertEqual({ok,created}, Method1),
+    test_recv_mqtt_response(RespTopic),
+
+    % step2,  send a READ command to device
+    CmdId = 206,
+    CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>,
+    Command =   #{
+                    <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                    <<"msgType">> => <<"read">>,
+                    <<"data">> => #{
+                        <<"path">> => <<"/3333/0/0">>
+                    }
+                },
+    CommandJson = emqx_json:encode(Command),
+    ?LOGT("CommandJson=~p", [CommandJson]),
+    test_mqtt_broker:publish(CommandTopic, CommandJson, 0),
+    timer:sleep(50),
+    Request2 = test_recv_coap_request(UdpSock),
+    #coap_message{method = Method2, payload=Payload2} = Request2,
+    ?LOGT("LwM2M client got ~p", [Request2]),
+    ?assertEqual(get, Method2),
+    ?assertEqual(<<>>, Payload2),
+    timer:sleep(50),
+
+    test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true),
+    timer:sleep(100),
+
+    ReadResult = emqx_json:encode(#{  <<"requestID">> => CmdId, <<"cacheID">> => CmdId,
+                                <<"msgType">> => <<"read">>,
+                                <<"data">> => #{
+                                    <<"code">> => <<"4.00">>,
+                                    <<"codeMsg">> => <<"bad_request">>,
+                                    <<"reqPath">> => <<"/3333/0/0">>
+                                }
+                             }),
+    ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)).
+
+
 case10_read_separate_ack(Config) ->
     UdpSock = ?config(sock, Config),
     Epn = "urn:oma:lwm2m:oma:3",

From 1d9ee25c920516e217d241f6444bdf15c3665275 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Thu, 4 Aug 2022 10:23:26 +0800
Subject: [PATCH 04/52] fix: support custom count function

---
 apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl | 12 ++++++++++--
 apps/emqx_management/src/emqx_mgmt_api.erl         | 13 +++++++++++--
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl
index 331d14a92..b09f99466 100644
--- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl
+++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl
@@ -132,7 +132,11 @@
 
 list_clientid(_Bindings, Params) ->
     SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
-    return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun)}).
+    CountFun = fun() ->
+        MatchSpec = [{{?TABLE, {clientid, '_'}, '$1', '$2'}, [], [true]}],
+        ets:select_count(?TABLE, MatchSpec)
+               end,
+    return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun, CountFun)}).
 
 lookup_clientid(#{clientid := Clientid}, _Params) ->
     return({ok, format(emqx_auth_mnesia_cli:lookup_user({clientid, urldecode(Clientid)}))}).
@@ -182,7 +186,11 @@ delete_clientid(#{clientid := Clientid}, _) ->
 
 list_username(_Bindings, Params) ->
     SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
-    return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun)}).
+    CountFun = fun() ->
+        MatchSpec = [{{?TABLE, {username, '_'}, '$1', '$2'}, [], [true]}],
+        ets:select_count(?TABLE, MatchSpec)
+               end,
+    return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun, CountFun)}).
 
 lookup_username(#{username := Username}, _Params) ->
     return({ok, format(emqx_auth_mnesia_cli:lookup_user({username, urldecode(Username)}))}).
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 66a5c1f9d..482dcac0e 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -24,6 +24,7 @@
 -export([ params2qs/2
         , node_query/4
         , node_query/5
+        , node_query/6
         , cluster_query/3
         , traverse_table/5
         , select_table/5
@@ -59,6 +60,11 @@ query_handle([Table]) when is_atom(Table) ->
 query_handle(Tables) ->
     qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]).
 
+count_size(Table, undefined) ->
+    count(Table);
+count_size(_Table, CountFun) ->
+    CountFun().
+
 count(Table) when is_atom(Table) ->
     ets:info(Table, size);
 count([Table]) when is_atom(Table) ->
@@ -83,9 +89,12 @@ limit(Params) ->
 %%--------------------------------------------------------------------
 
 node_query(Node, Params, {Tab, QsSchema}, QueryFun) ->
-    node_query(Node, Params, {Tab, QsSchema}, QueryFun, undefined).
+    node_query(Node, Params, {Tab, QsSchema}, QueryFun, undefined, undefined).
 
 node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun) ->
+    node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun, undefined).
+
+node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun, CountFun) ->
     {CodCnt, Qs} = params2qs(Params, QsSchema),
     Limit = limit(Params),
     Page  = page(Params),
@@ -95,7 +104,7 @@ node_query(Node, Params, {Tab, QsSchema}, QueryFun, SortFun) ->
     {_, Rows} = do_query(Node, Qs, QueryFun, Start, Limit+1),
     Meta = #{page => Page, limit => Limit},
     NMeta = case CodCnt =:= 0 of
-                true -> Meta#{count => count(Tab), hasnext => length(Rows) > Limit};
+                true -> Meta#{count => count_size(Tab, CountFun), hasnext => length(Rows) > Limit};
                 _ -> Meta#{count => -1, hasnext => length(Rows) > Limit}
             end,
     Data0 = lists:sublist(Rows, Limit),

From f97820f9138c3d5e06d1fac4c7aabd5a95061fca Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Fri, 5 Aug 2022 11:07:16 +0800
Subject: [PATCH 05/52] test: add auth_mnesia count test

---
 .../test/emqx_auth_mnesia_SUITE.erl           | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl
index 1dd979379..508ff93b3 100644
--- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl
+++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl
@@ -272,7 +272,8 @@ t_clientid_rest_api(_Config) ->
     clean_all_users(),
 
     {ok, Result1} = request_http_rest_list(["auth_clientid"]),
-    [] = get_http_data(Result1),
+    ?assertMatch(#{<<"data">> := [], <<"meta">> := #{<<"count">> := 0}},
+        emqx_json:decode(Result1, [return_maps])),
 
     Params1 = #{<<"clientid">> => ?CLIENTID, <<"password">> => ?PASSWORD},
     {ok, _} = request_http_rest_add(["auth_clientid"], Params1),
@@ -295,8 +296,27 @@ t_clientid_rest_api(_Config) ->
                   }, get_http_data(Result3)),
 
     {ok, Result4} = request_http_rest_list(["auth_clientid"]),
+    #{<<"data">> := Data4, <<"meta">> := #{<<"count">> := Count4}}
+        = emqx_json:decode(Result4, [return_maps]),
 
-    ?assertEqual(3, length(get_http_data(Result4))),
+    ?assertEqual(3, Count4),
+    ?assertEqual(3, length(Data4)),
+
+    UserNameParams = [ #{<<"username">> => ?USERNAME, <<"password">> => ?PASSWORD}
+        , #{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD}
+        , #{<<"username">> => <<"username2">>, <<"password">> => ?PASSWORD}
+    ],
+    {ok, _} = request_http_rest_add(["auth_username"], UserNameParams),
+
+    {ok, Result41} = request_http_rest_list(["auth_clientid"]),
+    %% the count clientid is not affected by username count.
+    ?assertEqual(Result4, Result41),
+
+    {ok, Result42} = request_http_rest_list(["auth_username"]),
+    #{<<"data">> := Data42, <<"meta">> := #{<<"count">> := Count42}}
+        = emqx_json:decode(Result42, [return_maps]),
+    ?assertEqual(3, Count42),
+    ?assertEqual(3, length(Data42)),
 
     {ok, Result5} = request_http_rest_list(["auth_clientid?_like_clientid=id"]),
     ?assertEqual(2, length(get_http_data(Result5))),

From 43e1087e131960942d9a1144aec2890b4cc3b8fe Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Fri, 5 Aug 2022 11:34:04 +0800
Subject: [PATCH 06/52] test: add auth_mnesia count test

---
 CHANGES-4.3.md                                       | 7 ++++++-
 apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src   | 2 +-
 apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src | 7 +++++--
 apps/emqx_management/src/emqx_management.app.src     | 2 +-
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index a9d89473e..6d1e0a4dc 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -8,7 +8,12 @@ File format:
 
 - Use weight-2 heading for releases
 - One list item per change topic
-  Change log ends with a list of github PRs
+  Change log ends with a list of GitHub PRs
+
+## v4.3.19
+
+### Bug fixes
+- Fix GET `/auth_clientid`& `/auth_username`'s count. [#8655](https://github.com/emqx/emqx/pull/8655)
 
 ## v4.3.18
 
diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src
index 592535f46..54d1317d7 100644
--- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src
+++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src
@@ -1,6 +1,6 @@
 {application, emqx_auth_mnesia,
  [{description, "EMQ X Authentication with Mnesia"},
-  {vsn, "4.3.7"}, % strict semver, bump manually
+  {vsn, "4.3.8"}, % strict semver, bump manually
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,mnesia]},
diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
index a849002d6..8fa8384d4 100644
--- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
+++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src
@@ -1,7 +1,9 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{<<"4\\.3\\.[5-6]">>,
+  [{<<"4\\.3\\.7">>,
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[5-6]">>,
     [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
@@ -28,7 +30,8 @@
      {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{<<"4\\.3\\.[5-6]">>,
+  [{"4.3.7",[{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[5-6]">>,
     [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
      {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index 937ec9db6..6a5ae3d07 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -1,6 +1,6 @@
 {application, emqx_management,
  [{description, "EMQ X Management API and CLI"},
-  {vsn, "4.3.15"}, % strict semver, bump manually!
+  {vsn, "4.3.16"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
   {applications, [kernel,stdlib,minirest]},

From 5ff864af156de67250b767b2202be7e0075eab95 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 5 Aug 2022 15:48:24 +0200
Subject: [PATCH 07/52] docs: update CHANGES-4.3.md

---
 CHANGES-4.3.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index a9d89473e..0c4e1dd21 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -10,6 +10,12 @@ File format:
 - One list item per change topic
   Change log ends with a list of github PRs
 
+## v4.3.19
+
+### Enhancements
+
+- Improve error message for LwM2M plugin when object ID is not valid [#8654](https://github.com/emqx/emqx/pull/8654).
+
 ## v4.3.18
 
 ### Enhancements

From f2a248d83ff6d0e61b7f8d1f637bb92054bed3ee Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Mon, 8 Aug 2022 09:35:26 +0800
Subject: [PATCH 08/52] chore: update CHANGES-4.3.md

Co-authored-by: Thales Macedo Garitezi <thalesmg@gmail.com>
---
 CHANGES-4.3.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index 6d1e0a4dc..d711c589b 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -13,7 +13,7 @@ File format:
 ## v4.3.19
 
 ### Bug fixes
-- Fix GET `/auth_clientid`& `/auth_username`'s count. [#8655](https://github.com/emqx/emqx/pull/8655)
+- Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655)
 
 ## v4.3.18
 

From 4869c94e974eb44fea44f4b6aca9eaaa96fb349f Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Mon, 8 Aug 2022 10:25:01 +0800
Subject: [PATCH 09/52] chore: replace / with _ in match_spec

---
 apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl    |  4 ++--
 apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl | 11 ++++++-----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl
index b09f99466..6e435ee11 100644
--- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl
+++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl
@@ -133,7 +133,7 @@
 list_clientid(_Bindings, Params) ->
     SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
     CountFun = fun() ->
-        MatchSpec = [{{?TABLE, {clientid, '_'}, '$1', '$2'}, [], [true]}],
+        MatchSpec = [{{?TABLE, {clientid, '_'}, '_', '_'}, [], [true]}],
         ets:select_count(?TABLE, MatchSpec)
                end,
     return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun, CountFun)}).
@@ -187,7 +187,7 @@ delete_clientid(#{clientid := Clientid}, _) ->
 list_username(_Bindings, Params) ->
     SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
     CountFun = fun() ->
-        MatchSpec = [{{?TABLE, {username, '_'}, '$1', '$2'}, [], [true]}],
+        MatchSpec = [{{?TABLE, {username, '_'}, '_', '_'}, [], [true]}],
         ets:select_count(?TABLE, MatchSpec)
                end,
     return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun, CountFun)}).
diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl
index 508ff93b3..0253110aa 100644
--- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl
+++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl
@@ -300,10 +300,10 @@ t_clientid_rest_api(_Config) ->
         = emqx_json:decode(Result4, [return_maps]),
 
     ?assertEqual(3, Count4),
-    ?assertEqual(3, length(Data4)),
+    ?assertEqual([<<"client2">>, <<"clientid1">>, ?CLIENTID],
+        lists:sort(lists:map(fun(#{<<"clientid">> := C}) -> C end, Data4))),
 
-    UserNameParams = [ #{<<"username">> => ?USERNAME, <<"password">> => ?PASSWORD}
-        , #{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD}
+    UserNameParams = [#{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD}
         , #{<<"username">> => <<"username2">>, <<"password">> => ?PASSWORD}
     ],
     {ok, _} = request_http_rest_add(["auth_username"], UserNameParams),
@@ -315,8 +315,9 @@ t_clientid_rest_api(_Config) ->
     {ok, Result42} = request_http_rest_list(["auth_username"]),
     #{<<"data">> := Data42, <<"meta">> := #{<<"count">> := Count42}}
         = emqx_json:decode(Result42, [return_maps]),
-    ?assertEqual(3, Count42),
-    ?assertEqual(3, length(Data42)),
+    ?assertEqual(2, Count42),
+    ?assertEqual([<<"username1">>, <<"username2">>],
+        lists:sort(lists:map(fun(#{<<"username">> := U}) -> U end, Data42))),
 
     {ok, Result5} = request_http_rest_list(["auth_clientid?_like_clientid=id"]),
     ?assertEqual(2, length(get_http_data(Result5))),

From 8dd966e8ef18113f9327d5727f3554721f692ee9 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Mon, 8 Aug 2022 16:22:10 +0800
Subject: [PATCH 10/52] fix: enable emqx_mod_module if default_plugins

---
 src/emqx.app.src     |  2 +-
 src/emqx.appup.src   | 16 ++++++++++++++--
 src/emqx_plugins.erl |  2 +-
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/emqx.app.src b/src/emqx.app.src
index 14a6f4fa4..14c63c75e 100644
--- a/src/emqx.app.src
+++ b/src/emqx.app.src
@@ -6,7 +6,7 @@
   %% the emqx `release' version, which in turn is comprised of several
   %% apps, one of which is this.  See `emqx_release.hrl' for more
   %% info.
-  {vsn, "4.3.19"}, % strict semver, bump manually!
+  {vsn, "4.3.20"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [ kernel
diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index c86661891..adc7be0c3 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -1,13 +1,19 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.18",[{load_module,emqx_app,brutal_purge,soft_purge,[]}]},
+  [{"4.3.19",
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
+   {"4.3.18",
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
     [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {update,emqx_broker_sup,supervisor},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
     [{load_module,emqx_session,brutal_purge,soft_purge,[]},
@@ -678,13 +684,19 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.18",[{load_module,emqx_app,brutal_purge,soft_purge,[]}]},
+  [{"4.3.19",
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
+   {"4.3.18",
+     [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
     [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {update,emqx_broker_sup,supervisor},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
     [{load_module,emqx_session,brutal_purge,soft_purge,[]},
diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl
index 3177f05e7..caf47b0c0 100644
--- a/src/emqx_plugins.erl
+++ b/src/emqx_plugins.erl
@@ -219,7 +219,7 @@ ensure_file(File) ->
         false ->
             DefaultPlugins = [ {emqx_management, true}
                              , {emqx_dashboard, true}
-                             , {emqx_modules, false}
+                             , {emqx_modules, true}
                              , {emqx_recon, true}
                              , {emqx_retainer, true}
                              , {emqx_telemetry, true}

From 50bb5a4f5ac8f4bc9c8197cce54b67779553df1b Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Tue, 9 Aug 2022 09:36:33 +0800
Subject: [PATCH 11/52] fix: emqx version is 4.3.19

---
 src/emqx.app.src   | 2 +-
 src/emqx.appup.src | 8 ++------
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/emqx.app.src b/src/emqx.app.src
index 14c63c75e..14a6f4fa4 100644
--- a/src/emqx.app.src
+++ b/src/emqx.app.src
@@ -6,7 +6,7 @@
   %% the emqx `release' version, which in turn is comprised of several
   %% apps, one of which is this.  See `emqx_release.hrl' for more
   %% info.
-  {vsn, "4.3.20"}, % strict semver, bump manually!
+  {vsn, "4.3.19"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [ kernel
diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index adc7be0c3..70d18ff48 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -1,9 +1,7 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.19",
-    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
+  [
    {"4.3.18",
     [{load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
@@ -684,9 +682,7 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.19",
-    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
+  [
    {"4.3.18",
      [{load_module,emqx_app,brutal_purge,soft_purge,[]},
       {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},

From 25228ed248db60f5d069c02ef5e2ef3cc9ad0c44 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Tue, 9 Aug 2022 09:37:05 +0800
Subject: [PATCH 12/52] test: fix compile warning

---
 apps/emqx_sn/examples/simple_example2.erl | 1 +
 apps/emqx_sn/examples/simple_example3.erl | 1 +
 apps/emqx_sn/examples/simple_example4.erl | 2 +-
 test/emqx_broker_sup_SUITE.erl            | 2 +-
 4 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_sn/examples/simple_example2.erl b/apps/emqx_sn/examples/simple_example2.erl
index b9ada6d22..9af9262e9 100644
--- a/apps/emqx_sn/examples/simple_example2.erl
+++ b/apps/emqx_sn/examples/simple_example2.erl
@@ -6,6 +6,7 @@
 -define(PORT, 1884).
 
 -export([start/0]).
+-export([gen_register_packet/2]).
 
 start() ->
     io:format("start to connect ~p:~p~n", [?HOST, ?PORT]),
diff --git a/apps/emqx_sn/examples/simple_example3.erl b/apps/emqx_sn/examples/simple_example3.erl
index 40f0bf572..6bc41f1e8 100644
--- a/apps/emqx_sn/examples/simple_example3.erl
+++ b/apps/emqx_sn/examples/simple_example3.erl
@@ -6,6 +6,7 @@
 -define(PORT, 1884).
 
 -export([start/0]).
+-export([gen_register_packet/2]).
 
 start() ->
     io:format("start to connect ~p:~p~n", [?HOST, ?PORT]),
diff --git a/apps/emqx_sn/examples/simple_example4.erl b/apps/emqx_sn/examples/simple_example4.erl
index 6beb5835c..1b4809626 100644
--- a/apps/emqx_sn/examples/simple_example4.erl
+++ b/apps/emqx_sn/examples/simple_example4.erl
@@ -5,7 +5,7 @@
 -define(HOST, {127,0,0,1}).
 -define(PORT, 1884).
 
--export([start/0]).
+-export([start/1]).
 
 start(LoopTimes) ->
     io:format("start to connect ~p:~p~n", [?HOST, ?PORT]),
diff --git a/test/emqx_broker_sup_SUITE.erl b/test/emqx_broker_sup_SUITE.erl
index 04881f308..20bcfbbf6 100644
--- a/test/emqx_broker_sup_SUITE.erl
+++ b/test/emqx_broker_sup_SUITE.erl
@@ -61,5 +61,5 @@ t_restart_shared_sub(Config) when is_list(Config) ->
        after 2000 ->
                  false
        end);
-t_restart_shared_sub({'end', Config}) ->
+t_restart_shared_sub({'end', _Config}) ->
     emqx:unsubscribe(<<"$share/grpa/t/a">>).

From e9a5c4295c725b26b9858c718b8f10b20ac6a5e5 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Tue, 9 Aug 2022 10:29:17 +0800
Subject: [PATCH 13/52] fix: bad NeedToLoad plugins

---
 src/emqx_plugins.erl        | 2 +-
 test/emqx_plugins_SUITE.erl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl
index caf47b0c0..764778b74 100644
--- a/src/emqx_plugins.erl
+++ b/src/emqx_plugins.erl
@@ -265,7 +265,7 @@ load_plugins(Names, Persistent) ->
         []       -> ok;
         NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound])
     end,
-    NeedToLoad = Names -- NotFound -- names(started_app),
+    NeedToLoad = (Names -- NotFound) -- names(started_app),
     lists:foreach(fun(Name) ->
                       Plugin = find_plugin(Name, Plugins),
                       load_plugin(Plugin#plugin.name, Persistent)
diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl
index cd4b68d86..66a88a047 100644
--- a/test/emqx_plugins_SUITE.erl
+++ b/test/emqx_plugins_SUITE.erl
@@ -100,7 +100,7 @@ t_ensure_default_loaded_plugins_file(Config) ->
        [ {emqx_bridge_mqtt, false}
        , {emqx_dashboard, true}
        , {emqx_management, true}
-       , {emqx_modules, false}
+       , {emqx_modules, true}
        , {emqx_recon, true}
        , {emqx_retainer, true}
        , {emqx_rule_engine, true}

From c93dbf7624402c66f71d9162979df5174cdcd099 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 9 Aug 2022 13:21:39 +0200
Subject: [PATCH 14/52] chore: add tzdata to alpine docker image

---
 deploy/docker/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 346d33630..40d03f86a 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -16,6 +16,7 @@ RUN apk add --no-cache \
     libc-dev \
     libstdc++ \
     bash \
+    tzdata \
     jq
 
 COPY . /emqx
@@ -48,7 +49,7 @@ COPY deploy/docker/docker-entrypoint.sh /usr/bin/
 COPY --from=builder /emqx/_build/$EMQX_NAME/rel/emqx /opt/emqx
 
 RUN ln -s /opt/emqx/bin/* /usr/local/bin/
-RUN apk add --no-cache curl ncurses-libs openssl sudo libstdc++ bash
+RUN apk add --no-cache curl ncurses-libs openssl sudo libstdc++ bash tzdata
 
 WORKDIR /opt/emqx
 

From 20d6a53fa188d55fb52b3bc333c10420d5f24652 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 9 Aug 2022 13:23:22 +0200
Subject: [PATCH 15/52] chore: add changelog for tzdata addition in alpine
 image

---
 CHANGES-4.3.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index 5f387222d..8e982cfa6 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -20,6 +20,7 @@ File format:
 ### Enhancements
 
 - Improve error message for LwM2M plugin when object ID is not valid [#8654](https://github.com/emqx/emqx/pull/8654).
+- Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671)
 
 ## v4.3.18
 

From 6630fe6318cbf4ddf486e6536550d198a7d3bbbb Mon Sep 17 00:00:00 2001
From: Rory Z <rory-z@outlook.com>
Date: Wed, 10 Aug 2022 09:54:19 +0800
Subject: [PATCH 16/52] ci: cancel latest tag for docker image

only EMQX 5.0 image can update latest tag
---
 .github/workflows/build_packages.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 84e02bf89..19ed368a6 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -365,9 +365,9 @@ jobs:
       id: meta
       with:
         images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
-        ## only stable tag is latest
+        ## only 5.0 is latest
         flavor: |
-          latest=${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'rc') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
+          latest=false
         tags: |
           type=ref,event=branch
           type=ref,event=pr

From dd5eea5ada7fd1ec5603ca9fd73f90a7426523bd Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 10 Aug 2022 14:34:35 +0200
Subject: [PATCH 17/52] build: bump relup helper to 2.1.0

---
 rebar.config.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rebar.config.erl b/rebar.config.erl
index db761e367..78e1ef94e 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -88,7 +88,7 @@ project_app_dirs() ->
     ["apps/*", alternative_lib_dir() ++ "/*", "."].
 
 plugins(HasElixir) ->
-    [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}}
+    [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.1.0"}}}
     , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}
       %% emqx main project does not require port-compiler
       %% pin at root level for deterministic

From 6a0d2c9d46a2a0708db1b31ea70434ed532754a2 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 11 Aug 2022 08:25:37 +0200
Subject: [PATCH 18/52] fix: add the 'die' function in bin/emqx

---
 bin/emqx | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/bin/emqx b/bin/emqx
index 20cf266b0..7a31f32ce 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -36,6 +36,13 @@ ERTS_LIB_DIR="$ERTS_DIR/../lib"
 # Echo to stderr on errors
 echoerr() { echo "$*" 1>&2; }
 
+die() {
+    set +x
+    echoerr "ERROR: $1"
+    errno=${2:-1}
+    exit "$errno"
+}
+
 assert_node_alive() {
     if ! relx_nodetool "ping" > /dev/null; then
         die "node_is_not_running!" 1

From 3a917605f585582e8f6cd2e9f61778b11c93f52d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 11 Aug 2022 13:15:45 +0200
Subject: [PATCH 19/52] fix(bin/emqx): avoid creating crash dump file when
 checking erlang

---
 bin/emqx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/emqx b/bin/emqx
index 7a31f32ce..13371f9da 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -52,7 +52,8 @@ assert_node_alive() {
 check_erlang_start() {
     # Fix bin permission
     find "$BINDIR" ! -executable -exec chmod a+x {} \;
-    "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()"
+    # set ERL_CRASH_DUMP_BYTES to zero so it will not write a crash dump file
+    env ERL_CRASH_DUMP_BYTES=0 "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()"
 }
 
 if ! check_erlang_start >/dev/null 2>&1; then

From 4531703d1430ba159d8de96861761a8e002b53aa Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 11 Aug 2022 13:17:56 +0200
Subject: [PATCH 20/52] refactor(bin/emqx): move file permission fix out from
 check_erlang_start

---
 bin/emqx | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index 13371f9da..f129a8447 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -33,6 +33,10 @@ export PROGNAME="erl"
 DYNLIBS_DIR="$RUNNER_ROOT_DIR/dynlibs"
 ERTS_LIB_DIR="$ERTS_DIR/../lib"
 
+# Fix bin permission for all erts bin files
+# the 'x' attributes may get lost if the files are extracted from a relup package
+find "$BINDIR" -exec chmod a+x {} \;
+
 # Echo to stderr on errors
 echoerr() { echo "$*" 1>&2; }
 
@@ -50,8 +54,6 @@ assert_node_alive() {
 }
 
 check_erlang_start() {
-    # Fix bin permission
-    find "$BINDIR" ! -executable -exec chmod a+x {} \;
     # set ERL_CRASH_DUMP_BYTES to zero so it will not write a crash dump file
     env ERL_CRASH_DUMP_BYTES=0 "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()"
 }

From 2ef8a985b34adc085bff8ce5fb4399b0afbad764 Mon Sep 17 00:00:00 2001
From: zhouzb <zhouzb@emqx.io>
Date: Fri, 12 Aug 2022 10:41:05 +0800
Subject: [PATCH 21/52] chore: bump vsn to 4.3.18

---
 include/emqx_release.hrl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl
index 77a02177b..305815de1 100644
--- a/include/emqx_release.hrl
+++ b/include/emqx_release.hrl
@@ -29,11 +29,10 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.3.18-beta.1"}).
+-define(EMQX_RELEASE, {opensource, "4.3.18"}).
 
 -else.
 
-
 -endif.
 
 -endif.

From 9064b5acb8248599323b7f7fa8fc4e789fb700a3 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.ru>
Date: Thu, 28 Jul 2022 15:42:57 +0300
Subject: [PATCH 22/52] chore(ci): make apps-version-check.sh accept new apps

---
 scripts/apps-version-check.sh | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh
index ffef46c87..2cb05d3ed 100755
--- a/scripts/apps-version-check.sh
+++ b/scripts/apps-version-check.sh
@@ -19,6 +19,14 @@ while read -r app; do
         app_path="."
     fi
     src_file="$app_path/src/$(basename "$app").app.src"
+
+    old_app_exists=0
+    git show "$latest_release":"$src_file" >/dev/null 2>&1 || old_app_exists="$?"
+    if [ "$old_app_exists" != "0" ]; then
+        echo "$app is new, skipping version check"
+        continue
+    fi
+
     old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')"
     now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')
     if [ "$old_app_version" = "$now_app_version" ]; then
@@ -48,7 +56,7 @@ while read -r app; do
         now_app_version_semver=($(parse_semver "$now_app_version"))
         if  [ "${old_app_version_semver[0]}" = "${now_app_version_semver[0]}" ] && \
             [ "${old_app_version_semver[1]}" = "${now_app_version_semver[1]}" ] && \
-            [ "$(( "${old_app_version_semver[2]}" + 1 ))" = "${now_app_version_semver[2]}" ]; then
+            [ "$(( old_app_version_semver[2] + 1 ))" = "${now_app_version_semver[2]}" ]; then
             true
         else
             echo "$src_file: non-strict semver version bump from $old_app_version to $now_app_version"

From a19fbe214f2971e9971185e705cbff7296101087 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.ru>
Date: Thu, 28 Apr 2022 15:58:17 +0300
Subject: [PATCH 23/52] feat(node_rebalance): implement node evacuation and
 rebalancing

---
 CHANGES-4.3.md                                |   2 +
 apps/emqx_eviction_agent/.gitignore           |  19 +
 apps/emqx_eviction_agent/README.md            |   9 +
 .../etc/emqx_eviction_agent.conf              |   3 +
 .../priv/emqx_eviction_agent.schema           |   0
 apps/emqx_eviction_agent/rebar.config         |   2 +
 .../src/emqx_eviction_agent.app.src           |  18 +
 .../src/emqx_eviction_agent.erl               | 291 ++++++++++++
 .../src/emqx_eviction_agent_api.erl           |  36 ++
 .../src/emqx_eviction_agent_app.erl           |  36 ++
 .../src/emqx_eviction_agent_channel.erl       | 299 +++++++++++++
 .../src/emqx_eviction_agent_cli.erl           |  42 ++
 .../src/emqx_eviction_agent_conn_sup.erl      |  33 ++
 .../src/emqx_eviction_agent_sup.erl           |  43 ++
 .../test/emqx_eviction_agent_SUITE.erl        | 232 ++++++++++
 .../test/emqx_eviction_agent_api_SUITE.erl    |  64 +++
 .../emqx_eviction_agent_channel_SUITE.erl     | 155 +++++++
 .../test/emqx_eviction_agent_cli_SUITE.erl    |  47 ++
 .../test/emqx_eviction_agent_test_helpers.erl |  55 +++
 .../test/emqx_mgmt_api_SUITE.erl              |  56 +--
 .../test/emqx_mgmt_api_test_helpers.erl       |  69 +++
 apps/emqx_node_rebalance/.gitignore           |  19 +
 apps/emqx_node_rebalance/README.md            |   9 +
 .../etc/emqx_node_rebalance.conf              |   3 +
 .../include/emqx_node_rebalance.hrl           |  31 ++
 .../priv/emqx_node_rebalance.schema           |   0
 apps/emqx_node_rebalance/rebar.config         |   2 +
 .../src/emqx_node_rebalance.app.src           |  19 +
 .../src/emqx_node_rebalance.erl               | 414 ++++++++++++++++++
 .../src/emqx_node_rebalance_agent.erl         | 127 ++++++
 .../src/emqx_node_rebalance_api.erl           | 243 ++++++++++
 .../src/emqx_node_rebalance_app.erl           |  34 ++
 .../src/emqx_node_rebalance_cli.erl           | 265 +++++++++++
 .../src/emqx_node_rebalance_evacuation.erl    | 298 +++++++++++++
 ...emqx_node_rebalance_evacuation_persist.erl | 109 +++++
 .../src/emqx_node_rebalance_status.erl        | 225 ++++++++++
 .../src/emqx_node_rebalance_sup.erl           |  44 ++
 .../test/emqx_node_rebalance_SUITE.erl        | 183 ++++++++
 .../test/emqx_node_rebalance_agent_SUITE.erl  | 163 +++++++
 .../test/emqx_node_rebalance_api_SUITE.erl    | 321 ++++++++++++++
 .../test/emqx_node_rebalance_cli_SUITE.erl    | 199 +++++++++
 .../emqx_node_rebalance_evacuation_SUITE.erl  | 194 ++++++++
 ...ode_rebalance_evacuation_persist_SUITE.erl | 107 +++++
 .../src/emqx_rule_engine.app.src              |   2 +-
 .../src/emqx_rule_engine.appup.src            |   6 +-
 data/loaded_plugins.tmpl                      |   2 +
 .../emqx_dashboard/src/emqx_dashboard.app.src |   2 +-
 rebar.config.erl                              |   4 +
 src/emqx.app.src                              |   2 +-
 src/emqx.appup.src                            |  46 +-
 src/emqx_channel.erl                          |  20 +-
 src/emqx_cm.erl                               |  46 +-
 src/emqx_plugins.erl                          |   2 +
 test/emqx_node_helpers.erl                    |  83 ++++
 test/emqx_plugins_SUITE.erl                   |   2 +
 test/emqx_shared_sub_SUITE.erl                |  61 +--
 56 files changed, 4668 insertions(+), 130 deletions(-)
 create mode 100644 apps/emqx_eviction_agent/.gitignore
 create mode 100644 apps/emqx_eviction_agent/README.md
 create mode 100644 apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf
 create mode 100644 apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema
 create mode 100644 apps/emqx_eviction_agent/rebar.config
 create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src
 create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent.erl
 create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl
 create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl
 create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl
 create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl
 create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl
 create mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl
 create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
 create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl
 create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl
 create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl
 create mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl
 create mode 100644 apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl
 create mode 100644 apps/emqx_node_rebalance/.gitignore
 create mode 100644 apps/emqx_node_rebalance/README.md
 create mode 100644 apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf
 create mode 100644 apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl
 create mode 100644 apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema
 create mode 100644 apps/emqx_node_rebalance/rebar.config
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance.erl
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl
 create mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl
 create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl
 create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl
 create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl
 create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl
 create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl
 create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl
 create mode 100644 test/emqx_node_helpers.erl

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index 1345ddc65..10304fc70 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -21,6 +21,7 @@ File format:
 
 - Improve error message for LwM2M plugin when object ID is not valid [#8654](https://github.com/emqx/emqx/pull/8654).
 - Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671)
+- Add node evacuation and cluster rebalancing features [#8597]
 
 ## v4.3.19
 
@@ -55,6 +56,7 @@ File format:
 - HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450]
 - Add check_conf cli to check config format. [#8486]
 - Optimize performance of shared subscription
+- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553)
 
 ## v4.3.16
 
diff --git a/apps/emqx_eviction_agent/.gitignore b/apps/emqx_eviction_agent/.gitignore
new file mode 100644
index 000000000..f1c455451
--- /dev/null
+++ b/apps/emqx_eviction_agent/.gitignore
@@ -0,0 +1,19 @@
+.rebar3
+_*
+.eunit
+*.o
+*.beam
+*.plt
+*.swp
+*.swo
+.erlang.cookie
+ebin
+log
+erl_crash.dump
+.rebar
+logs
+_build
+.idea
+*.iml
+rebar3.crashdump
+*~
diff --git a/apps/emqx_eviction_agent/README.md b/apps/emqx_eviction_agent/README.md
new file mode 100644
index 000000000..f9b8037bf
--- /dev/null
+++ b/apps/emqx_eviction_agent/README.md
@@ -0,0 +1,9 @@
+emqx_eviction_agent
+=====
+
+An OTP library
+
+Build
+-----
+
+    $ rebar3 compile
diff --git a/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf b/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf
new file mode 100644
index 000000000..011b7fb0f
--- /dev/null
+++ b/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf
@@ -0,0 +1,3 @@
+##--------------------------------------------------------------------
+## EMQX Eviction Agent Plugin
+##--------------------------------------------------------------------
diff --git a/apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema b/apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema
new file mode 100644
index 000000000..e69de29bb
diff --git a/apps/emqx_eviction_agent/rebar.config b/apps/emqx_eviction_agent/rebar.config
new file mode 100644
index 000000000..2656fd554
--- /dev/null
+++ b/apps/emqx_eviction_agent/rebar.config
@@ -0,0 +1,2 @@
+{erl_opts, [debug_info]}.
+{deps, []}.
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src
new file mode 100644
index 000000000..ffa28af61
--- /dev/null
+++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src
@@ -0,0 +1,18 @@
+{application, emqx_eviction_agent,
+ [{description, "EMQX Eviction Agent"},
+  {vsn, "4.3.0"},
+  {registered, [emqx_eviction_agent_sup,
+                emqx_eviction_agent,
+                emqx_eviction_agent_conn_sup]},
+  {applications,
+   [kernel,
+    stdlib
+   ]},
+  {mod, {emqx_eviction_agent_app,[]}},
+  {env,[]},
+  {modules, []},
+  {maintainers, ["EMQX Team <contact@emqx.io>"]},
+  {links, [{"Homepage", "https://emqx.io/"},
+           {"Github", "https://github.com/emqx"}
+          ]}
+ ]}.
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl
new file mode 100644
index 000000000..a8cf442cf
--- /dev/null
+++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl
@@ -0,0 +1,291 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent).
+
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/types.hrl").
+
+-include_lib("stdlib/include/qlc.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-export([start_link/0,
+         enable/2,
+         disable/1,
+         status/0,
+         connection_count/0,
+         session_count/0,
+         session_count/1,
+         evict_connections/1,
+         evict_sessions/2,
+         evict_sessions/3,
+         evict_session_channel/3
+        ]).
+
+-behaviour(gen_server).
+
+-export([init/1,
+         handle_call/3,
+         handle_info/2,
+         handle_cast/2,
+         code_change/3
+        ]).
+
+-export([on_connect/2,
+         on_connack/3]).
+
+-export([hook/0,
+         unhook/0]).
+
+-export_type([server_reference/0]).
+
+%%--------------------------------------------------------------------
+%% APIs
+%%--------------------------------------------------------------------
+
+-type server_reference() :: binary() | undefined.
+-type status() :: {enabled, conn_stats()} | disabled.
+-type conn_stats() :: #{connections := non_neg_integer(),
+                        sessions := non_neg_integer()}.
+-type kind() :: atom().
+
+-spec start_link() -> startlink_ret().
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+-spec enable(kind(), server_reference()) -> ok_or_error(eviction_agent_busy).
+enable(Kind, ServerReference) ->
+    gen_server:call(?MODULE, {enable, Kind, ServerReference}).
+
+-spec disable(kind()) -> ok.
+disable(Kind) ->
+    gen_server:call(?MODULE, {disable, Kind}).
+
+-spec status() -> status().
+status() ->
+    case enable_status() of
+        {enabled, _Kind, _ServerReference} ->
+            {enabled, stats()};
+        disabled ->
+           disabled
+    end.
+
+-spec evict_connections(pos_integer()) -> ok_or_error(disabled).
+evict_connections(N) ->
+    case enable_status() of
+        {enabled, _Kind, ServerReference} ->
+            ok = do_evict_connections(N, ServerReference);
+        disabled ->
+            {error, disabled}
+    end.
+
+-spec evict_sessions(pos_integer(), node() | [node()]) -> ok_or_error(disabled).
+evict_sessions(N, Node) when is_atom(Node) ->
+    evict_sessions(N, [Node]);
+evict_sessions(N, Nodes) when is_list(Nodes) andalso length(Nodes) > 0 ->
+    evict_sessions(N, Nodes, any).
+
+-spec evict_sessions(pos_integer(), node() | [node()], atom()) -> ok_or_error(disabled).
+evict_sessions(N, Node, ConnState) when is_atom(Node) ->
+    evict_sessions(N, [Node], ConnState);
+evict_sessions(N, Nodes, ConnState)
+  when is_list(Nodes) andalso length(Nodes) > 0 ->
+    case enable_status() of
+        {enabled, _Kind, _ServerReference} ->
+            ok = do_evict_sessions(N, Nodes, ConnState);
+        disabled ->
+            {error, disabled}
+    end.
+
+%%--------------------------------------------------------------------
+%% gen_server callbacks
+%%--------------------------------------------------------------------
+
+init([]) ->
+    _ = persistent_term:erase(?MODULE),
+    {ok, #{}}.
+
+%% enable
+handle_call({enable, Kind, ServerReference}, _From, St) ->
+    Reply = case enable_status() of
+                disabled ->
+                    ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference});
+                {enabled, Kind, _ServerReference} ->
+                    ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference});
+                {enabled, _OtherKind, _ServerReference} ->
+                    {error, eviction_agent_busy}
+            end,
+    {reply, Reply, St};
+
+%% disable
+handle_call({disable, Kind}, _From, St) ->
+    Reply = case enable_status() of
+                disabled ->
+                    {error, disabled};
+                {enabled, Kind, _ServerReference} ->
+                    _ = persistent_term:erase(?MODULE),
+                    ok;
+                {enabled, _OtherKind, _ServerReference} ->
+                    {error, eviction_agent_busy}
+            end,
+    {reply, Reply, St}.
+
+handle_info(Msg, St) ->
+    ?LOG(warning, "Unknown Msg: ~p, State: ~p", [Msg, St]),
+    {noreply, St}.
+
+handle_cast(Msg, St) ->
+    ?LOG(warning, "Unknown cast Msg: ~p, State: ~p", [Msg, St]),
+    {noreply, St}.
+
+code_change(_Vsn, State, _Extra) ->
+    {ok, State}.
+
+%%--------------------------------------------------------------------
+%% Hook callbacks
+%%--------------------------------------------------------------------
+
+on_connect(_ConnInfo, _Props) ->
+    case enable_status() of
+        {enabled, _Kind, _ServerReference} ->
+            {stop, {error, ?RC_USE_ANOTHER_SERVER}};
+        disabled ->
+            ignore
+    end.
+
+on_connack(#{proto_name := <<"MQTT">>, proto_ver := ?MQTT_PROTO_V5},
+           use_another_server,
+           Props) ->
+    case enable_status() of
+        {enabled, _Kind, ServerReference} ->
+            {ok, Props#{'Server-Reference' => ServerReference}};
+        disabled ->
+            {ok, Props}
+    end;
+on_connack(_ClientInfo, _Reason, Props) ->
+    {ok, Props}.
+
+%%--------------------------------------------------------------------
+%% Hook funcs
+%%--------------------------------------------------------------------
+
+hook() ->
+    ?tp(debug, eviction_agent_hook, #{}),
+    ok = emqx_hooks:put('client.connack', {?MODULE, on_connack, []}),
+    ok = emqx_hooks:put('client.connect', {?MODULE, on_connect, []}).
+
+unhook() ->
+    ?tp(debug, eviction_agent_unhook, #{}),
+    ok = emqx_hooks:del('client.connect', {?MODULE, on_connect}),
+    ok = emqx_hooks:del('client.connack', {?MODULE, on_connack}).
+
+enable_status() ->
+    persistent_term:get(?MODULE, disabled).
+
+% connection management
+stats() ->
+    #{
+        connections => connection_count(),
+        sessions => session_count()
+     }.
+
+connection_table() ->
+    emqx_cm:live_connection_table().
+
+connection_count() ->
+    table_count(connection_table()).
+
+channel_with_session_table(any) ->
+    qlc:q([{ClientId, ConnInfo, ClientInfo}
+           || {ClientId, _, ConnInfo, ClientInfo} <- emqx_cm:channel_with_session_table()]);
+channel_with_session_table(RequiredConnState) ->
+    qlc:q([{ClientId, ConnInfo, ClientInfo}
+           || {ClientId, ConnState, ConnInfo, ClientInfo} <- emqx_cm:channel_with_session_table(),
+              RequiredConnState =:= ConnState]).
+
+session_count() ->
+    session_count(any).
+
+session_count(ConnState) ->
+    table_count(channel_with_session_table(ConnState)).
+
+table_count(QH) ->
+    qlc:fold(fun(_, Acc) -> Acc + 1 end, 0, QH).
+
+take_connections(N) ->
+    ChanQH = qlc:q([ChanPid || {_ClientId, ChanPid} <- connection_table()]),
+    ChanPidCursor = qlc:cursor(ChanQH),
+    ChanPids = qlc:next_answers(ChanPidCursor, N),
+    ok = qlc:delete_cursor(ChanPidCursor),
+    ChanPids.
+
+take_channel_with_sessions(N, ConnState) ->
+    ChanPidCursor = qlc:cursor(channel_with_session_table(ConnState)),
+    Channels = qlc:next_answers(ChanPidCursor, N),
+    ok = qlc:delete_cursor(ChanPidCursor),
+    Channels.
+
+do_evict_connections(N, ServerReference) when N > 0 ->
+    ChanPids = take_connections(N),
+    ok = lists:foreach(
+           fun(ChanPid) ->
+                   disconnect_channel(ChanPid, ServerReference)
+           end,
+           ChanPids).
+
+do_evict_sessions(N, Nodes, ConnState) when N > 0 ->
+    Channels = take_channel_with_sessions(N, ConnState),
+    ok = lists:foreach(
+           fun({ClientId, ConnInfo, ClientInfo}) ->
+                   evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo)
+           end,
+           Channels).
+
+evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo) ->
+    Node = select_random(Nodes),
+    ?LOG(info, "Evicting client=~p to node=~p, conninfo=~p, clientinfo=~p",
+         [ClientId, Node, ConnInfo, ClientInfo]),
+    case rpc:call(Node, ?MODULE, evict_session_channel, [ClientId, ConnInfo, ClientInfo]) of
+        {badrpc, Reason} ->
+            ?LOG(error, "RPC error while evicting client=~p to node=~p: ~p",
+                 [ClientId, Node, Reason]),
+            {error, Reason};
+        {error, Reason} = Error ->
+            ?LOG(error, "Error evicting client=~p to node=~p: ~p",
+                 [ClientId, Node, Reason]),
+            Error;
+        Res -> Res
+    end.
+
+evict_session_channel(ClientId, ConnInfo, ClientInfo) ->
+    ?LOG(info, "Taking up client=~p, conninfo=~p, clientinfo=~p",
+         [ClientId, ConnInfo, ClientInfo]),
+    Result = emqx_eviction_agent_channel:start_supervised(
+               #{conninfo => ConnInfo,
+                 clientinfo => ClientInfo}),
+    ?LOG(info, "Taking up client=~p, result=~p",
+         [ClientId, Result]),
+    Result.
+
+disconnect_channel(ChanPid, ServerReference) ->
+    ChanPid ! {disconnect,
+               ?RC_USE_ANOTHER_SERVER,
+               use_another_server,
+               #{'Server-Reference' => ServerReference}}.
+
+select_random(List) when length(List) > 0 ->
+    lists:nth(rand:uniform(length(List)) , List).
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl
new file mode 100644
index 000000000..426f5978d
--- /dev/null
+++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl
@@ -0,0 +1,36 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_api).
+
+-include_lib("emqx/include/logger.hrl").
+
+-rest_api(#{name   => node_eviction_status,
+            method => 'GET',
+            path   => "/node_eviction/status",
+            func   => status,
+            descr  => "Get node eviction status"}).
+
+-export([status/2]).
+
+status(_Bindings, _Params) ->
+    case emqx_eviction_agent:status() of
+        disabled ->
+            {ok, #{status => disabled}};
+        {enabled, Stats} ->
+            {ok, #{status => enabled,
+                   stats => Stats}}
+    end.
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl
new file mode 100644
index 000000000..c13fcfb0d
--- /dev/null
+++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl
@@ -0,0 +1,36 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_app).
+
+-behaviour(application).
+
+-emqx_plugin(?MODULE).
+
+-export([ start/2
+        , stop/1
+        ]).
+
+start(_Type, _Args) ->
+    Env = application:get_all_env(emqx_eviction_agent),
+    ok = emqx_eviction_agent:hook(),
+    {ok, Sup} = emqx_eviction_agent_sup:start_link(Env),
+    ok = emqx_eviction_agent_cli:load(),
+    {ok, Sup}.
+
+stop(_State) ->
+    ok = emqx_eviction_agent:unhook(),
+    ok = emqx_eviction_agent_cli:unload().
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl
new file mode 100644
index 000000000..1f6fad00f
--- /dev/null
+++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl
@@ -0,0 +1,299 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% MQTT Channel
+-module(emqx_eviction_agent_channel).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/types.hrl").
+
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-logger_header("[Evicted Channel]").
+
+-export([start_link/1,
+         start_supervised/1,
+         call/2,
+         call/3,
+         cast/2,
+         stop/1
+        ]).
+
+-export([init/1,
+         handle_call/3,
+         handle_cast/2,
+         handle_info/2,
+         terminate/2,
+         code_change/3
+        ]).
+
+-type opts() :: #{conninfo := emqx_types:conninfo(),
+                  clientinfo := emqx_types:clientinfo()}.
+
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
+
+-spec start_supervised(opts()) -> startlink_ret().
+start_supervised(#{clientinfo := #{clientid := ClientId}} = Opts) ->
+    RandomId = integer_to_binary(erlang:unique_integer([positive])),
+    Id = <<ClientId/binary, "-", RandomId/binary>>,
+    ChildSpec = #{id => Id,
+                  start => {?MODULE, start_link, [Opts]},
+                  restart => temporary,
+                  shutdown => 5000,
+                  type => worker,
+                  modules => [?MODULE]
+                 },
+    supervisor:start_child(
+        emqx_eviction_agent_conn_sup,
+        ChildSpec).
+
+-spec start_link(opts()) -> startlink_ret().
+start_link(Opts) ->
+    gen_server:start_link(?MODULE, [Opts], []).
+
+-spec cast(pid(), term()) -> ok.
+cast(Pid, Req) ->
+    gen_server:cast(Pid, Req).
+
+-spec call(pid(), term()) -> term().
+call(Pid, Req) ->
+    call(Pid, Req, infinity).
+
+-spec call(pid(), term(), timeout()) -> term().
+call(Pid, Req, Timeout) ->
+    gen_server:call(Pid, Req, Timeout).
+
+-spec stop(pid()) -> ok.
+stop(Pid) ->
+    gen_server:stop(Pid).
+
+%%--------------------------------------------------------------------
+%% gen_server API
+%%--------------------------------------------------------------------
+
+init([#{conninfo := OldConnInfo, clientinfo := #{clientid := ClientId} = OldClientInfo}]) ->
+    process_flag(trap_exit, true),
+    ClientInfo = clientinfo(OldClientInfo),
+    ConnInfo = conninfo(OldConnInfo),
+    case open_session(ConnInfo, ClientInfo) of
+        {ok, Channel0} ->
+            case set_expiry_timer(Channel0) of
+                {ok, Channel1} ->
+                    ?LOG(
+                       info,
+                       "Channel initialized for client=~p on node=~p",
+                       [ClientId, node()]),
+                    {ok, Channel1, hibernate};
+                {error, Reason} ->
+                    {stop, Reason}
+            end;
+        {error, Reason} ->
+            {stop, Reason}
+    end.
+
+handle_call(kick, _From, Channel) ->
+    {stop, kicked, ok, Channel};
+
+handle_call(discard, _From, Channel) ->
+    {stop, discarded, ok, Channel};
+
+handle_call({takeover, 'begin'}, _From, #{session := Session} = Channel) ->
+    {reply, Session, Channel#{takeover => true}};
+
+handle_call({takeover, 'end'}, _From, #{session := Session,
+                                        clientinfo := #{clientid := ClientId},
+                                        pendings := Pendings} = Channel) ->
+    ok = emqx_session:takeover(Session),
+    %% TODO: Should not drain deliver here (side effect)
+    Delivers = emqx_misc:drain_deliver(),
+    AllPendings = lists:append(Delivers, Pendings),
+    ?tp(debug,
+        emqx_channel_takeover_end,
+        #{clientid => ClientId}),
+    {stop, normal, AllPendings, Channel};
+
+handle_call(list_acl_cache, _From, Channel) ->
+    {reply, [], Channel};
+
+handle_call({quota, _Policy}, _From, Channel) ->
+    {reply, ok, Channel};
+
+handle_call(Req, _From, Channel) ->
+    ?LOG(error, "Unexpected call: ~p", [Req]),
+    {reply, ignored, Channel}.
+
+handle_info(Deliver = {deliver, _Topic, _Msg}, Channel) ->
+    Delivers = [Deliver | emqx_misc:drain_deliver()],
+    {noreply, handle_deliver(Delivers, Channel)};
+
+handle_info(expire_session, Channel) ->
+    {stop, expired, Channel};
+
+handle_info(Info, Channel) ->
+    ?LOG(error, "Unexpected info: ~p", [Info]),
+    {noreply, Channel}.
+
+handle_cast(Msg, Channel) ->
+    ?LOG(error, "Unexpected cast: ~p", [Msg]),
+    {noreply, Channel}.
+
+terminate(Reason, #{clientinfo := ClientInfo, session := Session} = Channel) ->
+    ok = cancel_expiry_timer(Channel),
+    emqx_session:terminate(ClientInfo, Reason, Session).
+
+code_change(_OldVsn, Channel, _Extra) ->
+    {ok, Channel}.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+handle_deliver(Delivers,
+               #{takeover := true,
+                 pendings := Pendings,
+                 session := Session,
+                 clientinfo := #{clientid := ClientId} = ClientInfo} = Channel) ->
+    %% NOTE: Order is important here. While the takeover is in
+    %% progress, the session cannot enqueue messages, since it already
+    %% passed on the queue to the new connection in the session state.
+    NPendings = lists:append(
+                  Pendings,
+                  ignore_local(ClientInfo, maybe_nack(Delivers), ClientId, Session)),
+    Channel#{pendings => NPendings};
+
+handle_deliver(Delivers,
+               #{takeover := false,
+                 session := Session,
+                 clientinfo := #{clientid := ClientId} = ClientInfo} = Channel) ->
+    NSession = emqx_session:enqueue(
+                 ClientInfo,
+                 ignore_local(ClientInfo, maybe_nack(Delivers), ClientId, Session),
+                 Session),
+    Channel#{session => NSession}.
+
+cancel_expiry_timer(#{expiry_timer := TRef}) when is_reference(TRef) ->
+    _ = erlang:cancel_timer(TRef),
+    ok;
+cancel_expiry_timer(_) ->
+    ok.
+
+set_expiry_timer(#{conninfo := ConnInfo} = Channel) ->
+    case maps:get(expiry_interval, ConnInfo) of
+        ?UINT_MAX -> {ok, Channel};
+        I when I > 0 ->
+            Timer = erlang:send_after(timer:seconds(I), self(), expire_session),
+            {ok, Channel#{expiry_timer => Timer}};
+        _ ->
+            {error, should_be_expired}
+    end.
+
+open_session(ConnInfo, #{clientid := ClientId} = ClientInfo) ->
+    Channel = channel(ConnInfo, ClientInfo),
+    case emqx_cm:open_session(false, ClientInfo, ConnInfo) of
+        {ok, #{present := false}} ->
+            ?LOG(info, "No session for clientid=~p", [ClientId]),
+            {error, no_session};
+        {ok, #{session := Session, present := true, pendings := Pendings0}} ->
+            ?LOG(info, "Session opened for client=~p on node=~p", [ClientId, node()]),
+            Pendings1 = lists:usort(lists:append(Pendings0, emqx_misc:drain_deliver())),
+            NSession = emqx_session:enqueue(
+                         ClientInfo,
+                         ignore_local(
+                           ClientInfo,
+                           maybe_nack(Pendings1),
+                           ClientId,
+                           Session),
+                         Session),
+            NChannel = Channel#{session => NSession},
+            ok = emqx_cm:insert_channel_info(ClientId, info(NChannel), []),
+            ?LOG(info, "Channel info updated for client=~p on node=~p", [ClientId, node()]),
+            {ok, NChannel};
+        {error, Reason} = Error ->
+            ?LOG(error, "Failed to open session due to ~p", [Reason]),
+            Error
+    end.
+
+conninfo(OldConnInfo) ->
+    DisconnectedAt = maps:get(disconnected_at, OldConnInfo, erlang:system_time(millisecond)),
+    ConnInfo0 = maps:with(
+                  [socktype,
+                   sockname,
+                   peername,
+                   peercert,
+                   clientid,
+                   clean_start,
+                   receive_maximum,
+                   expiry_interval],
+                  OldConnInfo),
+    ConnInfo0#{
+      conn_mod => ?MODULE,
+      connected => false,
+      disconnected_at => DisconnectedAt
+     }.
+
+clientinfo(OldClientInfo) ->
+    maps:with(
+      [zone,
+       protocol,
+       peerhost,
+       sockport,
+       clientid,
+       username,
+       is_bridge,
+       is_superuser,
+       mountpoint],
+      OldClientInfo).
+
+channel(ConnInfo, ClientInfo) ->
+    #{conninfo => ConnInfo,
+      clientinfo => ClientInfo,
+      expiry_timer => undefined,
+      takeover => false,
+      resuming => false,
+      pendings => []
+     }.
+
+info(Channel) ->
+    #{conninfo => maps:get(conninfo, Channel, undefined),
+      clientinfo => maps:get(clientinfo, Channel, undefined),
+      session => maps:get(session, Channel, undefined),
+      conn_state => disconnected
+     }.
+
+ignore_local(ClientInfo, Delivers, Subscriber, Session) ->
+    Subs = emqx_session:info(subscriptions, Session),
+    lists:dropwhile(fun({deliver, Topic, #message{from = Publisher} = Msg}) ->
+                        case maps:find(Topic, Subs) of
+                            {ok, #{nl := 1}} when Subscriber =:= Publisher ->
+                                ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, no_local]),
+                                ok = emqx_metrics:inc('delivery.dropped'),
+                                ok = emqx_metrics:inc('delivery.dropped.no_local'),
+                                true;
+                            _ ->
+                                false
+                        end
+                    end, Delivers).
+
+maybe_nack(Delivers) ->
+    lists:filter(fun not_nacked/1, Delivers).
+
+not_nacked({deliver, _Topic, Msg}) ->
+    not (emqx_shared_sub:is_ack_required(Msg)
+         andalso (ok == emqx_shared_sub:nack_no_connection(Msg))).
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl
new file mode 100644
index 000000000..632f8e480
--- /dev/null
+++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl
@@ -0,0 +1,42 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_cli).
+
+%% APIs
+-export([ load/0
+        , unload/0
+        , cli/1
+        ]).
+
+load() ->
+    emqx_ctl:register_command(eviction, {?MODULE, cli}, []).
+
+unload() ->
+    emqx_ctl:unregister_command(eviction).
+
+cli(["status"]) ->
+    case emqx_eviction_agent:status() of
+        disabled ->
+            emqx_ctl:print("Eviction status: disabled~n");
+        {enabled, _Stats} ->
+            emqx_ctl:print("Eviction status: enabled~n")
+    end;
+
+cli(_) ->
+    emqx_ctl:usage(
+      [{"eviction status",
+        "Get current node eviction status"}]).
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl
new file mode 100644
index 000000000..2b11ce9d1
--- /dev/null
+++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl
@@ -0,0 +1,33 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_conn_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/1]).
+
+-export([init/1]).
+
+start_link(Env) ->
+	supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]).
+
+init([_Env]) ->
+    Childs = [],
+    {ok, {
+       {one_for_one, 10, 3600},
+       Childs}
+    }.
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl
new file mode 100644
index 000000000..9e28c2251
--- /dev/null
+++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl
@@ -0,0 +1,43 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/1]).
+
+-export([init/1]).
+
+start_link(Env) ->
+	supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]).
+
+init([_Env]) ->
+    Childs = [child_spec(worker, emqx_eviction_agent, []),
+              child_spec(supervisor, emqx_eviction_agent_conn_sup, [#{}])],
+    {ok, {
+       {one_for_one, 10, 3600},
+       Childs}
+    }.
+
+child_spec(Type, Mod, Args) ->
+    #{id => Mod,
+      start => {Mod, start_link, Args},
+      restart => permanent,
+      shutdown => 5000,
+      type => Type,
+      modules => [Mod]
+     }.
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
new file mode 100644
index 000000000..b77c0dee8
--- /dev/null
+++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
@@ -0,0 +1,232 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-import(emqx_eviction_agent_test_helpers,
+        [emqtt_connect/0, emqtt_connect/2]).
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent]),
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_ct_helpers:stop_apps([emqx_eviction_agent]).
+
+init_per_testcase(t_explicit_session_takeover, Config) ->
+    _ = emqx_eviction_agent:disable(test_eviction),
+    Node = emqx_node_helpers:start_slave(
+             evacuate1,
+             #{start_apps => [emqx, emqx_eviction_agent]}),
+    [{evacuate_node, Node} | Config];
+init_per_testcase(_TestCase, Config) ->
+    _ = emqx_eviction_agent:disable(test_eviction),
+    Config.
+
+end_per_testcase(t_explicit_session_takeover, Config) ->
+    _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)),
+    _ = emqx_eviction_agent:disable(test_eviction);
+end_per_testcase(_TestCase, _Config) ->
+    _ = emqx_eviction_agent:disable(test_eviction).
+
+t_enable_disable(_Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    ?assertMatch(
+       disabled,
+       emqx_eviction_agent:status()),
+
+    {ok, C0} = emqtt_connect(),
+    ok = emqtt:disconnect(C0),
+
+    ok = emqx_eviction_agent:enable(test_eviction, undefined),
+
+    ?assertMatch(
+       {error, eviction_agent_busy},
+       emqx_eviction_agent:enable(bar, undefined)),
+
+    ?assertMatch(
+       ok,
+       emqx_eviction_agent:enable(test_eviction, <<"srv">>)),
+
+    ?assertMatch(
+        {enabled, #{}},
+        emqx_eviction_agent:status()),
+
+    ?assertMatch(
+       {error, {use_another_server, #{}}},
+       emqtt_connect()),
+
+    ?assertMatch(
+        {error, eviction_agent_busy},
+        emqx_eviction_agent:disable(bar)),
+
+    ?assertMatch(
+        ok,
+        emqx_eviction_agent:disable(test_eviction)),
+
+    ?assertMatch(
+        {error, disabled},
+        emqx_eviction_agent:disable(test_eviction)),
+
+    ?assertMatch(
+       disabled,
+       emqx_eviction_agent:status()),
+
+    {ok, C1} = emqtt_connect(),
+    ok = emqtt:disconnect(C1).
+
+
+t_evict_connections_status(_Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    {ok, _C} = emqtt_connect(),
+
+    {error, disabled} = emqx_eviction_agent:evict_connections(1),
+
+    ok = emqx_eviction_agent:enable(test_eviction, undefined),
+
+    ?assertMatch(
+       {enabled, #{connections := 1, sessions := _}},
+       emqx_eviction_agent:status()),
+
+    ok = emqx_eviction_agent:evict_connections(1),
+
+    ct:sleep(100),
+
+    ?assertMatch(
+       {enabled, #{connections := 0, sessions := _}},
+       emqx_eviction_agent:status()),
+
+    ok = emqx_eviction_agent:disable(test_eviction).
+
+
+t_explicit_session_takeover(Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    {ok, C0} = emqtt_connect(<<"client_with_session">>, false),
+    {ok, _, _} = emqtt:subscribe(C0, <<"t1">>),
+
+    ok = emqx_eviction_agent:enable(test_eviction, undefined),
+
+    ?assertEqual(
+       1,
+       emqx_eviction_agent:connection_count()),
+
+    ok = emqx_eviction_agent:evict_connections(1),
+
+    receive
+        {'EXIT', C0, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok
+    after 1000 ->
+              ?assert(false, "Connection not evicted")
+    end,
+
+    ?assertEqual(
+       0,
+       emqx_eviction_agent:connection_count()),
+
+    ?assertEqual(
+       1,
+       emqx_eviction_agent:session_count()),
+
+    %% First, evacuate to the same node
+
+    ?check_trace(
+       ?wait_async_action(
+          emqx_eviction_agent:evict_sessions(1, node()),
+          #{?snk_kind := emqx_channel_takeover_end},
+          1000),
+       fun(_Result, Trace) ->
+        ?assertMatch(
+           [#{clientid := <<"client_with_session">>} | _ ],
+           ?of_kind(emqx_channel_takeover_end, Trace))
+       end),
+
+    ok = emqx_eviction_agent:disable(test_eviction),
+    ok = connect_and_publish(<<"t1">>, <<"MessageToEvictedSession1">>),
+    ok = emqx_eviction_agent:enable(test_eviction, undefined),
+
+    %% Evacuate to another node
+
+    TargetNodeForEvacuation = ?config(evacuate_node, Config),
+    ?check_trace(
+       ?wait_async_action(
+          emqx_eviction_agent:evict_sessions(1, TargetNodeForEvacuation),
+          #{?snk_kind := emqx_channel_takeover_end},
+          1000),
+       fun(_Result, Trace) ->
+        ?assertMatch(
+           [#{clientid := <<"client_with_session">>} | _ ],
+           ?of_kind(emqx_channel_takeover_end, Trace))
+       end),
+
+    ?assertEqual(
+       0,
+       emqx_eviction_agent:session_count()),
+
+    ?assertEqual(
+       1,
+       rpc:call(TargetNodeForEvacuation, emqx_eviction_agent, session_count, [])),
+
+    ok = emqx_eviction_agent:disable(test_eviction),
+
+    ct:pal("evicted chann info: ~p", [emqx_cm:get_chan_info(<<"client_with_session">>)]),
+
+    ok = connect_and_publish(<<"t1">>, <<"MessageToEvictedSession2">>),
+    ct:sleep(100),
+
+    {ok, C2} = emqtt_connect(<<"client_with_session">>, false),
+
+    ok = assert_receive_publish(
+           [#{payload => <<"MessageToEvictedSession1">>, topic => <<"t1">>},
+            #{payload => <<"MessageToEvictedSession2">>, topic => <<"t1">>}]),
+    ok = emqtt:disconnect(C2).
+
+t_disable_on_restart(_Config) ->
+    ok = emqx_eviction_agent:enable(test_eviction, undefined),
+
+    ok = supervisor:terminate_child(emqx_eviction_agent_sup, emqx_eviction_agent),
+    {ok, _} = supervisor:restart_child(emqx_eviction_agent_sup, emqx_eviction_agent),
+
+    ?assertEqual(
+        disabled,
+        emqx_eviction_agent:status()).
+
+assert_receive_publish([]) -> ok;
+assert_receive_publish([#{payload := Msg, topic := Topic} | Rest]) ->
+    receive
+        {publish, #{payload := Msg,
+                    topic := Topic}} ->
+            assert_receive_publish(Rest)
+    after 1000 ->
+              ?assert(false, "Message `" ++ binary_to_list(Msg) ++ "` is lost")
+    end.
+
+connect_and_publish(Topic, Message) ->
+    {ok, C} = emqtt_connect(),
+    emqtt:publish(C, Topic, Message),
+    ok = emqtt:disconnect(C).
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl
new file mode 100644
index 000000000..f9585c0e6
--- /dev/null
+++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl
@@ -0,0 +1,64 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_api_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-import(emqx_mgmt_api_test_helpers,
+        [request_api/3,
+         auth_header_/0,
+         api_path/1]).
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_management]),
+    Config.
+
+end_per_suite(Config) ->
+    emqx_ct_helpers:stop_apps([emqx_management, emqx_eviction_agent]),
+    Config.
+
+t_status(_Config) ->
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"disabled">>}},
+       api_get(["node_eviction", "status"])),
+
+    ok = emqx_eviction_agent:enable(apitest, undefined),
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"enabled">>,
+              <<"stats">> := #{}}},
+        api_get(["node_eviction", "status"])),
+
+    ok = emqx_eviction_agent:disable(apitest),
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"disabled">>}},
+       api_get(["node_eviction", "status"])).
+
+api_get(Path) ->
+    case request_api(get, api_path(Path), auth_header_()) of
+        {ok, ResponseBody} ->
+            {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])};
+        {error, _} = Error -> Error
+    end.
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl
new file mode 100644
index 000000000..cbbf1be5a
--- /dev/null
+++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl
@@ -0,0 +1,155 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_channel_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-define(CLIENT_ID, <<"client_with_session">>).
+
+-import(emqx_eviction_agent_test_helpers,
+        [emqtt_connect/2]).
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent]),
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_ct_helpers:stop_apps([emqx_eviction_agent]).
+
+t_start_no_session(_Config) ->
+    Opts = #{clientinfo => #{clientid => ?CLIENT_ID,
+                             zone => internal},
+             conninfo => #{clientid => ?CLIENT_ID,
+                           receive_maximum => 32}},
+    ?assertMatch(
+       {error, {no_session, _}},
+       emqx_eviction_agent_channel:start_supervised(Opts)).
+
+t_start_no_expire(_Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    _ = emqtt_connect(?CLIENT_ID, false),
+
+    Opts = #{clientinfo => #{clientid => ?CLIENT_ID,
+                             zone => internal},
+             conninfo => #{clientid => ?CLIENT_ID,
+                           receive_maximum => 32,
+                           expiry_interval => 0}},
+    ?assertMatch(
+       {error, {should_be_expired, _}},
+       emqx_eviction_agent_channel:start_supervised(Opts)).
+
+t_start_infinite_expire(_Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    _ = emqtt_connect(?CLIENT_ID, false),
+
+    Opts = #{clientinfo => #{clientid => ?CLIENT_ID,
+                             zone => internal},
+             conninfo => #{clientid => ?CLIENT_ID,
+                           receive_maximum => 32,
+                           expiry_interval => ?UINT_MAX}},
+    ?assertMatch(
+       {ok, _},
+       emqx_eviction_agent_channel:start_supervised(Opts)).
+
+t_kick(_Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    _ = emqtt_connect(?CLIENT_ID, false),
+    Opts = evict_session_opts(?CLIENT_ID),
+
+    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts),
+
+    ?assertEqual(
+       ok,
+       emqx_eviction_agent_channel:call(Pid, kick)).
+
+t_discard(_Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    _ = emqtt_connect(?CLIENT_ID, false),
+    Opts = evict_session_opts(?CLIENT_ID),
+
+    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts),
+
+    ?assertEqual(
+       ok,
+       emqx_eviction_agent_channel:call(Pid, discard)).
+
+t_stop(_Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    _ = emqtt_connect(?CLIENT_ID, false),
+    Opts = evict_session_opts(?CLIENT_ID),
+
+    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts),
+
+    ?assertEqual(
+       ok,
+       emqx_eviction_agent_channel:stop(Pid)).
+
+
+t_ignored_calls(_Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    _ = emqtt_connect(?CLIENT_ID, false),
+    Opts = evict_session_opts(?CLIENT_ID),
+
+    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts),
+
+    ok = emqx_eviction_agent_channel:cast(Pid, unknown),
+    Pid ! unknown,
+
+    ?assertEqual(
+       [],
+       emqx_eviction_agent_channel:call(Pid, list_acl_cache)),
+
+    ?assertEqual(
+       ok,
+       emqx_eviction_agent_channel:call(Pid, {quota, quota})),
+
+    ?assertEqual(
+       ignored,
+       emqx_eviction_agent_channel:call(Pid, unknown)).
+
+t_expire(_Config) ->
+    erlang:process_flag(trap_exit, true),
+
+    _ = emqtt_connect(?CLIENT_ID, false),
+    #{conninfo := ConnInfo} = Opts0 = evict_session_opts(?CLIENT_ID),
+    Opts1 = Opts0#{conninfo => ConnInfo#{expiry_interval => 1}},
+
+    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts1),
+
+    ct:sleep(1500),
+
+    ?assertNot(is_process_alive(Pid)).
+
+evict_session_opts(ClientId) ->
+    maps:with(
+      [conninfo, clientinfo],
+      emqx_cm:get_chan_info(ClientId)).
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl
new file mode 100644
index 000000000..9c05cdb21
--- /dev/null
+++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl
@@ -0,0 +1,47 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_cli_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent]),
+    Config.
+
+end_per_suite(Config) ->
+    _ = emqx_eviction_agent:disable(foo),
+    emqx_ct_helpers:stop_apps([emqx_eviction_agent]),
+    Config.
+
+t_status(_Config) ->
+    %% usage
+    ok = emqx_eviction_agent_cli:cli(["foobar"]),
+
+    %% status
+    ok = emqx_eviction_agent_cli:cli(["status"]),
+
+    ok = emqx_eviction_agent:enable(foo, undefined),
+
+    %% status
+    ok = emqx_eviction_agent_cli:cli(["status"]).
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl
new file mode 100644
index 000000000..68b86f059
--- /dev/null
+++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl
@@ -0,0 +1,55 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_eviction_agent_test_helpers).
+
+-export([emqtt_connect/0,
+         emqtt_connect/2,
+         emqtt_connect_many/1,
+         emqtt_try_connect/0]).
+
+emqtt_connect() ->
+    emqtt_connect(<<"client1">>, true).
+
+emqtt_connect(ClientId, CleanStart) ->
+    {ok, C} = emqtt:start_link(
+                [{clientid, ClientId},
+                 {clean_start, CleanStart},
+                 {proto_ver, v5},
+                 {properties, #{'Session-Expiry-Interval' => 600}}
+                ]),
+    case emqtt:connect(C) of
+        {ok, _} -> {ok, C};
+        {error, _} = Error -> Error
+    end.
+
+emqtt_connect_many(Count) ->
+    lists:map(
+      fun(N) ->
+              NBin = integer_to_binary(N),
+              ClientId = <<"client-", NBin/binary>>,
+              {ok, C} = emqtt_connect(ClientId, false),
+              C
+      end,
+      lists:seq(1, Count)).
+
+emqtt_try_connect() ->
+    case emqtt_connect() of
+        {ok, C} ->
+            emqtt:disconnect(C),
+            ok;
+        {error, _} = Error -> Error
+    end.
diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
index cfed74c16..fda5c2adf 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -25,13 +25,12 @@
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx_management/include/emqx_mgmt.hrl").
 
--define(CONTENT_TYPE, "application/x-www-form-urlencoded").
-
--define(HOST, "http://127.0.0.1:8081/").
-
--define(API_VERSION, "v4").
-
--define(BASE_PATH, "api").
+-import(emqx_mgmt_api_test_helpers,
+        [request_api/3,
+         request_api/4,
+         request_api/5,
+         auth_header_/0,
+         api_path/1]).
 
 all() ->
     emqx_ct:all(?MODULE).
@@ -657,49 +656,6 @@ t_data_import_content(_) ->
     application:stop(emqx_rule_engine),
     application:stop(emqx_dashboard).
 
-request_api(Method, Url, Auth) ->
-    request_api(Method, Url, [], Auth, []).
-
-request_api(Method, Url, QueryParams, Auth) ->
-    request_api(Method, Url, QueryParams, Auth, []).
-
-request_api(Method, Url, QueryParams, Auth, []) ->
-    NewUrl = case QueryParams of
-                 "" -> Url;
-                 _ -> Url ++ "?" ++ QueryParams
-             end,
-    do_request_api(Method, {NewUrl, [Auth]});
-request_api(Method, Url, QueryParams, Auth, Body) ->
-    NewUrl = case QueryParams of
-                 "" -> Url;
-                 _ -> Url ++ "?" ++ QueryParams
-             end,
-    do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}).
-
-do_request_api(Method, Request)->
-    ct:pal("Method: ~p, Request: ~p", [Method, Request]),
-    case httpc:request(Method, Request, [], []) of
-        {error, socket_closed_remotely} ->
-            {error, socket_closed_remotely};
-        {ok, {{"HTTP/1.1", Code, _}, _, Return} }
-            when Code =:= 200 orelse Code =:= 201 ->
-            {ok, Return};
-        {ok, {Reason, _, _}} ->
-            {error, Reason}
-    end.
-
-auth_header_() ->
-    AppId = <<"admin">>,
-    AppSecret = <<"public">>,
-    auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)).
-
-auth_header_(User, Pass) ->
-    Encoded = base64:encode_to_string(lists:append([User,":",Pass])),
-    {"Authorization","Basic " ++ Encoded}.
-
-api_path(Parts)->
-    ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts).
-
 filter(List, Key, Value) ->
     lists:filter(fun(Item) ->
         maps:get(Key, Item) == Value
diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl b/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl
new file mode 100644
index 000000000..a943ca760
--- /dev/null
+++ b/apps/emqx_management/test/emqx_mgmt_api_test_helpers.erl
@@ -0,0 +1,69 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_mgmt_api_test_helpers).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-define(HOST, "http://127.0.0.1:8081/").
+
+-define(API_VERSION, "v4").
+
+-define(BASE_PATH, "api").
+
+request_api(Method, Url, Auth) ->
+    request_api(Method, Url, [], Auth, []).
+
+request_api(Method, Url, QueryParams, Auth) ->
+    request_api(Method, Url, QueryParams, Auth, []).
+
+request_api(Method, Url, QueryParams, Auth, []) ->
+    NewUrl = case QueryParams of
+                 "" -> Url;
+                 _ -> Url ++ "?" ++ QueryParams
+             end,
+    do_request_api(Method, {NewUrl, [Auth]});
+request_api(Method, Url, QueryParams, Auth, Body) ->
+    NewUrl = case QueryParams of
+                 "" -> Url;
+                 _ -> Url ++ "?" ++ QueryParams
+             end,
+    do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}).
+
+do_request_api(Method, Request)->
+    ct:pal("Method: ~p, Request: ~p", [Method, Request]),
+    case httpc:request(Method, Request, [], []) of
+        {error, socket_closed_remotely} ->
+            {error, socket_closed_remotely};
+        {ok, {{"HTTP/1.1", Code, _}, _, Return} }
+            when Code =:= 200 orelse Code =:= 201 ->
+            {ok, Return};
+        {ok, {Reason, _, _}} ->
+            {error, Reason}
+    end.
+
+auth_header_() ->
+    AppId = <<"admin">>,
+    AppSecret = <<"public">>,
+    auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)).
+
+auth_header_(User, Pass) ->
+    Encoded = base64:encode_to_string(lists:append([User,":",Pass])),
+    {"Authorization","Basic " ++ Encoded}.
+
+api_path(Parts)->
+    ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts).
diff --git a/apps/emqx_node_rebalance/.gitignore b/apps/emqx_node_rebalance/.gitignore
new file mode 100644
index 000000000..f1c455451
--- /dev/null
+++ b/apps/emqx_node_rebalance/.gitignore
@@ -0,0 +1,19 @@
+.rebar3
+_*
+.eunit
+*.o
+*.beam
+*.plt
+*.swp
+*.swo
+.erlang.cookie
+ebin
+log
+erl_crash.dump
+.rebar
+logs
+_build
+.idea
+*.iml
+rebar3.crashdump
+*~
diff --git a/apps/emqx_node_rebalance/README.md b/apps/emqx_node_rebalance/README.md
new file mode 100644
index 000000000..2e56f62cd
--- /dev/null
+++ b/apps/emqx_node_rebalance/README.md
@@ -0,0 +1,9 @@
+emqx_node_rebalance
+=====
+
+An OTP library
+
+Build
+-----
+
+    $ rebar3 compile
diff --git a/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf b/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf
new file mode 100644
index 000000000..8ace22435
--- /dev/null
+++ b/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf
@@ -0,0 +1,3 @@
+##--------------------------------------------------------------------
+## EMQX Node Rebalance Plugin
+##--------------------------------------------------------------------
diff --git a/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl b/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl
new file mode 100644
index 000000000..1903b87cc
--- /dev/null
+++ b/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl
@@ -0,0 +1,31 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-define(DEFAULT_CONN_EVICT_RATE, 500).
+-define(DEFAULT_SESS_EVICT_RATE, 500).
+
+-define(DEFAULT_WAIT_HEALTH_CHECK, 60). %% sec
+-define(DEFAULT_WAIT_TAKEOVER, 60). %% sec
+
+-define(DEFAULT_ABS_CONN_THRESHOLD, 1000).
+-define(DEFAULT_ABS_SESS_THRESHOLD, 1000).
+
+-define(DEFAULT_REL_CONN_THRESHOLD, 1.1).
+-define(DEFAULT_REL_SESS_THRESHOLD, 1.1).
+
+-define(EVICT_INTERVAL, 1000).
+
+-define(EVACUATION_FILENAME, <<".evacuation">>).
diff --git a/apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema b/apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema
new file mode 100644
index 000000000..e69de29bb
diff --git a/apps/emqx_node_rebalance/rebar.config b/apps/emqx_node_rebalance/rebar.config
new file mode 100644
index 000000000..2656fd554
--- /dev/null
+++ b/apps/emqx_node_rebalance/rebar.config
@@ -0,0 +1,2 @@
+{erl_opts, [debug_info]}.
+{deps, []}.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src
new file mode 100644
index 000000000..761cd1d6f
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src
@@ -0,0 +1,19 @@
+{application, emqx_node_rebalance,
+ [{description, "EMQX Node Rebalance"},
+  {vsn, "4.3.0"},
+  {registered, [emqx_node_rebalance_sup,
+                emqx_node_rebalance,
+                emqx_node_rebalance_agent,
+                emqx_node_rebalance_evacuation]},
+  {applications,
+   [kernel,
+    stdlib
+   ]},
+  {mod, {emqx_node_rebalance_app,[]}},
+  {env,[]},
+  {modules, []},
+  {maintainers, ["EMQX Team <contact@emqx.io>"]},
+  {links, [{"Homepage", "https://emqx.io/"},
+           {"Github", "https://github.com/emqx"}
+          ]}
+ ]}.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl
new file mode 100644
index 000000000..1edbbbe3b
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl
@@ -0,0 +1,414 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance).
+
+-include("emqx_node_rebalance.hrl").
+
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/types.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-export([start/1,
+         status/0,
+         status/1,
+         stop/0
+        ]).
+
+-export([start_link/0]).
+
+-behavior(gen_statem).
+
+-export([init/1,
+         callback_mode/0,
+         handle_event/4,
+         code_change/4
+        ]).
+
+-export([is_node_available/0,
+         available_nodes/1,
+         connection_count/0,
+         session_count/0,
+         disconnected_session_count/0]).
+
+%%--------------------------------------------------------------------
+%% APIs
+%%--------------------------------------------------------------------
+
+-type start_opts() :: #{conn_evict_rate => pos_integer(),
+                        sess_evict_rate => pos_integer(),
+                        wait_health_check => pos_integer(),
+                        wait_takeover => pos_integer(),
+                        abs_conn_threshold => pos_integer(),
+                        rel_conn_threshold => number(),
+                        abs_sess_threshold => pos_integer(),
+                        rel_sess_threshold => number(),
+                        nodes => [node()]
+                       }.
+-type start_error() :: already_started | [{node(), term()}].
+
+
+-spec start(start_opts()) -> ok_or_error(start_error()).
+start(StartOpts) ->
+    Opts = maps:merge(default_opts(), StartOpts),
+    gen_statem:call(?MODULE, {start, Opts}).
+
+-spec stop() -> ok_or_error(not_started).
+stop() ->
+    gen_statem:call(?MODULE, stop).
+
+-spec status() -> disabled | {enabled, map()}.
+status() ->
+    gen_statem:call(?MODULE, status).
+
+-spec status(pid()) -> disabled | {enabled, map()}.
+status(Pid) ->
+    gen_statem:call(Pid, status).
+
+-spec start_link() -> startlink_ret().
+start_link() ->
+    gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+-spec available_nodes(list(node())) -> list(node()).
+available_nodes(Nodes) when is_list(Nodes) ->
+    {Available, _} = rpc:multicall(Nodes, ?MODULE, is_node_available, []),
+    lists:filter(fun is_atom/1, Available).
+
+%%--------------------------------------------------------------------
+%% gen_statem callbacks
+%%--------------------------------------------------------------------
+
+callback_mode() -> handle_event_function.
+
+%% states: disabled, wait_health_check, evicting_conns, wait_takeover, evicting_sessions
+
+init([]) ->
+    ?tp(debug, emqx_node_rebalance_started, #{}),
+    {ok, disabled, #{}}.
+
+%% start
+handle_event({call, From},
+             {start, #{wait_health_check := WaitHealthCheck} = Opts},
+             disabled,
+             #{} = Data) ->
+    case enable_rebalance(Data#{opts => Opts}) of
+        {ok, NewData} ->
+            ?LOG(warning, "Node rebalance enabled: ~p", [Opts]),
+            {next_state,
+             wait_health_check,
+             NewData,
+             [{state_timeout, seconds(WaitHealthCheck), evict_conns},
+              {reply, From, ok}]};
+        {error, Reason} ->
+            ?LOG(warning, "Node rebalance enabling failed: ~p", [Reason]),
+            {keep_state_and_data,
+             [{reply, From, {error, Reason}}]}
+    end;
+handle_event({call, From}, {start, _Opts}, _State, #{}) ->
+    {keep_state_and_data,
+     [{reply, From, {error, already_started}}]};
+
+%% stop
+handle_event({call, From}, stop, disabled, #{}) ->
+    {keep_state_and_data,
+     [{reply, From, {error, not_started}}]};
+handle_event({call, From}, stop, _State, Data) ->
+    ok = disable_rebalance(Data),
+    ?LOG(warning, "Node rebalance stopped"),
+    {next_state,
+     disabled,
+     deinit(Data),
+     [{reply, From, ok}]};
+
+%% status
+handle_event({call, From}, status, disabled, #{}) ->
+    {keep_state_and_data,
+     [{reply, From, disabled}]};
+handle_event({call, From}, status, State, Data) ->
+    Stats = get_stats(State, Data),
+    {keep_state_and_data,
+     [{reply, From, {enabled, Stats#{state => State,
+                                     coordinator_node => node()}}}]};
+
+%% conn eviction
+handle_event(state_timeout,
+             evict_conns,
+             wait_health_check,
+             Data) ->
+    ?LOG(warning, "Node rebalance wait_health_check over"),
+    {next_state,
+     evicting_conns,
+     Data,
+     [{state_timeout, 0, evict_conns}]};
+
+handle_event(state_timeout,
+             evict_conns,
+             evicting_conns,
+             #{opts := #{wait_takeover := WaitTakeover,
+                         evict_interval := EvictInterval}} = Data) ->
+    case evict_conns(Data) of
+        ok ->
+            ?LOG(warning, "Node rebalance evict_conns over"),
+            {next_state,
+             wait_takeover,
+             Data,
+             [{state_timeout, seconds(WaitTakeover), evict_sessions}]};
+        {continue, NewData} ->
+            {keep_state,
+             NewData,
+             [{state_timeout, EvictInterval, evict_conns}]}
+    end;
+
+handle_event(state_timeout,
+             evict_sessions,
+             wait_takeover,
+             Data) ->
+    ?LOG(warning, "Node rebalance wait_takeover over"),
+    {next_state,
+     evicting_sessions,
+     Data,
+     [{state_timeout, 0, evict_sessions}]};
+
+handle_event(state_timeout,
+             evict_sessions,
+             evicting_sessions,
+             #{opts := #{evict_interval := EvictInterval}} = Data) ->
+    case evict_sessions(Data) of
+        ok ->
+            ?tp(debug, emqx_node_rebalance_evict_sess_over, #{}),
+            ?LOG(warning, "Node rebalance evict_sess over"),
+            ok = disable_rebalance(Data),
+            ?LOG(warning, "Rebalance finished successfully"),
+            {next_state,
+             disabled,
+             deinit(Data)};
+        {continue, NewData} ->
+            {keep_state,
+             NewData,
+             [{state_timeout, EvictInterval, evict_sessions}]}
+    end;
+
+handle_event({call, From}, Msg, State, Data) ->
+    ?LOG(warning, "Unknown call: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
+    {keep_state_and_data,
+     [{reply, From, ignored}]};
+
+handle_event(info, Msg, State, Data) ->
+    ?LOG(warning, "Unknown Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
+    keep_state_and_data;
+
+handle_event(cast, Msg, State, Data) ->
+    ?LOG(warning, "Unknown cast Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
+    keep_state_and_data.
+
+code_change(_Vsn, State, Data, _Extra) ->
+    {ok, State, Data}.
+
+%%--------------------------------------------------------------------
+%% internal funs
+%%--------------------------------------------------------------------
+
+enable_rebalance(#{opts := Opts} = Data) ->
+    Nodes = maps:get(nodes, Opts),
+    ConnCounts = multicall(Nodes, {?MODULE, connection_count, []}),
+    SessCounts = multicall(Nodes, {?MODULE, session_count, []}),
+    {_, Counts} = lists:unzip(ConnCounts),
+    Avg = avg(Counts),
+    {DonorCounts,
+     RecipientCounts} = lists:partition(
+                          fun({_Node, Count}) ->
+                                  Count >= Avg
+                          end,
+                          ConnCounts),
+    ?LOG(warning, "Enabling rebalance: ConnCounts=~p, DonorCounts=~p, RecipientCounts=~p",
+         [ConnCounts, DonorCounts, RecipientCounts]),
+    {DonorNodes, _} = lists:unzip(DonorCounts),
+    {RecipientNodes, _} = lists:unzip(RecipientCounts),
+    case need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) of
+        false -> {error, nothing_to_balance};
+        true ->
+            _ = multicall(DonorNodes, {emqx_node_rebalance_agent, enable, [self()]}),
+            {ok, Data#{donors => DonorNodes,
+                       recipients => RecipientNodes,
+                       initial_conn_counts => maps:from_list(ConnCounts),
+                       initial_sess_counts => maps:from_list(SessCounts)}}
+    end.
+
+disable_rebalance(#{donors := DonorNodes}) ->
+    _ = multicall(DonorNodes, {emqx_node_rebalance_agent, disable, [self()]}),
+    ok.
+
+evict_conns(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) ->
+    DonorNodeCounts = multicall(DonorNodes, {?MODULE, connection_count, []}),
+    {_, DonorCounts} = lists:unzip(DonorNodeCounts),
+    RecipientNodeCounts = multicall(RecipientNodes, {?MODULE, connection_count, []}),
+    {_, RecipientCounts} = lists:unzip(RecipientNodeCounts),
+
+    DonorAvg = avg(DonorCounts),
+    RecipientAvg = avg(RecipientCounts),
+    Thresholds = thresholds(conn, Opts),
+    NewData = Data#{donor_conn_avg => DonorAvg,
+                    recipient_conn_avg => RecipientAvg,
+                    donor_conn_counts => maps:from_list(DonorNodeCounts),
+                    recipient_conn_counts => maps:from_list(RecipientNodeCounts)},
+    case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of
+        true -> ok;
+        false ->
+            ConnEvictRate = maps:get(conn_evict_rate, Opts),
+            NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts),
+            ?LOG(warning, "Node rebalance, evict_conns, nodes=~p, counts=~p",
+                 [NodesToEvict, ConnEvictRate]),
+            _ = multicall(NodesToEvict, {emqx_eviction_agent, evict_connections, [ConnEvictRate]}),
+            {continue, NewData}
+    end.
+
+evict_sessions(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) ->
+    DonorNodeCounts = multicall(DonorNodes, {?MODULE, disconnected_session_count, []}),
+    {_, DonorCounts} = lists:unzip(DonorNodeCounts),
+    RecipientNodeCounts = multicall(RecipientNodes, {?MODULE, disconnected_session_count, []}),
+    {_, RecipientCounts} = lists:unzip(RecipientNodeCounts),
+
+    DonorAvg = avg(DonorCounts),
+    RecipientAvg = avg(RecipientCounts),
+    Thresholds = thresholds(sess, Opts),
+    NewData = Data#{donor_sess_avg => DonorAvg,
+                    recipient_sess_avg => RecipientAvg,
+                    donor_sess_counts => maps:from_list(DonorNodeCounts),
+                    recipient_sess_counts => maps:from_list(RecipientNodeCounts)},
+    case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of
+        true -> ok;
+        false ->
+            SessEvictRate = maps:get(sess_evict_rate, Opts),
+            NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts),
+            ?LOG(warning, "Node rebalance, evict_sessions, nodes=~p, counts=~p",
+                 [NodesToEvict, SessEvictRate]),
+            _ = multicall(NodesToEvict,
+                          {emqx_eviction_agent,
+                           evict_sessions,
+                           [SessEvictRate, RecipientNodes, disconnected]}),
+            {continue, NewData}
+    end.
+
+need_rebalance([] = _DonorNodes, _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> false;
+need_rebalance(_DonorNodes, [] = _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> false;
+need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) ->
+    DonorConnAvg = avg_for_nodes(DonorNodes, ConnCounts),
+    RecipientConnAvg = avg_for_nodes(RecipientNodes, ConnCounts),
+    DonorSessAvg = avg_for_nodes(DonorNodes, SessCounts),
+    RecipientSessAvg = avg_for_nodes(RecipientNodes, SessCounts),
+    Result = (not within_thresholds(DonorConnAvg, RecipientConnAvg, thresholds(conn, Opts)))
+        orelse (not within_thresholds(DonorSessAvg, RecipientSessAvg, thresholds(sess, Opts))),
+    ?tp(debug, emqx_node_rebalance_need_rebalance,
+        #{donors => DonorNodes,
+          recipients => RecipientNodes,
+          conn_counts => ConnCounts,
+          sess_counts => SessCounts,
+          opts => Opts,
+          result => Result
+         }),
+    Result.
+
+avg_for_nodes(Nodes, Counts) ->
+    avg(maps:values(maps:with(Nodes, maps:from_list(Counts)))).
+
+within_thresholds(Value, GoalValue, {AbsThres, RelThres}) ->
+    (Value =< GoalValue + AbsThres) orelse (Value =< GoalValue * RelThres).
+
+thresholds(conn, #{abs_conn_threshold := Abs, rel_conn_threshold := Rel}) ->
+    {Abs, Rel};
+thresholds(sess, #{abs_sess_threshold := Abs, rel_sess_threshold := Rel}) ->
+    {Abs, Rel}.
+
+nodes_to_evict(Goal, NodeCounts) ->
+    {Nodes, _} = lists:unzip(
+                   lists:filter(
+                     fun({_Node, Count}) ->
+                             Count > Goal
+                     end,
+                     NodeCounts)),
+    Nodes.
+
+get_stats(disabled, _Data) -> #{};
+get_stats(_State, Data) -> Data.
+
+avg(List) when length(List) >= 1 ->
+    lists:sum(List) / length(List).
+
+multicall(Nodes, {M, F, A}) ->
+    case rpc:multicall(Nodes, M, F, A) of
+        {Results, []} ->
+            case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of
+                {OkResults, []} ->
+                    [{Node, ok_result(Result)} || {Node, Result} <- OkResults];
+                {_, BadResults} ->
+                    error({bad_nodes, BadResults})
+            end;
+        {_, [_BadNode | _] = BadNodes} ->
+            error({bad_nodes, BadNodes})
+    end.
+
+is_ok({_Node, {ok, _}}) -> true;
+is_ok({_Node, ok}) -> true;
+is_ok(_) -> false.
+
+ok_result({ok, Result}) -> Result;
+ok_result(ok) -> ok.
+
+connection_count() ->
+    {ok, emqx_eviction_agent:connection_count()}.
+
+session_count() ->
+    {ok, emqx_eviction_agent:session_count()}.
+
+disconnected_session_count() ->
+    {ok, emqx_eviction_agent:session_count(disconnected)}.
+
+default_opts() ->
+    #{
+      conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE,
+      abs_conn_threshold => ?DEFAULT_ABS_CONN_THRESHOLD,
+      rel_conn_threshold => ?DEFAULT_REL_CONN_THRESHOLD,
+
+      sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE,
+      abs_sess_threshold => ?DEFAULT_ABS_SESS_THRESHOLD,
+      rel_sess_threshold => ?DEFAULT_REL_SESS_THRESHOLD,
+
+      wait_health_check => ?DEFAULT_WAIT_HEALTH_CHECK,
+      wait_takeover => ?DEFAULT_WAIT_TAKEOVER,
+
+      evict_interval => ?EVICT_INTERVAL,
+
+      nodes => all_nodes()
+     }.
+
+
+deinit(Data) ->
+    Keys = [recipient_conn_avg, recipient_sess_avg, donor_conn_avg, donor_sess_avg,
+            recipient_conn_counts, recipient_sess_counts, donor_conn_counts, donor_sess_counts,
+            initial_conn_counts, initial_sess_counts,
+            opts],
+    maps:without(Keys, Data).
+
+is_node_available() ->
+    true = is_pid(whereis(emqx_node_rebalance_agent)),
+    disabled = emqx_eviction_agent:status(),
+    node().
+
+all_nodes() ->
+    ekka_mnesia:cluster_nodes(all).
+
+seconds(Sec) ->
+    round(timer:seconds(Sec)).
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl
new file mode 100644
index 000000000..ade043d6b
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl
@@ -0,0 +1,127 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_agent).
+
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/types.hrl").
+
+-include_lib("stdlib/include/qlc.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-export([start_link/0,
+         enable/1,
+         disable/1,
+         status/0
+        ]).
+
+-export([init/1,
+         handle_call/3,
+         handle_info/2,
+         handle_cast/2,
+         code_change/3
+        ]).
+
+-define(ENABLE_KIND, emqx_node_rebalance).
+
+%%--------------------------------------------------------------------
+%% APIs
+%%--------------------------------------------------------------------
+
+-type status() :: {enabled, pid()} | disabled.
+
+-spec start_link() -> startlink_ret().
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+-spec enable(pid()) -> ok_or_error(already_enabled | eviction_agent_busy).
+enable(CoordinatorPid) ->
+    gen_server:call(?MODULE, {enable, CoordinatorPid}).
+
+-spec disable(pid()) -> ok_or_error(already_disabled | invalid_coordinator).
+disable(CoordinatorPid) ->
+    gen_server:call(?MODULE, {disable, CoordinatorPid}).
+
+-spec status() -> status().
+status() ->
+    gen_server:call(?MODULE, status).
+
+%%--------------------------------------------------------------------
+%% gen_server callbacks
+%%--------------------------------------------------------------------
+
+init([]) ->
+    {ok, #{}}.
+
+handle_call({enable, CoordinatorPid}, _From, St) ->
+    case St of
+        #{coordinator_pid := _Pid} ->
+            {reply, {error, already_enabled}, St};
+        _ ->
+            true = link(CoordinatorPid),
+            EvictionAgentPid = whereis(emqx_eviction_agent),
+            true = link(EvictionAgentPid),
+            case emqx_eviction_agent:enable(?ENABLE_KIND, undefined) of
+                ok ->
+                    {reply, ok, #{coordinator_pid => CoordinatorPid,
+                                  eviction_agent_pid => EvictionAgentPid}};
+                {error, eviction_agent_busy} ->
+                    true = unlink(EvictionAgentPid),
+                    true = unlink(CoordinatorPid),
+                    {reply, {error, eviction_agent_busy}, St}
+            end
+    end;
+
+handle_call({disable, CoordinatorPid}, _From, St) ->
+    case St of
+        #{coordinator_pid := CoordinatorPid,
+          eviction_agent_pid := EvictionAgentPid} ->
+            _ = emqx_eviction_agent:disable(?ENABLE_KIND),
+            true = unlink(EvictionAgentPid),
+            true = unlink(CoordinatorPid),
+            NewSt = maps:without(
+                      [coordinator_pid, eviction_agent_pid],
+                      St),
+            {reply, ok, NewSt};
+        #{coordinator_pid := _CoordinatorPid} ->
+            {reply, {error, invalid_coordinator}, St};
+        #{} ->
+            {reply, {error, already_disabled}, St}
+    end;
+
+handle_call(status, _From, St) ->
+    case St of
+        #{coordinator_pid := Pid} ->
+            {reply, {enabled, Pid}, St};
+        _ ->
+            {reply, disabled, St}
+    end;
+
+handle_call(Msg, _From, St) ->
+    ?LOG(warning, "Unknown call: ~p, State: ~p", [Msg, St]),
+    {reply, ignored, St}.
+
+handle_info(Msg, St) ->
+    ?LOG(warning, "Unknown Msg: ~p, State: ~p", [Msg, St]),
+    {noreply, St}.
+
+handle_cast(Msg, St) ->
+    ?LOG(warning, "Unknown cast Msg: ~p, State: ~p", [Msg, St]),
+    {noreply, St}.
+
+code_change(_Vsn, State, _Extra) ->
+    {ok, State}.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
new file mode 100644
index 000000000..6ba221cd8
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
@@ -0,0 +1,243 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_api).
+
+-import(minirest,  [return/1]).
+
+-rest_api(#{name   => load_rebalance_status,
+            method => 'GET',
+            path   => "/load_rebalance/status",
+            func   => status,
+            descr  => "Get load rebalance status"}).
+
+-rest_api(#{name   => load_rebalance_global_status,
+            method => 'GET',
+            path   => "/load_rebalance/global_status",
+            func   => global_status,
+            descr  => "Get status of all rebalance/evacuation processes across the cluster"}).
+
+-rest_api(#{name   => load_rebalance_availability_check,
+            method => 'GET',
+            path   => "/load_rebalance/availability_check",
+            func   => availability_check,
+            descr  => "Node rebalance availability check"}).
+
+-rest_api(#{name   => load_rebalance_start,
+            method => 'POST',
+            path   => "/load_rebalance/:bin:node/start",
+            func   => rebalance_start,
+            descr  => "Start rebalancing with the node as coordinator"}).
+
+-rest_api(#{name   => load_rebalance_stop,
+            method => 'POST',
+            path   => "/load_rebalance/:bin:node/stop",
+            func   => rebalance_stop,
+            descr  => "Stop rebalancing coordinated by the node"}).
+
+-rest_api(#{name   => load_rebalance_evacuation_start,
+            method => 'POST',
+            path   => "/load_rebalance/:bin:node/evacuation/start",
+            func   => rebalance_evacuation_start,
+            descr  => "Start evacuation on a node "}).
+
+-rest_api(#{name   => load_rebalance_evacuation_stop,
+            method => 'POST',
+            path   => "/load_rebalance/:bin:node/evacuation/stop",
+            func   => rebalance_evacuation_stop,
+            descr  => "Stop evacuation on the node"}).
+
+-export([status/2,
+         availability_check/2,
+         global_status/2,
+
+         rebalance_evacuation_start/2,
+         rebalance_evacuation_stop/2,
+         rebalance_start/2,
+         rebalance_stop/2
+        ]).
+
+status(_Bindings, _Params) ->
+    case emqx_node_rebalance_status:local_status() of
+        disabled ->
+            {ok, #{status => disabled}};
+        {rebalance, Stats} ->
+            {ok, format_status(rebalance, Stats)};
+        {evacuation, Stats} ->
+            {ok, format_status(evacuation, Stats)}
+    end.
+
+global_status(_Bindings, _Params) ->
+    #{evacuations := Evacuations,
+      rebalances := Rebalances} = emqx_node_rebalance_status:global_status(),
+      {ok, #{evacuations => maps:from_list(Evacuations),
+             rebalances => maps:from_list(Rebalances)}}.
+
+availability_check(_Bindings, _Params) ->
+    case emqx_eviction_agent:status() of
+        disabled ->
+            {200, #{}};
+        {enabled, _Stats} ->
+            {503, #{}}
+    end.
+
+rebalance_evacuation_start(#{node := NodeBin}, Params) ->
+    validated(
+      fun() ->
+              {Node, Opts} = validate_evacuation(NodeBin, params(Params)),
+              rpc(Node, emqx_node_rebalance_evacuation, start, [Opts])
+      end).
+
+rebalance_evacuation_stop(#{node := NodeBin}, _Params) ->
+    validated(
+      fun() ->
+              Node = parse_node(NodeBin),
+              rpc(Node, emqx_node_rebalance_evacuation, stop, [])
+      end).
+
+rebalance_start(#{node := NodeBin}, Params) ->
+    validated(
+      fun() ->
+              {Node, Opts} = validate_rebalance(NodeBin, params(Params)),
+              rpc(Node, emqx_node_rebalance, start, [Opts])
+      end).
+
+rebalance_stop(#{node := NodeBin}, _Params) ->
+    validated(
+      fun() ->
+              Node = parse_node(NodeBin),
+              rpc(Node, emqx_node_rebalance, stop, [])
+      end).
+
+rpc(Node, M, F, A) ->
+    case rpc:call(Node, M, F, A) of
+        ok -> return({ok, []});
+        {error, Error} ->
+            return({error, 400, io_lib:format("~p", [Error])});
+        {badrpc, _} ->
+            return({error, 400, io_lib:format("Error communicating with node ~p", [Node])});
+        Unknown ->
+            return({error, 400, io_lib:format("Unrecognized rpc result from node ~p: ~p",
+                                              [Node, Unknown])})
+    end.
+
+format_status(Process, Stats) ->
+    Stats#{process => Process, status => enabled}.
+
+validate_evacuation(Node, Params) ->
+    NodeToEvacuate = parse_node(Node),
+    OptList = lists:map(
+                fun validate_evacuation_param/1,
+                Params),
+    {NodeToEvacuate, maps:from_list(OptList)}.
+
+validate_rebalance(Node, Params) ->
+    CoordinatorNode = parse_node(Node),
+    OptList = lists:map(
+                fun validate_rebalance_param/1,
+                Params),
+    {CoordinatorNode, maps:from_list(OptList)}.
+
+validate_evacuation_param({<<"conn_evict_rate">>, Value}) ->
+    validate_pos_int(conn_evict_rate, Value);
+validate_evacuation_param({<<"sess_evict_rate">>, Value}) ->
+    validate_pos_int(sess_evict_rate, Value);
+validate_evacuation_param({<<"redirect_to">>, Value}) ->
+    validate_binary(server_reference, Value);
+validate_evacuation_param({<<"wait_takeover">>, Value}) ->
+    validate_pos_int(wait_takeover, Value);
+validate_evacuation_param({<<"migrate_to">>, Value}) ->
+    validate_nodes(migrate_to, Value);
+validate_evacuation_param(Value) ->
+    validation_error(io_lib:format("Unknown evacuation param: ~p", [Value])).
+
+validate_rebalance_param({<<"wait_health_check">>, Value}) ->
+    validate_pos_int(wait_health_check, Value);
+validate_rebalance_param({<<"conn_evict_rate">>, Value}) ->
+    validate_pos_int(conn_evict_rate, Value);
+validate_rebalance_param({<<"sess_evict_rate">>, Value}) ->
+    validate_pos_int(sess_evict_rate, Value);
+validate_rebalance_param({<<"abs_conn_threshold">>, Value}) ->
+    validate_pos_int(abs_conn_threshold, Value);
+validate_rebalance_param({<<"rel_conn_threshold">>, Value}) ->
+    validate_fraction(rel_conn_threshold, Value);
+validate_rebalance_param({<<"abs_sess_threshold">>, Value}) ->
+    validate_pos_int(abs_sess_threshold, Value);
+validate_rebalance_param({<<"rel_sess_threshold">>, Value}) ->
+    validate_fraction(rel_sess_threshold, Value);
+validate_rebalance_param({<<"wait_takeover">>, Value}) ->
+    validate_pos_int(wait_takeover, Value);
+validate_rebalance_param({<<"nodes">>, Value}) ->
+    validate_nodes(nodes, Value);
+validate_rebalance_param(Value) ->
+    validation_error(io_lib:format("Unknown rebalance param: ~p", [Value])).
+
+validate_binary(Name, Value) when is_binary(Value) ->
+    {Name, Value};
+validate_binary(Name, _Value) ->
+    validation_error("invalid string in " ++ atom_to_list(Name)).
+
+validate_pos_int(Name, Value) ->
+    case is_integer(Value) andalso Value > 0 of
+        true -> {Name, Value};
+        false ->
+            validation_error("invalid " ++  atom_to_list(Name) ++ " value")
+    end.
+
+validate_fraction(Name, Value) ->
+    case is_number(Value) andalso Value > 1.0 of
+        true -> {Name, Value};
+        false ->
+            validation_error("invalid " ++  atom_to_list(Name) ++ " value")
+    end.
+
+validate_nodes(Name, NodeList) when is_list(NodeList) ->
+    Nodes = lists:map(
+              fun parse_node/1,
+              NodeList),
+    case emqx_node_rebalance_evacuation:available_nodes(Nodes) of
+        [] ->
+            validation_error(io_lib:format("no available nodes list in ~p: ~p", [Name, Nodes]));
+        Nodes ->
+            {Name, Nodes};
+        OtherNodes ->
+            validation_error(
+              io_lib:format("unavailable nodes in ~p: ~p",
+                            [Name, Nodes -- OtherNodes]))
+    end;
+validate_nodes(Name, Nodes) ->
+    validation_error(io_lib:format("invalid node list in ~p: ~p", [Name, Nodes])).
+
+validated(Fun) ->
+    try
+        Fun()
+    catch throw:{validation_error, Error} ->
+              return({error, 400, iolist_to_binary(Error)})
+    end.
+
+validation_error(Error) ->
+    throw({validation_error, Error}).
+
+parse_node(Bin) when is_binary(Bin) ->
+    try
+        binary_to_existing_atom(Bin)
+    catch
+        error:badarg ->
+            validation_error("invalid node: " ++ [Bin])
+    end.
+
+params([{}]) -> [];
+params(Params) -> Params.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl
new file mode 100644
index 000000000..3cba331de
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl
@@ -0,0 +1,34 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_app).
+
+-behaviour(application).
+
+-emqx_plugin(?MODULE).
+
+-export([ start/2
+        , stop/1
+        ]).
+
+start(_Type, _Args) ->
+    Env = application:get_all_env(emqx_node_rebalance),
+    {ok, Sup} = emqx_node_rebalance_sup:start_link(Env),
+    ok = emqx_node_rebalance_cli:load(),
+    {ok, Sup}.
+
+stop(_State) ->
+    emqx_node_rebalance_cli:unload().
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl
new file mode 100644
index 000000000..10d0912c4
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl
@@ -0,0 +1,265 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_cli).
+
+%% APIs
+-export([ load/0
+        , unload/0
+        , cli/1
+        ]).
+
+load() ->
+    emqx_ctl:register_command(rebalance, {?MODULE, cli}, []).
+
+unload() ->
+    emqx_ctl:unregister_command(rebalance).
+
+cli(["start" | StartArgs]) ->
+    case start_args(StartArgs) of
+        {evacuation, Opts} ->
+            case emqx_node_rebalance_evacuation:status() of
+                disabled ->
+                    ok = emqx_node_rebalance_evacuation:start(Opts),
+                    emqx_ctl:print("Rebalance(evacuation) started~n"),
+                    true;
+                {enabled, _} ->
+                    emqx_ctl:print("Rebalance is already enabled~n"),
+                    false
+            end;
+        {rebalance, Opts} ->
+            case emqx_node_rebalance:start(Opts) of
+                ok ->
+                    emqx_ctl:print("Rebalance started~n"),
+                    true;
+                {error, Reason} ->
+                    emqx_ctl:print("Rebalance start error: ~p~n", [Reason]),
+                    false
+            end;
+        {error, Error} ->
+            emqx_ctl:print("Rebalance start error: ~s~n", [Error]),
+            false
+    end;
+cli(["node-status", NodeStr]) ->
+    Node = list_to_atom(NodeStr),
+    node_status(emqx_node_rebalance_status:local_status(Node));
+cli(["node-status"]) ->
+    node_status(emqx_node_rebalance_status:local_status());
+cli(["status"]) ->
+    #{evacuations := Evacuations,
+      rebalances := Rebalances} = emqx_node_rebalance_status:global_status(),
+    lists:foreach(
+      fun({Node, Status}) ->
+        emqx_ctl:print("--------------------------------------------------------------------~n"),
+        emqx_ctl:print("Node ~p: evacuation~n~s",
+                       [Node, emqx_node_rebalance_status:format_local_status(Status)])
+      end,
+      Evacuations),
+    lists:foreach(
+      fun({Node, Status}) ->
+        emqx_ctl:print("--------------------------------------------------------------------~n"),
+        emqx_ctl:print("Node ~p: rebalance coordinator~n~s",
+                       [Node, emqx_node_rebalance_status:format_coordinator_status(Status)])
+      end,
+      Rebalances);
+cli(["stop"]) ->
+    case emqx_node_rebalance_evacuation:status() of
+        {enabled, _} ->
+            ok = emqx_node_rebalance_evacuation:stop(),
+            emqx_ctl:print("Rebalance(evacuation) stopped~n"),
+            true;
+        disabled ->
+            case emqx_node_rebalance:status() of
+                {enabled, _} ->
+                    ok = emqx_node_rebalance:stop(),
+                    emqx_ctl:print("Rebalance stopped~n"),
+                    true;
+                disabled ->
+                    emqx_ctl:print("Rebalance is already disabled~n"),
+                    false
+            end
+    end;
+cli(_) ->
+    emqx_ctl:usage(
+      [{"rebalance start --evacuation \\\n"
+        "    [--redirect-to \"Host1:Port1 Host2:Port2 ...\"] \\\n"
+        "    [--conn-evict-rate CountPerSec] \\\n"
+        "    [--migrate-to \"node1@host1 node2@host2 ...\"] \\\n"
+        "    [--wait-takeover Secs] \\\n"
+        "    [--sess-evict-rate CountPerSec]",
+        "Start current node evacuation with optional server redirect to the specified servers"},
+
+       {"rebalance start \\\n"
+        "    [--nodes \"node1@host1 node2@host2\"] \\\n"
+        "    [--wait-health-check Secs] \\\n"
+        "    [--conn-evict-rate ConnPerSec] \\\n"
+        "    [--abs-conn-threshold Count] \\\n"
+        "    [--rel-conn-threshold Fraction] \\\n"
+        "    [--conn-evict-rate ConnPerSec] \\\n"
+        "    [--wait-takeover Secs] \\\n"
+        "    [--sess-evict-rate CountPerSec] \\\n"
+        "    [--abs-sess-threshold Count] \\\n"
+        "    [--rel-sess-threshold Fraction]",
+        "Start current node evacuation with optional server redirect to the specified servers"},
+
+       {"rebalance node-status",
+        "Get current node rebalance status"},
+
+       {"rebalance node-status \"node1@host1\"",
+        "Get remote node rebalance status"},
+
+       {"rebalance status",
+        "Get statuses of all current rebalance/evacuation processes across the cluster"},
+
+       {"rebalance stop",
+        "Stop node rebalance"}]).
+
+node_status(NodeStatus) ->
+    case NodeStatus of
+        {Process, Status} when Process =:= evacuation orelse Process =:= rebalance ->
+            emqx_ctl:print("Rebalance type: ~p~n~s~n",
+                           [Process, emqx_node_rebalance_status:format_local_status(Status)]);
+        disabled ->
+            emqx_ctl:print("Rebalance disabled~n");
+        Other ->
+            emqx_ctl:print("Error detecting rebalance status: ~p~n", [Other])
+    end.
+
+start_args(Args) ->
+    case collect_args(Args, #{}) of
+        {ok, #{"--evacuation" := true} = Collected} ->
+            case validate_evacuation(maps:to_list(Collected), #{}) of
+                {ok, Validated} ->
+                    {evacuation, Validated};
+                {error, _} = Error -> Error
+            end;
+        {ok, #{} = Collected} ->
+            case validate_rebalance(maps:to_list(Collected), #{}) of
+                {ok, Validated} ->
+                    {rebalance, Validated};
+                {error, _} = Error -> Error
+            end;
+        {error, _} = Error -> Error
+    end.
+
+collect_args([], Map) -> {ok, Map};
+
+%% evacuation
+collect_args(["--evacuation" | Args], Map) ->
+    collect_args(Args, Map#{"--evacuation" => true});
+collect_args(["--redirect-to", ServerReference | Args], Map) ->
+    collect_args(Args, Map#{"--redirect-to" => ServerReference});
+collect_args(["--migrate-to", MigrateTo | Args], Map) ->
+    collect_args(Args, Map#{"--migrate-to" => MigrateTo});
+%% rebalance
+collect_args(["--nodes", Nodes | Args], Map) ->
+    collect_args(Args, Map#{"--nodes" => Nodes});
+collect_args(["--wait-health-check", WaitHealthCheck | Args], Map) ->
+    collect_args(Args, Map#{"--wait-health-check" => WaitHealthCheck});
+collect_args(["--abs-conn-threshold", AbsConnThres | Args], Map) ->
+    collect_args(Args, Map#{"--abs-conn-threshold" => AbsConnThres});
+collect_args(["--rel-conn-threshold", RelConnThres | Args], Map) ->
+    collect_args(Args, Map#{"--rel-conn-threshold" => RelConnThres});
+collect_args(["--abs-sess-threshold", AbsSessThres | Args], Map) ->
+    collect_args(Args, Map#{"--abs-sess-threshold" => AbsSessThres});
+collect_args(["--rel-sess-threshold", RelSessThres | Args], Map) ->
+    collect_args(Args, Map#{"--rel-sess-threshold" => RelSessThres});
+%% common
+collect_args(["--conn-evict-rate", ConnEvictRate | Args], Map) ->
+    collect_args(Args, Map#{"--conn-evict-rate" => ConnEvictRate});
+collect_args(["--wait-takeover", WaitTakeover | Args], Map) ->
+    collect_args(Args, Map#{"--wait-takeover" => WaitTakeover});
+collect_args(["--sess-evict-rate", SessEvictRate | Args], Map) ->
+    collect_args(Args, Map#{"--sess-evict-rate" => SessEvictRate});
+%% fallback
+collect_args(Args, _Map) ->
+    {error, io_lib:format("unknown arguments: ~p", [Args])}.
+
+validate_evacuation([], Map) ->
+    {ok, Map};
+validate_evacuation([{"--evacuation", _} | Rest], Map) ->
+    validate_evacuation(Rest, Map);
+validate_evacuation([{"--redirect-to", ServerReference} | Rest], Map) ->
+    validate_evacuation(Rest, Map#{server_reference => list_to_binary(ServerReference)});
+validate_evacuation([{"--conn-evict-rate", _} | _] = Opts, Map) ->
+    validate_pos_int(conn_evict_rate, Opts, Map, fun validate_evacuation/2);
+validate_evacuation([{"--sess-evict-rate", _} | _] = Opts, Map) ->
+    validate_pos_int(sess_evict_rate, Opts, Map, fun validate_evacuation/2);
+validate_evacuation([{"--wait-takeover", _} | _] = Opts, Map) ->
+    validate_pos_int(wait_takeover, Opts, Map, fun validate_evacuation/2);
+validate_evacuation([{"--migrate-to", MigrateTo} | Rest], Map) ->
+    Nodes = lists:map(fun list_to_atom/1, string:tokens(MigrateTo, ", ")),
+    case emqx_node_rebalance_evacuation:available_nodes(Nodes) of
+        [] ->
+            {error, "invalid --migrate-to, no nodes"};
+        Nodes ->
+            validate_evacuation(Rest, Map#{migrate_to => Nodes});
+        OtherNodes ->
+            {error,
+             io_lib:format("invalid --migrate-to, unavailable nodes: ~p",
+                           [Nodes -- OtherNodes])}
+    end;
+validate_evacuation(Rest, _Map) ->
+    {error, io_lib:format("unknown evacuation arguments: ~p", [Rest])}.
+
+validate_rebalance([], Map) ->
+    {ok, Map};
+validate_rebalance([{"--wait-health-check", _} | _] = Opts, Map) ->
+    validate_pos_int(wait_health_check, Opts, Map, fun validate_rebalance/2);
+validate_rebalance([{"--conn-evict-rate", _} | _] = Opts, Map) ->
+    validate_pos_int(conn_evict_rate, Opts, Map, fun validate_rebalance/2);
+validate_rebalance([{"--sess-evict-rate", _} | _] = Opts, Map) ->
+    validate_pos_int(sess_evict_rate, Opts, Map, fun validate_rebalance/2);
+validate_rebalance([{"--abs-conn-threshold", _} | _] = Opts, Map) ->
+    validate_pos_int(abs_conn_threshold, Opts, Map, fun validate_rebalance/2);
+validate_rebalance([{"--rel-conn-threshold", _} | _] = Opts, Map) ->
+    validate_fraction(rel_conn_threshold, Opts, Map, fun validate_rebalance/2);
+validate_rebalance([{"--abs-sess-threshold", _} | _] = Opts, Map) ->
+    validate_pos_int(abs_sess_threshold, Opts, Map, fun validate_rebalance/2);
+validate_rebalance([{"--rel-sess-threshold", _} | _] = Opts, Map) ->
+    validate_fraction(rel_sess_threshold, Opts, Map, fun validate_rebalance/2);
+validate_rebalance([{"--wait-takeover", _} | _] = Opts, Map) ->
+    validate_pos_int(wait_takeover, Opts, Map, fun validate_rebalance/2);
+validate_rebalance([{"--nodes", NodeStr} | Rest], Map) ->
+    Nodes = lists:map(fun list_to_atom/1, string:tokens(NodeStr, ", ")),
+    case emqx_node_rebalance:available_nodes(Nodes) of
+        [] ->
+            {error, "invalid --nodes, no nodes"};
+        Nodes ->
+            validate_rebalance(Rest, Map#{nodes => Nodes});
+        OtherNodes ->
+            {error,
+             io_lib:format("invalid --nodes, unavailable nodes: ~p",
+                           [Nodes -- OtherNodes])}
+    end;
+validate_rebalance(Rest, _Map) ->
+    {error, io_lib:format("unknown rebalance arguments: ~p", [Rest])}.
+
+validate_fraction(Name, [{OptionName, Value} | Rest], Map, Next) ->
+    case string:to_float(Value) of
+        {Num, ""} when Num > 1.0 ->
+           Next(Rest, Map#{Name => Num});
+        _ ->
+            {error, "invalid " ++ OptionName ++ " value"}
+    end.
+
+validate_pos_int(Name, [{OptionName, Value} | Rest], Map, Next) ->
+    case string:to_integer(Value) of
+        {Int, ""} when Int > 0 ->
+           Next(Rest, Map#{Name => Int});
+        _ ->
+            {error, "invalid " ++ OptionName ++ " value"}
+    end.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl
new file mode 100644
index 000000000..5dc99d736
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl
@@ -0,0 +1,298 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_evacuation).
+
+-include("emqx_node_rebalance.hrl").
+
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/types.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-export([start/1,
+         status/0,
+         stop/0
+        ]).
+
+-export([start_link/0]).
+
+-behavior(gen_statem).
+
+-export([init/1,
+         callback_mode/0,
+         handle_event/4,
+         code_change/4
+        ]).
+
+-export([is_node_available/0,
+         available_nodes/1]).
+
+-ifdef(TEST).
+-export([migrate_to/1]).
+-endif.
+
+%%--------------------------------------------------------------------
+%% APIs
+%%--------------------------------------------------------------------
+
+-define(EVICT_INTERVAL_NO_NODES, 30000).
+
+-type migrate_to() :: [node()] | undefined.
+
+-type start_opts() :: #{server_reference => emqx_eviction_agent:server_reference(),
+                        conn_evict_rate => pos_integer(),
+                        sess_evict_rate => pos_integer(),
+                        wait_takeover => pos_integer(),
+                        migrate_to => migrate_to()
+                       }.
+-type start_error() :: already_started | eviction_agent_busy.
+-type stats() :: #{
+                   initial_conns := non_neg_integer(),
+                   initial_sessions := non_neg_integer(),
+                   current_conns := non_neg_integer(),
+                   current_sessions := non_neg_integer(),
+                   conn_evict_rate := pos_integer(),
+                   sess_evict_rate := pos_integer(),
+                   server_reference := emqx_eviction_agent:server_reference(),
+                   migrate_to := migrate_to()
+                  }.
+-type status() :: {started, stats()} | stopped.
+
+-spec start(start_opts()) -> ok_or_error(start_error()).
+start(StartOpts) ->
+    Opts = maps:merge(default_opts(), StartOpts),
+    gen_statem:call(?MODULE, {start, Opts}).
+
+-spec stop() -> ok_or_error(not_started).
+stop() ->
+    gen_statem:call(?MODULE, stop).
+
+-spec status() -> status().
+status() ->
+    gen_statem:call(?MODULE, status).
+
+-spec start_link() -> startlink_ret().
+start_link() ->
+    gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+-spec available_nodes(list(node())) -> list(node()).
+available_nodes(Nodes) when is_list(Nodes) ->
+    {Available, _} = rpc:multicall(Nodes, ?MODULE, is_node_available, []),
+    lists:filter(fun is_atom/1, Available).
+
+%%--------------------------------------------------------------------
+%% gen_statem callbacks
+%%--------------------------------------------------------------------
+
+callback_mode() -> handle_event_function.
+
+%% states: disabled, evicting_conns, waiting_takeover, evicting_sessions, prohibiting
+
+init([]) ->
+    case emqx_node_rebalance_evacuation_persist:read(default_opts()) of
+        {ok, #{server_reference := ServerReference} = Opts} ->
+            ?LOG(warning, "Restoring evacuation state: ~p", [Opts]),
+            case emqx_eviction_agent:enable(?MODULE, ServerReference) of
+                ok ->
+                    Data = init_data(#{}, Opts),
+                    ok = warn_enabled(),
+                    {ok, evicting_conns, Data, [{state_timeout, 0, evict_conns}]};
+                {error, eviction_agent_busy} ->
+                    emqx_node_rebalance_evacuation_persist:clear(),
+                    {ok, disabled, #{}}
+            end;
+        none ->
+            {ok, disabled, #{}}
+    end.
+
+%% start
+handle_event({call, From},
+             {start, #{server_reference := ServerReference} = Opts},
+             disabled,
+             #{} = Data) ->
+    case emqx_eviction_agent:enable(?MODULE, ServerReference) of
+        ok ->
+            NewData = init_data(Data, Opts),
+            ok = emqx_node_rebalance_evacuation_persist:save(Opts),
+            ?LOG(warning, "Node evacuation started"),
+            {next_state,
+             evicting_conns,
+             NewData,
+             [{state_timeout, 0, evict_conns},
+              {reply, From, ok}]};
+        {error, eviction_agent_busy} ->
+            {keep_state_and_data,
+              [{reply, From, {error, eviction_agent_busy}}]}
+    end;
+handle_event({call, From}, {start, _Opts}, _State, #{}) ->
+    {keep_state_and_data,
+     [{reply, From, {error, already_started}}]};
+
+%% stop
+handle_event({call, From}, stop, disabled, #{}) ->
+    {keep_state_and_data,
+     [{reply, From, {error, not_started}}]};
+handle_event({call, From}, stop, _State, Data) ->
+    ok = emqx_node_rebalance_evacuation_persist:clear(),
+    _ = emqx_eviction_agent:disable(?MODULE),
+    ?LOG(warning, "Node evacuation stopped"),
+    {next_state,
+     disabled,
+     deinit(Data),
+     [{reply, From, ok}]};
+
+%% status
+handle_event({call, From}, status, disabled, #{}) ->
+    {keep_state_and_data,
+     [{reply, From, disabled}]};
+handle_event({call, From}, status, State, #{migrate_to := MigrateTo} = Data) ->
+    Stats = maps:with(
+              [initial_conns, current_conns,
+               initial_sessions, current_sessions,
+               server_reference, conn_evict_rate, sess_evict_rate],
+              Data),
+    {keep_state_and_data,
+     [{reply, From, {enabled, Stats#{state => State, migrate_to => migrate_to(MigrateTo)}}}]};
+
+%% conn eviction
+handle_event(state_timeout,
+             evict_conns,
+             evicting_conns,
+             #{conn_evict_rate := ConnEvictRate,
+               wait_takeover := WaitTakeover} = Data) ->
+    case emqx_eviction_agent:status() of
+        {enabled, #{connections := Conns}} when Conns > 0 ->
+            ok = emqx_eviction_agent:evict_connections(ConnEvictRate),
+            ?tp(debug, node_evacuation_evict_conn, #{conn_evict_rate => ConnEvictRate}),
+            ?LOG(warning, "Node evacuation evict_conns, count=~p, conn_evict_rate=~p",
+                 [Conns, ConnEvictRate]),
+            NewData = Data#{current_conns => Conns},
+            {keep_state,
+             NewData,
+             [{state_timeout, ?EVICT_INTERVAL, evict_conns}]};
+        {enabled, #{connections := 0}} ->
+            NewData = Data#{current_conns => 0},
+            ?LOG(warning, "Node evacuation evict_conns over"),
+            {next_state,
+             waiting_takeover,
+             NewData,
+             [{state_timeout, timer:seconds(WaitTakeover), evict_sessions}]}
+    end;
+
+handle_event(state_timeout,
+             evict_sessions,
+             waiting_takeover,
+             Data) ->
+            ?LOG(warning, "Node evacuation wait_takeover over"),
+            {next_state,
+             evicting_sessions,
+             Data,
+             [{state_timeout, 0, evict_sessions}]};
+
+%% session eviction
+handle_event(state_timeout,
+             evict_sessions,
+             evicting_sessions,
+             #{sess_evict_rate := SessEvictRate,
+               migrate_to := MigrateTo,
+               current_sessions := CurrSessCount} = Data) ->
+    case emqx_eviction_agent:status() of
+        {enabled, #{sessions := SessCount}} when SessCount > 0 ->
+            case migrate_to(MigrateTo) of
+                [] ->
+                    ?LOG(warning,
+                         "No nodes are available to evacuate sessions, session_count=~p",
+                         [CurrSessCount]),
+                    {keep_state_and_data,
+                     [{state_timeout, ?EVICT_INTERVAL_NO_NODES, evict_sessions}]};
+                Nodes ->
+                    ok = emqx_eviction_agent:evict_sessions(SessEvictRate, Nodes),
+                    ?LOG(warning, "Node evacuation evict_sessions, count=~p, sess_evict_rate=~p,"
+                         "target_nodes=~p", [SessCount, SessEvictRate, Nodes]),
+                    NewData = Data#{current_sessions => SessCount},
+                    {keep_state,
+                     NewData,
+                     [{state_timeout, ?EVICT_INTERVAL, evict_sessions}]}
+            end;
+        {enabled, #{sessions := 0}} ->
+            ?tp(debug, node_evacuation_evict_sess_over, #{}),
+            ?LOG(warning, "Node evacuation evict_sessions over"),
+            NewData = Data#{current_sessions => 0},
+            {next_state,
+             prohibiting,
+             NewData}
+    end;
+
+handle_event({call, From}, Msg, State, Data) ->
+    ?LOG(warning, "Unknown call: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
+    {keep_state_and_data,
+     [{reply, From, ignored}]};
+
+handle_event(info, Msg, State, Data) ->
+    ?LOG(warning, "Unknown Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
+    keep_state_and_data;
+
+handle_event(cast, Msg, State, Data) ->
+    ?LOG(warning, "Unknown cast Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
+    keep_state_and_data.
+
+code_change(_Vsn, State, Data, _Extra) ->
+    {ok, State, Data}.
+
+%%--------------------------------------------------------------------
+%% internal funs
+%%--------------------------------------------------------------------
+
+default_opts() ->
+    #{
+      server_reference => undefined,
+      conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE,
+      sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE,
+      wait_takeover => ?DEFAULT_WAIT_TAKEOVER,
+      migrate_to => undefined
+     }.
+
+init_data(Data0, Opts) ->
+    Data1 = maps:merge(Data0, Opts),
+    {enabled, #{connections := ConnCount, sessions := SessCount}} = emqx_eviction_agent:status(),
+    Data1#{
+      initial_conns => ConnCount,
+      current_conns => ConnCount,
+      initial_sessions => SessCount,
+      current_sessions => SessCount
+     }.
+
+deinit(Data) ->
+    Keys = [initial_conns, current_conns, initial_sessions, current_sessions]
+           ++ maps:keys(default_opts()),
+    maps:without(Keys, Data).
+
+warn_enabled() ->
+    Msg = "Node evacuation is enabled. The node will not receive connections.",
+    ?LOG(warning, Msg),
+    io:format(standard_error, "~s~n", [Msg]).
+
+migrate_to(undefined) ->
+    migrate_to(all_nodes());
+migrate_to(Nodes) when is_list(Nodes) ->
+    available_nodes(Nodes).
+
+is_node_available() ->
+    disabled = emqx_eviction_agent:status(),
+    node().
+
+all_nodes() ->
+    ekka_mnesia:cluster_nodes(all) -- [node()].
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl
new file mode 100644
index 000000000..06d8800da
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl
@@ -0,0 +1,109 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_evacuation_persist).
+
+-export([save/1,
+         clear/0,
+         read/1]).
+
+
+-ifdef(TEST).
+-export([evacuation_filepath/0]).
+-endif.
+
+-include("emqx_node_rebalance.hrl").
+-include_lib("emqx/include/types.hrl").
+
+%%--------------------------------------------------------------------
+%% APIs
+%%--------------------------------------------------------------------
+
+%% do not persist `migrate_to`:
+%% * after restart there is nothing to migrate
+%% * this value may be invalid after node was offline
+-type start_opts() :: #{server_reference => emqx_eviction_agent:server_reference(),
+                        conn_evict_rate => pos_integer(),
+                        sess_evict_rate => pos_integer(),
+                        wait_takeover => pos_integer()
+                       }.
+
+-spec save(start_opts()) -> ok_or_error(term()).
+save(#{server_reference := ServerReference,
+       conn_evict_rate := ConnEvictRate,
+       sess_evict_rate := SessEvictRate,
+       wait_takeover := WaitTakeover} = Data)
+  when (is_binary(ServerReference) orelse ServerReference =:= undefined) andalso
+       is_integer(ConnEvictRate) andalso ConnEvictRate > 0 andalso
+       is_integer(SessEvictRate) andalso SessEvictRate > 0 andalso
+       is_integer(WaitTakeover) andalso WaitTakeover >= 0 ->
+    Filepath = evacuation_filepath(),
+    case filelib:ensure_dir(Filepath) of
+        ok ->
+            JsonData = emqx_json:encode(
+                         prepare_for_encode(maps:with(persist_keys(), Data)),
+                         [pretty]),
+            file:write_file(Filepath, JsonData);
+        {error, _} = Error -> Error
+    end.
+
+-spec clear() -> ok.
+clear() ->
+    file:delete(evacuation_filepath()).
+
+-spec read(start_opts()) -> {ok, start_opts()} | none.
+read(DefaultOpts) ->
+    case file:read_file(evacuation_filepath()) of
+        {ok, Data} ->
+            case emqx_json:safe_decode(Data, [return_maps]) of
+                {ok, Map} when is_map(Map) ->
+                    {ok, map_to_opts(DefaultOpts, Map)};
+                _NotAMap ->
+                    {ok, DefaultOpts}
+            end;
+        {error, _} ->
+            none
+    end.
+
+%%--------------------------------------------------------------------
+%% Internal funcs
+%%--------------------------------------------------------------------
+
+persist_keys() ->
+    [server_reference,
+     conn_evict_rate,
+     sess_evict_rate,
+     wait_takeover].
+
+prepare_for_encode(#{server_reference := undefined} = Data) ->
+    Data#{server_reference => null};
+prepare_for_encode(Data) -> Data.
+
+format_after_decode(#{server_reference := null} = Data) ->
+    Data#{server_reference => undefined};
+format_after_decode(Data) -> Data.
+
+map_to_opts(DefaultOpts, Map) ->
+    format_after_decode(
+      map_to_opts(
+        maps:to_list(DefaultOpts), Map, #{})).
+
+map_to_opts([], _Map, Opts) -> Opts;
+map_to_opts([{Key, DefaultVal} | Rest], Map, Opts) ->
+    map_to_opts(Rest, Map, Opts#{Key => maps:get(atom_to_binary(Key), Map, DefaultVal)}).
+
+evacuation_filepath() ->
+    filename:join([emqx:get_env(data_dir), ?EVACUATION_FILENAME]).
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl
new file mode 100644
index 000000000..4002d7c13
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl
@@ -0,0 +1,225 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_status).
+
+-export([local_status/0,
+         local_status/1,
+         global_status/0,
+         format_local_status/1,
+         format_coordinator_status/1]).
+
+%% For RPC
+-export([evacuation_status/0,
+         rebalance_status/0]).
+
+%%--------------------------------------------------------------------
+%% APIs
+%%--------------------------------------------------------------------
+
+-spec local_status() -> disabled | {evacuation, map()} | {rebalance, map()}.
+local_status() ->
+    case emqx_node_rebalance_evacuation:status() of
+        {enabled, Status} ->
+            {evacuation, evacuation(Status)};
+        disabled ->
+            case emqx_node_rebalance_agent:status() of
+                {enabled, CoordinatorPid} ->
+                    case emqx_node_rebalance:status(CoordinatorPid) of
+                        {enabled, Status} ->
+                            local_rebalance(Status, node());
+                        disabled ->
+                            disabled
+                    end;
+                disabled ->
+                    disabled
+            end
+    end.
+
+-spec local_status(node()) -> disabled | {evacuation, map()} | {rebalance, map()}.
+local_status(Node) ->
+    rpc:call(Node, ?MODULE, ?FUNCTION_NAME, []).
+
+-spec format_local_status(map()) -> iodata().
+format_local_status(Status) ->
+    format_status(Status, local_status_field_format_order()).
+
+-spec global_status() -> #{rebalances := [{node(), map()}], evacuations := [{node(), map()}]}.
+global_status() ->
+    Nodes = ekka_mnesia:cluster_nodes(all),
+    {RebalanceResults, _} = rpc:multicall(Nodes, ?MODULE, rebalance_status, []),
+    Rebalances = [{Node, coordinator_rebalance(Status)} || {Node, {enabled, Status}} <- RebalanceResults],
+    {EvacuatioResults, _} = rpc:multicall(Nodes, ?MODULE, evacuation_status, []),
+    Evacuations = [{Node, evacuation(Status)} || {Node, {enabled, Status}} <- EvacuatioResults],
+    #{rebalances => Rebalances, evacuations => Evacuations}.
+
+-spec format_coordinator_status(map()) -> iodata().
+format_coordinator_status(Status) ->
+    format_status(Status, coordinator_status_field_format_order()).
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+evacuation(Status) ->
+    #{
+      state => maps:get(state, Status),
+      connection_eviction_rate => maps:get(conn_evict_rate, Status),
+      session_eviction_rate => maps:get(sess_evict_rate, Status),
+      connection_goal => 0,
+      session_goal => 0,
+      session_recipients => maps:get(migrate_to, Status),
+      stats => #{
+        initial_connected => maps:get(initial_conns, Status),
+        current_connected => maps:get(current_conns, Status),
+        initial_sessions => maps:get(initial_sessions, Status),
+        current_sessions => maps:get(current_sessions, Status)
+      }
+     }.
+
+local_rebalance(#{donors := Donors} = Stats, Node) ->
+    case lists:member(Node, Donors) of
+        true -> {rebalance, donor_rebalance(Stats, Node)};
+        false -> disabled
+    end.
+
+donor_rebalance(Status, Node) ->
+    Opts = maps:get(opts, Status),
+    InitialConnCounts = maps:get(initial_conn_counts, Status),
+    InitialSessCounts = maps:get(initial_sess_counts, Status),
+
+    CurrentStats = #{
+                     initial_connected => maps:get(Node, InitialConnCounts),
+                     initial_sessions => maps:get(Node, InitialSessCounts),
+                     current_connected => emqx_eviction_agent:connection_count(),
+                     current_sessions => emqx_eviction_agent:session_count(),
+                     current_disconnected_sessions => emqx_eviction_agent:session_count(
+                                                       disconnected)
+                    },
+    maps:from_list(
+      [
+       {state, maps:get(state, Status)},
+       {coordinator_node, maps:get(coordinator_node, Status)},
+       {connection_eviction_rate, maps:get(conn_evict_rate, Opts)},
+       {session_eviction_rate, maps:get(sess_evict_rate, Opts)},
+       {recipients, maps:get(recipients, Status)},
+       {stats, CurrentStats}
+      ] ++
+      [{connection_goal, maps:get(recipient_conn_avg, Status)}
+       || maps:is_key(recipient_conn_avg, Status)
+      ] ++
+      [{disconnected_session_goal, maps:get(recipient_sess_avg, Status)}
+       || maps:is_key(recipient_sess_avg, Status)
+      ]).
+
+coordinator_rebalance(Status) ->
+    Opts = maps:get(opts, Status),
+    maps:from_list(
+      [
+       {state, maps:get(state, Status)},
+       {coordinator_node, maps:get(coordinator_node, Status)},
+       {connection_eviction_rate, maps:get(conn_evict_rate, Opts)},
+       {session_eviction_rate, maps:get(sess_evict_rate, Opts)},
+       {recipients, maps:get(recipients, Status)},
+       {donors, maps:get(donors, Status)}
+      ] ++
+      [{connection_goal, maps:get(recipient_conn_avg, Status)}
+       || maps:is_key(recipient_conn_avg, Status)
+      ] ++
+      [{disconnected_session_goal, maps:get(recipient_sess_avg, Status)}
+       || maps:is_key(recipient_sess_avg, Status)
+      ] ++
+      [{donor_conn_avg, maps:get(donor_conn_avg, Status)}
+       || maps:is_key(donor_conn_avg, Status)
+      ] ++
+      [{donor_sess_avg, maps:get(donor_sess_avg, Status)}
+       || maps:is_key(donor_sess_avg, Status)
+      ]).
+
+local_status_field_format_order() ->
+    [state,
+     coordinator_node,
+     connection_eviction_rate,
+     session_eviction_rate,
+     connection_goal,
+     session_goal,
+     disconnected_session_goal,
+     session_recipients,
+     recipients,
+     stats].
+
+coordinator_status_field_format_order() ->
+    [state,
+     coordinator_node,
+     donors,
+     recipients,
+     connection_eviction_rate,
+     session_eviction_rate,
+     connection_goal,
+     disconnected_session_goal,
+     donor_conn_avg,
+     donor_sess_avg].
+
+format_status(Status, FieldOrder) ->
+    Fields = lists:flatmap(
+               fun(FieldName) ->
+                       maps:to_list(maps:with([FieldName], Status))
+               end,
+               FieldOrder),
+    lists:map(
+      fun format_local_status_field/1,
+      Fields).
+
+format_local_status_field({state, State}) ->
+    io_lib:format("Rebalance state: ~p~n", [State]);
+format_local_status_field({coordinator_node, Node}) ->
+    io_lib:format("Coordinator node: ~p~n", [Node]);
+format_local_status_field({connection_eviction_rate, ConnEvictRate}) ->
+    io_lib:format("Connection eviction rate: ~p connections/second~n", [ConnEvictRate]);
+format_local_status_field({session_eviction_rate, SessEvictRate}) ->
+    io_lib:format("Session eviction rate: ~p sessions/second~n", [SessEvictRate]);
+format_local_status_field({connection_goal, ConnGoal}) ->
+    io_lib:format("Connection goal: ~p~n", [ConnGoal]);
+format_local_status_field({session_goal, SessGoal}) ->
+    io_lib:format("Session goal: ~p~n", [SessGoal]);
+format_local_status_field({disconnected_session_goal, DisconnSessGoal}) ->
+    io_lib:format("Disconnected session goal: ~p~n", [DisconnSessGoal]);
+format_local_status_field({session_recipients, SessionRecipients}) ->
+    io_lib:format("Session recipient  nodes: ~p~n", [SessionRecipients]);
+format_local_status_field({recipients, Recipients}) ->
+    io_lib:format("Recipient nodes: ~p~n", [Recipients]);
+format_local_status_field({donors, Donors}) ->
+    io_lib:format("Donor nodes: ~p~n", [Donors]);
+format_local_status_field({donor_conn_avg, DonorConnAvg}) ->
+    io_lib:format("Current average donor node connection count: ~p~n", [DonorConnAvg]);
+format_local_status_field({donor_sess_avg, DonorSessAvg}) ->
+    io_lib:format("Current average donor node disconnected session count: ~p~n", [DonorSessAvg]);
+format_local_status_field({stats, Stats}) ->
+    format_local_stats(Stats).
+
+format_local_stats(Stats) ->
+    ["Channel statistics:\n" |
+     lists:map(
+       fun({Name, Value}) ->
+               io_lib:format("  ~p: ~p~n", [Name, Value])
+       end,
+       maps:to_list(Stats))].
+
+evacuation_status() ->
+    {node(), emqx_node_rebalance_evacuation:status()}.
+
+rebalance_status() ->
+    {node(), emqx_node_rebalance:status()}.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl
new file mode 100644
index 000000000..e677eb22e
--- /dev/null
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl
@@ -0,0 +1,44 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/1]).
+
+-export([init/1]).
+
+start_link(Env) ->
+	supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]).
+
+init([_Env]) ->
+    Childs = [child_spec(emqx_node_rebalance_evacuation, []),
+              child_spec(emqx_node_rebalance_agent, []),
+              child_spec(emqx_node_rebalance, [])],
+    {ok, {
+       {one_for_one, 10, 3600},
+       Childs}
+    }.
+
+child_spec(Mod, Args) ->
+    #{id => Mod,
+      start => {Mod, start_link, Args},
+      restart => permanent,
+      shutdown => 5000,
+      type => worker,
+      modules => [Mod]
+     }.
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl
new file mode 100644
index 000000000..c5c27aec0
--- /dev/null
+++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl
@@ -0,0 +1,183 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-import(emqx_eviction_agent_test_helpers,
+        [emqtt_connect_many/1]).
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
+    Config.
+
+end_per_suite(Config) ->
+    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
+    Config.
+
+init_per_testcase(_Case, Config) ->
+    _ = emqx_node_rebalance:stop(),
+    Node = emqx_node_helpers:start_slave(
+             recipient1,
+             #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}),
+    [{recipient_node, Node} | Config].
+
+end_per_testcase(_Case, Config) ->
+    _ = emqx_node_helpers:stop_slave(?config(recipient_node, Config)),
+    _ = emqx_node_rebalance:stop().
+
+t_rebalance(Config) ->
+    process_flag(trap_exit, true),
+    RecipientNode = ?config(recipient_node, Config),
+
+    Nodes = [node(), RecipientNode],
+
+    _Conns = emqtt_connect_many(500),
+
+    Opts = #{conn_evict_rate => 10,
+             sess_evict_rate => 10,
+             evict_interval => 10,
+             abs_conn_threshold => 50,
+             abs_sess_threshold => 50,
+             rel_conn_threshold => 1.0,
+             rel_sess_threshold => 1.0,
+             wait_health_check => 0.01,
+             wait_takeover => 0.01,
+             nodes => Nodes
+            },
+
+    ?check_trace(
+       ?wait_async_action(
+          emqx_node_rebalance:start(Opts),
+          #{?snk_kind := emqx_node_rebalance_evict_sess_over},
+          10000),
+       fun({ok, _}, Trace) ->
+        ?assertMatch(
+           [_ | _],
+           ?of_kind(emqx_node_rebalance_evict_sess_over, Trace))
+       end),
+
+    DonorConnCount = emqx_eviction_agent:connection_count(),
+    DonorSessCount = emqx_eviction_agent:session_count(),
+    DonorDSessCount = emqx_eviction_agent:session_count(disconnected),
+
+    RecipientConnCount = rpc:call(RecipientNode, emqx_eviction_agent, connection_count, []),
+    RecipientSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, []),
+    RecipientDSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, [disconnected]),
+
+    ct:pal("Donor: conn=~p, sess=~p, dsess=~p",
+             [DonorConnCount, DonorSessCount, DonorDSessCount]),
+    ct:pal("Recipient: conn=~p, sess=~p, dsess=~p",
+             [RecipientConnCount, RecipientSessCount, RecipientDSessCount]),
+
+    ?assert(DonorConnCount - 50 =< RecipientConnCount),
+    ?assert(DonorDSessCount - 50 =< RecipientDSessCount).
+
+t_rebalance_node_crash(Config) ->
+    process_flag(trap_exit, true),
+    RecipientNode = ?config(recipient_node, Config),
+
+    Nodes = [node(), RecipientNode],
+
+    _Conns = emqtt_connect_many(50),
+
+    Opts = #{conn_evict_rate => 10,
+             sess_evict_rate => 10,
+             evict_interval => 10,
+             abs_conn_threshold => 50,
+             abs_sess_threshold => 50,
+             rel_conn_threshold => 1.0,
+             rel_sess_threshold => 1.0,
+             wait_health_check => 0.01,
+             wait_takeover => 0.01,
+             nodes => Nodes
+            },
+
+    ok = emqx_node_rebalance:start(Opts),
+
+    ?check_trace(
+       ?wait_async_action(
+          emqx_node_helpers:stop_slave(?config(recipient_node, Config)),
+          #{?snk_kind := emqx_node_rebalance_started},
+          1000),
+       fun(_Result, _Trace) -> ok end),
+
+    ?assertEqual(
+       disabled,
+       emqx_node_rebalance:status()).
+
+t_no_need_to_rebalance(_Config) ->
+    process_flag(trap_exit, true),
+
+    ?assertEqual(
+        {error, nothing_to_balance},
+        emqx_node_rebalance:start(#{})),
+
+    _Conns = emqtt_connect_many(50),
+
+    ?assertEqual(
+        {error, nothing_to_balance},
+        emqx_node_rebalance:start(#{})).
+
+t_unknown_mesages(Config) ->
+    process_flag(trap_exit, true),
+    RecipientNode = ?config(recipient_node, Config),
+
+    Nodes = [node(), RecipientNode],
+
+    _Conns = emqtt_connect_many(500),
+
+    Opts = #{wait_health_check => 100,
+             abs_conn_threshold => 50,
+             nodes => Nodes
+            },
+
+    Pid = whereis(emqx_node_rebalance),
+
+    Pid ! unknown,
+    ok = gen_server:cast(Pid, unknown),
+    ?assertEqual(
+       ignored,
+       gen_server:call(Pid, unknown)),
+
+    ok = emqx_node_rebalance:start(Opts),
+
+    Pid ! unknown,
+    ok = gen_server:cast(Pid, unknown),
+    ?assertEqual(
+       ignored,
+       gen_server:call(Pid, unknown)).
+
+t_available_nodes(Config) ->
+    rpc:call(?config(recipient_node, Config),
+             emqx_eviction_agent,
+             enable,
+             [test_rebalance, undefined]),
+    ?assertEqual(
+       [node()],
+       emqx_node_rebalance:available_nodes(
+         [node(), ?config(recipient_node, Config)])).
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl
new file mode 100644
index 000000000..30edd51fe
--- /dev/null
+++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl
@@ -0,0 +1,163 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_agent_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
+    Config.
+
+end_per_suite(Config) ->
+    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
+    Config.
+
+init_per_testcase(_Case, Config) ->
+    _ = emqx_node_rebalance_evacuation:stop(),
+    Node = emqx_node_helpers:start_slave(
+             evacuate1,
+             #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}),
+    [{evacuate_node, Node} | Config].
+
+end_per_testcase(_Case, Config) ->
+    _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)),
+    _ = emqx_node_rebalance_evacuation:stop().
+
+t_enable_disable(_Config) ->
+    ?assertEqual(
+       disabled,
+       emqx_node_rebalance_agent:status()),
+
+    ?assertEqual(
+       ok,
+       emqx_node_rebalance_agent:enable(self())),
+
+    ?assertEqual(
+       {error, already_enabled},
+       emqx_node_rebalance_agent:enable(self())),
+
+    ?assertEqual(
+       {enabled, self()},
+       emqx_node_rebalance_agent:status()),
+
+    ?assertEqual(
+       {error, invalid_coordinator},
+       emqx_node_rebalance_agent:disable(spawn_link(fun() -> ok end))),
+
+    ?assertEqual(
+       ok,
+       emqx_node_rebalance_agent:disable(self())),
+
+    ?assertEqual(
+       {error, already_disabled},
+       emqx_node_rebalance_agent:disable(self())),
+
+    ?assertEqual(
+       disabled,
+       emqx_node_rebalance_agent:status()).
+
+t_enable_egent_busy(_Config) ->
+    ok = emqx_eviction_agent:enable(rebalance_test, undefined),
+
+    ?assertEqual(
+       {error, eviction_agent_busy},
+       emqx_node_rebalance_agent:enable(self())),
+
+    ok = emqx_eviction_agent:disable(rebalance_test).
+
+% The following tests verify that emqx_node_rebalance_agent correctly links
+% coordinator process with emqx_eviction_agent-s.
+
+t_rebalance_agent_coordinator_fail(Config) ->
+    process_flag(trap_exit, true),
+
+    Node = ?config(evacuate_node, Config),
+
+
+    CoordinatorPid = spawn_link(
+                       fun() ->
+                               receive
+                                done -> ok
+                               end
+                       end),
+
+    ?assertEqual(
+       disabled,
+       rpc:call(Node, emqx_eviction_agent, status, [])),
+
+    ?assertEqual(
+       ok,
+       rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid])),
+
+    ?assertMatch(
+       {enabled, _},
+       rpc:call(Node, emqx_eviction_agent, status, [])),
+
+    EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]),
+    true = link(EvictionAgentPid),
+
+    true = exit(CoordinatorPid, kill),
+
+    receive
+        {'EXIT', EvictionAgentPid, _} -> true
+    after
+        1000 -> ?assert(false, "emqx_eviction_agent did not exit")
+    end.
+
+t_rebalance_agent_fail(Config) ->
+    process_flag(trap_exit, true),
+
+    Node = ?config(evacuate_node, Config),
+
+    CoordinatorPid = spawn_link(
+                       fun() ->
+                               receive
+                                done -> ok
+                               end
+                       end),
+
+    ?assertEqual(
+       ok,
+       rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid])),
+
+    EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]),
+    true = exit(EvictionAgentPid, kill),
+
+    receive
+        {'EXIT', CoordinatorPid, _} -> true
+    after
+        1000 -> ?assert(false, "emqx_eviction_agent did not exit")
+    end.
+
+t_unknown_messages(_Config) ->
+    Pid = whereis(emqx_node_rebalance_agent),
+
+    ok = gen_server:cast(Pid, unknown),
+
+    Pid ! unknown,
+
+    ignored = gen_server:call(Pid, unknown).
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl
new file mode 100644
index 000000000..a15b8e8ab
--- /dev/null
+++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl
@@ -0,0 +1,321 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_api_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-import(emqx_mgmt_api_test_helpers,
+        [request_api/3,
+         request_api/5,
+         auth_header_/0,
+         api_path/1]).
+
+-import(emqx_eviction_agent_test_helpers,
+        [emqtt_connect_many/1]).
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance, emqx_management]),
+    Config.
+
+end_per_suite(Config) ->
+    emqx_ct_helpers:stop_apps([emqx_management, emqx_node_rebalance, emqx_eviction_agent]),
+    Config.
+
+init_per_testcase(Case, Config)
+  when Case =:= t_start_evacuation_validation
+       orelse Case =:= t_start_rebalance_validation
+       orelse Case =:= t_start_stop_rebalance ->
+    _ = emqx_node_rebalance:stop(),
+    _ = emqx_node_rebalance_evacuation:stop(),
+    Node = emqx_node_helpers:start_slave(
+             recipient1,
+             #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}),
+    [{recipient_node, Node} | Config];
+init_per_testcase(_Case, Config) ->
+    _ = emqx_node_rebalance:stop(),
+    _ = emqx_node_rebalance_evacuation:stop(),
+    Config.
+
+end_per_testcase(Case, Config)
+  when Case =:= t_start_evacuation_validation
+       orelse Case =:= t_start_rebalance_validation
+       orelse Case =:= t_start_stop_rebalance ->
+    _ = emqx_node_helpers:stop_slave(?config(recipient_node, Config)),
+    _ = emqx_node_rebalance:stop(),
+    _ = emqx_node_rebalance_evacuation:stop();
+end_per_testcase(_Case, _Config) ->
+    _ = emqx_node_rebalance:stop(),
+    _ = emqx_node_rebalance_evacuation:stop().
+
+t_start_evacuation_validation(Config) ->
+    BadOpts = [#{conn_evict_rate => <<"conn">>},
+               #{sess_evict_rate => <<"sess">>},
+               #{redirect_to => 123},
+               #{wait_takeover => <<"wait">>},
+               #{migrate_to => []},
+               #{migrate_to => <<"migrate_to">>},
+               #{migrate_to => [<<"bad_node">>]},
+               #{migrate_to => [<<"bad_node">>, atom_to_binary(node())]},
+               #{unknown => <<"Value">>}
+              ],
+    lists:foreach(
+      fun(Opts) ->
+              ?assertMatch(
+                 {ok, #{}},
+                 api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"],
+                          Opts)),
+
+              ?assertMatch(
+                 {ok, #{<<"status">> := <<"disabled">>}},
+                 api_get(["load_rebalance", "status"]))
+
+      end,
+      BadOpts),
+    ?assertMatch(
+       {ok, #{}},
+       api_post(["load_rebalance", "bad@node", "evacuation", "start"],
+                #{})),
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"disabled">>}},
+       api_get(["load_rebalance", "status"])),
+
+    ?assertMatch(
+       {ok, #{}},
+       api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"],
+                #{conn_evict_rate => 10,
+                  sess_evict_rate => 10,
+                  wait_takeover => 10,
+                  redirect_to => <<"srv">>,
+                  migrate_to => [atom_to_binary(?config(recipient_node, Config))]})),
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"enabled">>}},
+       api_get(["load_rebalance", "status"])).
+
+
+t_start_rebalance_validation(Config) ->
+    BadOpts = [#{conn_evict_rate => <<"conn">>},
+               #{sess_evict_rate => <<"sess">>},
+               #{abs_conn_threshold => <<"act">>},
+               #{rel_conn_threshold => <<"rct">>},
+               #{abs_sess_threshold => <<"act">>},
+               #{rel_sess_threshold => <<"rct">>},
+               #{wait_takeover => <<"wait">>},
+               #{wait_health_check => <<"wait">>},
+               #{nodes => <<"nodes">>},
+               #{nodes => []},
+               #{nodes => [<<"bad_node">>]},
+               #{nodes => [<<"bad_node">>, atom_to_binary(node())]},
+               #{unknown => <<"Value">>}
+              ],
+    lists:foreach(
+      fun(Opts) ->
+              ?assertMatch(
+                 {ok, #{}},
+                 api_post(["load_rebalance", atom_to_list(node()), "start"],
+                          Opts)),
+
+              ?assertMatch(
+                 {ok, #{<<"status">> := <<"disabled">>}},
+                 api_get(["load_rebalance", "status"]))
+
+      end,
+      BadOpts),
+    ?assertMatch(
+       {ok, #{}},
+       api_post(["load_rebalance", "bad@node", "start"],
+                #{})),
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"disabled">>}},
+       api_get(["load_rebalance", "status"])),
+
+    _Conns = emqtt_connect_many(50),
+
+    ?assertMatch(
+       {ok, #{}},
+       api_post(["load_rebalance", atom_to_list(node()), "start"],
+                #{conn_evict_rate => 10,
+                  sess_evict_rate => 10,
+                  wait_takeover => 10,
+                  wait_health_check => 10,
+                  abs_conn_threshold => 10,
+                  rel_conn_threshold => 1.001,
+                  abs_sess_threshold => 10,
+                  rel_sess_threshold => 1.001,
+                  nodes => [atom_to_binary(?config(recipient_node, Config)),
+                            atom_to_binary(node())]})),
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"enabled">>}},
+       api_get(["load_rebalance", "status"])).
+
+t_start_stop_evacuation(_Config) ->
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"disabled">>}},
+        api_get(["load_rebalance", "status"])),
+
+    ?assertMatch(
+       {ok, #{}},
+        api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"],
+                 #{conn_evict_rate => 10,
+                   sess_evict_rate => 20})),
+
+    ?assertMatch(
+       {ok, #{<<"state">> := _,
+              <<"process">> := <<"evacuation">>,
+              <<"connection_eviction_rate">> := 10,
+              <<"session_eviction_rate">> := 20,
+              <<"connection_goal">> := 0,
+              <<"session_goal">> := 0,
+              <<"stats">> := #{
+                               <<"initial_connected">> := _,
+                               <<"current_connected">> := _,
+                               <<"initial_sessions">> := _,
+                               <<"current_sessions">> := _
+                              }}},
+        api_get(["load_rebalance", "status"])),
+
+    ?assertMatch(
+       {ok, #{<<"rebalances">> := #{},
+              <<"evacuations">> :=
+              #{<<"test@127.0.0.1">> := #{<<"state">> := _,
+                                          <<"connection_eviction_rate">> := 10,
+                                          <<"session_eviction_rate">> := 20,
+                                          <<"connection_goal">> := 0,
+                                          <<"session_goal">> := 0,
+                                          <<"stats">> := #{
+                                                           <<"initial_connected">> := _,
+                                                           <<"current_connected">> := _,
+                                                           <<"initial_sessions">> := _,
+                                                           <<"current_sessions">> := _
+                                                          }
+                                         }
+               }}},
+        api_get(["load_rebalance", "global_status"])),
+
+    ?assertMatch(
+       {ok, #{}},
+        api_post(["load_rebalance", atom_to_list(node()), "evacuation", "stop"],
+                 #{})),
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"disabled">>}},
+        api_get(["load_rebalance", "status"])),
+
+    ?assertMatch(
+       {ok, #{<<"evacuations">> := #{}, <<"rebalances">> := #{}}},
+        api_get(["load_rebalance", "global_status"])).
+
+t_start_stop_rebalance(Config) ->
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"disabled">>}},
+        api_get(["load_rebalance", "status"])),
+
+    _Conns = emqtt_connect_many(100),
+
+    ?assertMatch(
+       {ok, #{}},
+        api_post(["load_rebalance", atom_to_list(node()), "start"],
+                 #{conn_evict_rate => 10,
+                   sess_evict_rate => 20,
+                   abs_conn_threshold => 10})),
+
+    ?assertMatch(
+       {ok, #{<<"state">> := _,
+              <<"process">> := <<"rebalance">>,
+              <<"coordinator_node">> := _,
+              <<"connection_eviction_rate">> := 10,
+              <<"session_eviction_rate">> := 20,
+              <<"stats">> := #{
+                               <<"initial_connected">> := _,
+                               <<"current_connected">> := _,
+                               <<"initial_sessions">> := _,
+                               <<"current_sessions">> := _
+                              }}},
+        api_get(["load_rebalance", "status"])),
+
+    DonorNode = atom_to_binary(node()),
+    RecipientNode = atom_to_binary(?config(recipient_node, Config)),
+
+    ?assertMatch(
+       {ok, #{<<"evacuations">> := #{},
+              <<"rebalances">> :=
+              #{<<"test@127.0.0.1">> := #{<<"state">> := _,
+                                          <<"coordinator_node">> := _,
+                                          <<"connection_eviction_rate">> := 10,
+                                          <<"session_eviction_rate">> := 20,
+                                          <<"donors">> := [DonorNode],
+                                          <<"recipients">> := [RecipientNode]
+                                         }
+               }}},
+        api_get(["load_rebalance", "global_status"])),
+
+    ?assertMatch(
+       {ok, #{}},
+        api_post(["load_rebalance", atom_to_list(node()), "stop"],
+                 #{})),
+
+    ?assertMatch(
+       {ok, #{<<"status">> := <<"disabled">>}},
+        api_get(["load_rebalance", "status"])),
+
+    ?assertMatch(
+       {ok, #{<<"evacuations">> := #{}, <<"rebalances">> := #{}}},
+        api_get(["load_rebalance", "global_status"])).
+
+t_availability_check(_Config) ->
+
+    ?assertMatch(
+       {ok, #{}},
+       api_get(["load_rebalance", "availability_check"])),
+
+    ok = emqx_node_rebalance_evacuation:start(#{}),
+
+    ?assertMatch(
+       {error, {_, 503, _}},
+       api_get(["load_rebalance", "availability_check"])),
+
+    ok = emqx_node_rebalance_evacuation:stop(),
+
+    ?assertMatch(
+       {ok, #{}},
+       api_get(["load_rebalance", "availability_check"])).
+
+api_get(Path) ->
+    case request_api(get, api_path(Path), auth_header_()) of
+        {ok, ResponseBody} ->
+            {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])};
+        {error, _} = Error -> Error
+    end.
+
+api_post(Path, Data) ->
+    case request_api(post, api_path(Path), [], auth_header_(), Data) of
+        {ok, ResponseBody} ->
+            {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])};
+        {error, _} = Error -> Error
+    end.
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl
new file mode 100644
index 000000000..943746298
--- /dev/null
+++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl
@@ -0,0 +1,199 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_cli_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-import(emqx_eviction_agent_test_helpers,
+        [emqtt_connect_many/1]).
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
+    Config.
+
+end_per_suite(Config) ->
+    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
+    Config.
+
+
+init_per_testcase(t_rebalance, Config) ->
+    _ = emqx_node_rebalance_evacuation:stop(),
+    Node = emqx_node_helpers:start_slave(
+             evacuate1,
+             #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}),
+    [{evacuate_node, Node} | Config];
+init_per_testcase(_Case, Config) ->
+    _ = emqx_node_rebalance_evacuation:stop(),
+    _ = emqx_node_rebalance:stop(),
+    Config.
+
+end_per_testcase(t_rebalance, Config) ->
+    _ = emqx_node_rebalance_evacuation:stop(),
+    _ = emqx_node_rebalance:stop(),
+    _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config));
+end_per_testcase(_Case, _Config) ->
+    _ = emqx_node_rebalance_evacuation:stop(),
+    _ = emqx_node_rebalance:stop().
+
+t_evacuation(_Config) ->
+    %% usage
+    ok = emqx_node_rebalance_cli:cli(["foobar"]),
+
+    %% status
+    ok = emqx_node_rebalance_cli:cli(["status"]),
+    ok = emqx_node_rebalance_cli:cli(["node-status"]),
+    ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]),
+
+    %% start with invalid args
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--evacuation", "--foo-bar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--evacuation", "--conn-evict-rate", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--evacuation", "--sess-evict-rate", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--evacuation", "--wait-takeover", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--evacuation",
+                                     "--migrate-to", "nonexistent@node"])),
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--evacuation",
+                                     "--migrate-to", ""])),
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--evacuation",
+                                     "--unknown-arg"])),
+    ?assert(
+        emqx_node_rebalance_cli:cli(["start", "--evacuation",
+                                     "--conn-evict-rate", "10",
+                                     "--sess-evict-rate", "10",
+                                     "--wait-takeover", "10",
+                                     "--migrate-to", atom_to_list(node()),
+                                     "--redirect-to", "srv"])),
+
+    %% status
+    ok = emqx_node_rebalance_cli:cli(["status"]),
+    ok = emqx_node_rebalance_cli:cli(["node-status"]),
+    ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]),
+
+    ?assertMatch(
+       {enabled, #{}},
+       emqx_node_rebalance_evacuation:status()),
+
+    %% already enabled
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--evacuation",
+                                     "--conn-evict-rate", "10",
+                                     "--redirect-to", "srv"])),
+
+    %% stop
+    true = emqx_node_rebalance_cli:cli(["stop"]),
+
+    false = emqx_node_rebalance_cli:cli(["stop"]),
+
+    ?assertEqual(
+       disabled,
+       emqx_node_rebalance_evacuation:status()).
+
+t_rebalance(Config) ->
+    %% start with invalid args
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--foo-bar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--conn-evict-rate", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--abs-conn-threshold", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--rel-conn-threshold", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--sess-evict-rate", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--abs-sess-threshold", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--rel-sess-threshold", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--wait-takeover", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start", "--wait-health-check", "foobar"])),
+
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start",
+                                     "--nodes", "nonexistent@node"])),
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start",
+                                     "--nodes", ""])),
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start",
+                                     "--nodes", atom_to_list(?config(evacuate_node, Config))])),
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start",
+                                     "--unknown-arg"])),
+
+    _ = emqtt_connect_many(20),
+
+    ?assert(
+        emqx_node_rebalance_cli:cli(["start",
+                                     "--conn-evict-rate", "10",
+                                     "--abs-conn-threshold", "10",
+                                     "--rel-conn-threshold", "1.1",
+                                     "--sess-evict-rate", "10",
+                                     "--abs-sess-threshold", "10",
+                                     "--rel-sess-threshold", "1.1",
+                                     "--wait-takeover", "10",
+                                     "--nodes", atom_to_list(node()) ++ ","
+                                                ++ atom_to_list(?config(evacuate_node, Config))
+                                    ])),
+
+    %% status
+    ok = emqx_node_rebalance_cli:cli(["status"]),
+    ok = emqx_node_rebalance_cli:cli(["node-status"]),
+    ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]),
+
+    ?assertMatch(
+       {enabled, #{}},
+       emqx_node_rebalance:status()),
+
+    %% already enabled
+    ?assertNot(
+        emqx_node_rebalance_cli:cli(["start"])),
+
+    %% stop
+    true = emqx_node_rebalance_cli:cli(["stop"]),
+
+    false = emqx_node_rebalance_cli:cli(["stop"]),
+
+    ?assertEqual(
+       disabled,
+       emqx_node_rebalance:status()).
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl
new file mode 100644
index 000000000..23f35f61e
--- /dev/null
+++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl
@@ -0,0 +1,194 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_evacuation_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-import(emqx_eviction_agent_test_helpers,
+        [emqtt_connect/0, emqtt_connect/2, emqtt_try_connect/0]).
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
+    Config.
+
+end_per_suite(Config) ->
+    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
+    Config.
+
+init_per_testcase(_Case, Config) ->
+    _ = emqx_node_rebalance_evacuation:stop(),
+    Node = emqx_node_helpers:start_slave(
+             evacuate1,
+             #{start_apps => [emqx, emqx_eviction_agent]}),
+    [{evacuate_node, Node} | Config].
+
+end_per_testcase(_Case, Config) ->
+    _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)),
+    _ = emqx_node_rebalance_evacuation:stop().
+
+t_agent_busy(Config) ->
+
+    ok = emqx_eviction_agent:enable(other_rebalance, undefined),
+
+    ?assertEqual(
+       {error, eviction_agent_busy},
+       emqx_node_rebalance_evacuation:start(opts(Config))),
+
+    emqx_eviction_agent:disable(other_rebalance).
+
+
+t_already_started(Config) ->
+    ok = emqx_node_rebalance_evacuation:start(opts(Config)),
+
+    ?assertEqual(
+       {error, already_started},
+       emqx_node_rebalance_evacuation:start(opts(Config))),
+
+    ok = emqx_node_rebalance_evacuation:stop().
+
+t_not_started(_Config) ->
+    ?assertEqual(
+       {error, not_started},
+       emqx_node_rebalance_evacuation:stop()).
+
+t_start(Config) ->
+    process_flag(trap_exit, true),
+
+    ok = emqx_node_rebalance_evacuation:start(opts(Config)),
+    ?assertMatch(
+       {error, {use_another_server, #{}}},
+       emqtt_try_connect()),
+    ok = emqx_node_rebalance_evacuation:stop().
+
+t_persistence(Config) ->
+    process_flag(trap_exit, true),
+
+    ok = emqx_node_rebalance_evacuation:start(opts(Config)),
+
+    ?assertMatch(
+       {error, {use_another_server, #{}}},
+       emqtt_try_connect()),
+
+    ok = supervisor:terminate_child(emqx_node_rebalance_sup, emqx_node_rebalance_evacuation),
+    {ok, _} = supervisor:restart_child(emqx_node_rebalance_sup, emqx_node_rebalance_evacuation),
+
+    ?assertMatch(
+       {error, {use_another_server, #{}}},
+       emqtt_try_connect()),
+    ?assertMatch(
+       {enabled, #{conn_evict_rate := 10}},
+       emqx_node_rebalance_evacuation:status()),
+
+    ok = emqx_node_rebalance_evacuation:stop().
+
+t_conn_evicted(Config) ->
+    process_flag(trap_exit, true),
+
+    {ok, C} = emqtt_connect(),
+
+    ?check_trace(
+       ?wait_async_action(
+          emqx_node_rebalance_evacuation:start(opts(Config)),
+          #{?snk_kind := node_evacuation_evict_conn},
+          1000),
+       fun(_Result, _Trace) -> ok end),
+
+    ct:sleep(100),
+
+    ?assertMatch(
+       {error, {use_another_server, #{}}},
+       emqtt_try_connect()),
+
+    ?assertNot(
+       is_process_alive(C)).
+
+t_migrate_to(Config) ->
+    ?assertEqual(
+       [?config(evacuate_node, Config)],
+       emqx_node_rebalance_evacuation:migrate_to(undefined)),
+
+    ?assertEqual(
+       [],
+       emqx_node_rebalance_evacuation:migrate_to(['unknown@node'])),
+
+    rpc:call(?config(evacuate_node, Config),
+             emqx_eviction_agent,
+             enable,
+             [test_rebalance, undefined]),
+
+    ?assertEqual(
+       [],
+       emqx_node_rebalance_evacuation:migrate_to(undefined)).
+
+t_session_evicted(Config) ->
+    process_flag(trap_exit, true),
+
+    {ok, C} = emqtt_connect(<<"client_with_sess">>, false),
+
+    ?check_trace(
+       ?wait_async_action(
+          emqx_node_rebalance_evacuation:start(opts(Config)),
+          #{?snk_kind := node_evacuation_evict_sess_over},
+          5000),
+       fun(_Result, Trace) ->
+        ?assertMatch(
+           [_ | _],
+           ?of_kind(node_evacuation_evict_sess_over, Trace))
+       end),
+
+    receive
+        {'EXIT', C, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok
+    after 1000 ->
+              ?assert(false, "Connection not evicted")
+    end,
+
+    [ChannelPid] = emqx_cm_registry:lookup_channels(<<"client_with_sess">>),
+
+    ?assertEqual(
+       ?config(evacuate_node, Config),
+       node(ChannelPid)).
+
+t_unknown_messages(Config) ->
+    ok = emqx_node_rebalance_evacuation:start(opts(Config)),
+
+    whereis(emqx_node_rebalance_evacuation) ! unknown,
+
+    gen_server:cast(emqx_node_rebalance_evacuation, unknown),
+
+    ?assertEqual(
+       ignored,
+       gen_server:call(emqx_node_rebalance_evacuation, unknown)),
+
+    ok = emqx_node_rebalance_evacuation:stop().
+
+opts(Config) ->
+    #{
+      server_reference => <<"srv">>,
+      conn_evict_rate => 10,
+      sess_evict_rate => 10,
+      wait_takeover => 1,
+      migrate_to => [?config(evacuate_node, Config)]
+     }.
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl
new file mode 100644
index 000000000..ba1a12775
--- /dev/null
+++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl
@@ -0,0 +1,107 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_rebalance_evacuation_persist_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+all() ->
+    emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
+    Config.
+
+end_per_suite(Config) ->
+    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
+    Config.
+
+init_per_testcase(_Case, Config) ->
+    _ = emqx_node_rebalance_evacuation_persist:clear(),
+    Config.
+
+end_per_testcase(_Case, _Config) ->
+    _ = emqx_node_rebalance_evacuation_persist:clear().
+
+t_save_read(_Config) ->
+    DefaultOpts = #{server_reference => <<"default_ref">>,
+                    conn_evict_rate => 2001,
+                    sess_evict_rate => 2002,
+                    wait_takeover => 2003
+                   },
+
+    Opts0 = #{server_reference => <<"ref">>,
+             conn_evict_rate => 1001,
+             sess_evict_rate => 1002,
+             wait_takeover => 1003
+            },
+    ok = emqx_node_rebalance_evacuation_persist:save(Opts0),
+
+    {ok, ReadOpts0} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts),
+    ?assertEqual(Opts0, ReadOpts0),
+
+    Opts1 = Opts0#{server_reference => undefined},
+    ok = emqx_node_rebalance_evacuation_persist:save(Opts1),
+
+    {ok, ReadOpts1} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts),
+    ?assertEqual(Opts1, ReadOpts1).
+
+t_read_default(_Config) ->
+    ok = write_evacuation_file(<<"{}">>),
+
+    DefaultOpts = #{server_reference => <<"ref">>,
+                    conn_evict_rate => 1001,
+                    sess_evict_rate => 1002,
+                    wait_takeover => 1003
+                   },
+
+    {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts),
+    ?assertEqual(DefaultOpts, ReadOpts).
+
+t_read_bad_data(_Config) ->
+    ok = write_evacuation_file(<<"{bad json">>),
+
+    DefaultOpts = #{server_reference => <<"ref">>,
+                    conn_evict_rate => 1001,
+                    sess_evict_rate => 1002,
+                    wait_takeover => 1003
+                   },
+
+    {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts),
+    ?assertEqual(DefaultOpts, ReadOpts).
+
+t_clear(_Config) ->
+    ok = write_evacuation_file(<<"{}">>),
+
+    ?assertMatch(
+        {ok, _},
+        emqx_node_rebalance_evacuation_persist:read(#{})),
+
+    ok = emqx_node_rebalance_evacuation_persist:clear(),
+
+    ?assertEqual(
+        none,
+        emqx_node_rebalance_evacuation_persist:read(#{})).
+
+write_evacuation_file(Json) ->
+    ok = filelib:ensure_dir(emqx_node_rebalance_evacuation_persist:evacuation_filepath()),
+    ok = file:write_file(
+           emqx_node_rebalance_evacuation_persist:evacuation_filepath(),
+           Json).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index 259efa08d..90620e2df 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -1,6 +1,6 @@
 {application, emqx_rule_engine,
  [{description, "EMQ X Rule Engine"},
-  {vsn, "4.3.13"}, % strict semver, bump manually!
+  {vsn, "4.3.14"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_rule_engine_sup, emqx_rule_registry]},
   {applications, [kernel,stdlib,rulesql,getopt]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index ba39c4ec9..1c5fb0e7b 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -1,7 +1,8 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
@@ -165,7 +166,8 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl
index d0dac7fe1..62b3205cd 100644
--- a/data/loaded_plugins.tmpl
+++ b/data/loaded_plugins.tmpl
@@ -6,3 +6,5 @@
 {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}.
 {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}.
 {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}.
+{emqx_eviction_agent, true}.
+{emqx_node_rebalance, true}.
diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
index 1281145f0..3c9a4e3fa 100644
--- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
@@ -1,6 +1,6 @@
 {application, emqx_dashboard,
  [{description, "EMQ X Web Dashboard"},
-  {vsn, "4.3.14"}, % strict semver, bump manually!
+  {vsn, "4.3.15"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_dashboard_sup]},
   {applications, [kernel,stdlib,mnesia,minirest]},
diff --git a/rebar.config.erl b/rebar.config.erl
index 78e1ef94e..04ffaefbd 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -51,6 +51,8 @@ overrides() ->
     [ {add, [ {extra_src_dirs, [{"etc", [{recursive,true}]}]}
             , {erl_opts, [{compile_info, [{emqx_vsn, get_vsn()}]}]}
             ]}
+
+    , {add, relx, [{erl_opts, [{d, 'RLX_LOG', rlx_log}]}]}
     , {add, snabbkaffe,
        [{erl_opts, common_compile_opts()}]}
     ] ++ community_plugin_overrides().
@@ -295,6 +297,8 @@ relx_plugin_apps(ReleaseType) ->
     , emqx_recon
     , emqx_rule_engine
     , emqx_sasl
+    , emqx_eviction_agent
+    , emqx_node_rebalance
     ]
     ++ [emqx_telemetry || not is_enterprise()]
     ++ relx_plugin_apps_per_rel(ReleaseType)
diff --git a/src/emqx.app.src b/src/emqx.app.src
index 14a6f4fa4..14c63c75e 100644
--- a/src/emqx.app.src
+++ b/src/emqx.app.src
@@ -6,7 +6,7 @@
   %% the emqx `release' version, which in turn is comprised of several
   %% apps, one of which is this.  See `emqx_release.hrl' for more
   %% info.
-  {vsn, "4.3.19"}, % strict semver, bump manually!
+  {vsn, "4.3.20"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [ kernel
diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index 70d18ff48..b2be51df5 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -1,12 +1,19 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [
+  [{"4.3.19",
+    [{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]}]},
    {"4.3.18",
-    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
-    [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {update,emqx_broker_sup,supervisor},
@@ -14,7 +21,8 @@
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
-    [{load_module,emqx_session,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {update,emqx_broker_sup,supervisor},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]},
@@ -30,7 +38,8 @@
      {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]}]},
    {"4.3.15",
-    [{add_module,emqx_calendar},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -55,7 +64,8 @@
      {update,emqx_os_mon,{advanced,[]}},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.14",
-    [{add_module,emqx_calendar},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
      {apply,{emqx_exclusive_subscription,on_add_module,[]}},
@@ -682,12 +692,19 @@
      {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [
+  [{"4.3.19",
+    [{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]}]},
    {"4.3.18",
-     [{load_module,emqx_app,brutal_purge,soft_purge,[]},
-      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
-    [{load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {update,emqx_broker_sup,supervisor},
@@ -695,7 +712,8 @@
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
-    [{load_module,emqx_session,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {update,emqx_broker_sup,supervisor},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]},
@@ -711,7 +729,8 @@
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription}]},
    {"4.3.15",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
@@ -735,7 +754,8 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.3.14",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl
index bf292f10b..852c171fe 100644
--- a/src/emqx_channel.erl
+++ b/src/emqx_channel.erl
@@ -22,6 +22,8 @@
 -include("logger.hrl").
 -include("types.hrl").
 
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
 -logger_header("[Channel]").
 
 -ifdef(TEST).
@@ -850,11 +852,14 @@ handle_out(disconnect, ReasonCode, Channel) when is_integer(ReasonCode) ->
     ReasonName = disconnect_reason(ReasonCode),
     handle_out(disconnect, {ReasonCode, ReasonName}, Channel);
 
-handle_out(disconnect, {ReasonCode, ReasonName}, Channel = ?IS_MQTT_V5) ->
-    Packet = ?DISCONNECT_PACKET(ReasonCode),
+handle_out(disconnect, {ReasonCode, ReasonName}, Channel) ->
+    handle_out(disconnect, {ReasonCode, ReasonName, #{}}, Channel);
+
+handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel = ?IS_MQTT_V5) ->
+    Packet = ?DISCONNECT_PACKET(ReasonCode, Props),
     {ok, [{outgoing, Packet}, {close, ReasonName}], Channel};
 
-handle_out(disconnect, {_ReasonCode, ReasonName}, Channel) ->
+handle_out(disconnect, {_ReasonCode, ReasonName, _Props}, Channel) ->
     {ok, {close, ReasonName}, Channel};
 
 handle_out(auth, {ReasonCode, Properties}, Channel) ->
@@ -954,11 +959,15 @@ handle_call({takeover, 'begin'}, Channel = #channel{session = Session}) ->
     reply(Session, Channel#channel{takeover = true});
 
 handle_call({takeover, 'end'}, Channel = #channel{session  = Session,
-                                                  pendings = Pendings}) ->
+                                                  pendings = Pendings,
+                                                  conninfo = #{clientid := ClientId}}) ->
     ok = emqx_session:takeover(Session),
     %% TODO: Should not drain deliver here (side effect)
     Delivers = emqx_misc:drain_deliver(),
     AllPendings = lists:append(Delivers, Pendings),
+    ?tp(debug,
+        emqx_channel_takeover_end,
+        #{clientid => ClientId}),
     disconnect_and_shutdown(takeovered, AllPendings, Channel);
 
 handle_call(list_acl_cache, Channel) ->
@@ -1019,6 +1028,9 @@ handle_info(clean_acl_cache, Channel) ->
     ok = emqx_acl_cache:empty_acl_cache(),
     {ok, Channel};
 
+handle_info({disconnect, ReasonCode, ReasonName, Props}, Channel) ->
+    handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel);
+
 handle_info(Info, Channel) ->
     ?LOG(error, "Unexpected info: ~p", [Info]),
     {ok, Channel}.
diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl
index 09a2ac9a0..09169b160 100644
--- a/src/emqx_cm.erl
+++ b/src/emqx_cm.erl
@@ -22,6 +22,8 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 -include("types.hrl").
+-include_lib("stdlib/include/qlc.hrl").
+-include_lib("stdlib/include/ms_transform.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -logger_header("[CM]").
@@ -60,7 +62,9 @@
         , lookup_channels/2
         ]).
 
--export([all_channels/0]).
+-export([all_channels/0,
+         channel_with_session_table/0,
+         live_connection_table/0]).
 
 %% gen_server callbacks
 -export([ init/1
@@ -149,8 +153,11 @@ connection_closed(ClientId) ->
     connection_closed(ClientId, self()).
 
 -spec(connection_closed(emqx_types:clientid(), chan_pid()) -> true).
-connection_closed(ClientId, ChanPid) ->
-    ets:delete_object(?CHAN_CONN_TAB, {ClientId, ChanPid}).
+connection_closed(_ClientId, _ChanPid) ->
+    %% We can't clean CHAN_CONN_TAB because records for dead connections
+    %% are required for `get_chann_conn_mod/1` function, and `get_chann_conn_mod/1`
+    %% is used for takeover.
+    true.
 
 %% @doc Get info of a channel.
 -spec(get_chan_info(emqx_types:clientid()) -> maybe(emqx_types:infos())).
@@ -425,6 +432,38 @@ all_channels() ->
     Pat = [{{'_', '$1'}, [], ['$1']}],
     ets:select(?CHAN_TAB, Pat).
 
+%% @doc Get clientinfo for all clients with sessions
+channel_with_session_table() ->
+    Ms = ets:fun2ms(
+           fun({{ClientId, _ChanPid},
+                Info,
+                _Stats}) ->
+                   {ClientId, Info}
+           end),
+    Table = ets:table(?CHAN_INFO_TAB, [{traverse, {select, Ms}}]),
+    qlc:q([ {ClientId, ConnState, ConnInfo, ClientInfo}
+            || {ClientId,
+                #{conn_state := ConnState,
+                  clientinfo := ClientInfo,
+                  conninfo := #{clean_start := false} = ConnInfo}} <- Table
+          ]).
+
+%% @doc Get all local connection query handle
+live_connection_table() ->
+    Ms = ets:fun2ms(
+           fun({{ClientId, ChanPid}, _}) ->
+                   {ClientId, ChanPid}
+           end),
+    Table = ets:table(?CHAN_CONN_TAB, [{traverse, {select, Ms}}]),
+    qlc:q([{ClientId, ChanPid} || {ClientId, ChanPid} <- Table, is_channel_connected(ClientId, ChanPid)]).
+
+is_channel_connected(ClientId, ChanPid) when node(ChanPid) =:= node() ->
+    case get_chan_info(ClientId, ChanPid) of
+        #{conn_state := disconnected} -> false;
+        _ -> true
+    end;
+is_channel_connected(_ClientId, _ChanPid) -> false.
+
 %% @doc Lookup channels.
 -spec(lookup_channels(emqx_types:clientid()) -> list(chan_pid())).
 lookup_channels(ClientId) ->
@@ -523,4 +562,3 @@ get_chann_conn_mod(ClientId, ChanPid) when node(ChanPid) == node() ->
     end;
 get_chann_conn_mod(ClientId, ChanPid) ->
     rpc_call(node(ChanPid), get_chann_conn_mod, [ClientId, ChanPid], ?T_GET_INFO).
-
diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl
index 764778b74..4088988c2 100644
--- a/src/emqx_plugins.erl
+++ b/src/emqx_plugins.erl
@@ -225,6 +225,8 @@ ensure_file(File) ->
                              , {emqx_telemetry, true}
                              , {emqx_rule_engine, true}
                              , {emqx_bridge_mqtt, false}
+                             , {emqx_eviction_agent, true}
+                             , {emqx_node_rebalance, true}
                              ],
             write_loaded(DefaultPlugins);
         true ->
diff --git a/test/emqx_node_helpers.erl b/test/emqx_node_helpers.erl
new file mode 100644
index 000000000..16d4be000
--- /dev/null
+++ b/test/emqx_node_helpers.erl
@@ -0,0 +1,83 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_node_helpers).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-define(SLAVE_START_APPS, [emqx]).
+
+-export([start_slave/1,
+         start_slave/2,
+         stop_slave/1]).
+
+start_slave(Name) ->
+    start_slave(Name, #{}).
+
+start_slave(Name, Opts) ->
+    {ok, Node} = ct_slave:start(list_to_atom(atom_to_list(Name) ++ "@" ++ host()),
+                                [{kill_if_fail, true},
+                                 {monitor_master, true},
+                                 {init_timeout, 10000},
+                                 {startup_timeout, 10000},
+                                 {erl_flags, ebin_path()}]),
+
+    pong = net_adm:ping(Node),
+    setup_node(Node, Opts),
+    Node.
+
+stop_slave(Node) ->
+    rpc:call(Node, ekka, leave, []),
+    ct_slave:stop(Node).
+
+host() ->
+    [_, Host] = string:tokens(atom_to_list(node()), "@"), Host.
+
+ebin_path() ->
+    string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
+
+is_lib(Path) ->
+    string:prefix(Path, code:lib_dir()) =:= nomatch.
+
+setup_node(Node, #{} = Opts) ->
+    Listeners = maps:get(listeners, Opts, []),
+    StartApps = maps:get(start_apps, Opts, ?SLAVE_START_APPS),
+    DefaultEnvHandler =
+        fun(emqx) ->
+                application:set_env(
+                  emqx,
+                  listeners,
+                  Listeners),
+                application:set_env(gen_rpc, port_discovery, stateless),
+                ok;
+           (_) ->
+                ok
+        end,
+    EnvHandler = maps:get(env_handler, Opts, DefaultEnvHandler),
+
+    [ok = rpc:call(Node, application, load, [App]) || App <- [gen_rpc, emqx]],
+    ok = rpc:call(Node, emqx_ct_helpers, start_apps, [StartApps, EnvHandler]),
+
+    rpc:call(Node, ekka, join, [node()]),
+
+    %% Sanity check. Assert that `gen_rpc' is set up correctly:
+    ?assertEqual( Node
+                , gen_rpc:call(Node, erlang, node, [])
+                ),
+    ?assertEqual( node()
+                , gen_rpc:call(Node, gen_rpc, call, [node(), erlang, node, []])
+                ),
+    ok.
diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl
index 66a88a047..b95030889 100644
--- a/test/emqx_plugins_SUITE.erl
+++ b/test/emqx_plugins_SUITE.erl
@@ -99,8 +99,10 @@ t_ensure_default_loaded_plugins_file(Config) ->
     ?assertEqual(
        [ {emqx_bridge_mqtt, false}
        , {emqx_dashboard, true}
+       , {emqx_eviction_agent, true}
        , {emqx_management, true}
        , {emqx_modules, true}
+       , {emqx_node_rebalance, true}
        , {emqx_recon, true}
        , {emqx_retainer, true}
        , {emqx_rule_engine, true}
diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl
index 02c4c6598..0eb63dbf0 100644
--- a/test/emqx_shared_sub_SUITE.erl
+++ b/test/emqx_shared_sub_SUITE.erl
@@ -380,7 +380,7 @@ t_local(_) ->
 
     emqtt:stop(ConnPid1),
     emqtt:stop(ConnPid2),
-    stop_slave(Node),
+    emqx_node_helpers:stop_slave(Node),
 
     ?assertEqual(local, emqx_shared_sub:strategy(<<"local_group">>)),
     ?assertEqual(local, RemoteLocalGroupStrategy),
@@ -415,7 +415,7 @@ t_local_fallback(_) ->
     {true, UsedSubPid2} = last_message(<<"hello2">>, [ConnPid1]),
 
     emqtt:stop(ConnPid1),
-    stop_slave(Node),
+    emqx_node_helpers:stop_slave(Node),
 
     ?assertEqual(UsedSubPid1, UsedSubPid2),
     ok.
@@ -536,55 +536,8 @@ recv_msgs(Count, Msgs) ->
     end.
 
 start_slave(Name, Port) ->
-    {ok, Node} = ct_slave:start(list_to_atom(atom_to_list(Name) ++ "@" ++ host()),
-                                [{kill_if_fail, true},
-                                 {monitor_master, true},
-                                 {init_timeout, 10000},
-                                 {startup_timeout, 10000},
-                                 {erl_flags, ebin_path()}]),
-
-    pong = net_adm:ping(Node),
-    ok = setup_node(Node, Port),
-    Node.
-
-stop_slave(Node) ->
-    rpc:call(Node, ekka, leave, []),
-    ct_slave:stop(Node).
-
-host() ->
-    [_, Host] = string:tokens(atom_to_list(node()), "@"), Host.
-
-ebin_path() ->
-    string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " ").
-
-is_lib(Path) ->
-    string:prefix(Path, code:lib_dir()) =:= nomatch.
-
-setup_node(Node, Port) ->
-    EnvHandler =
-        fun(emqx) ->
-                application:set_env(
-                  emqx,
-                  listeners,
-                  [#{listen_on => {{127,0,0,1},Port},
-                     name => "internal",
-                     opts => [{zone,internal}],
-                     proto => tcp}]),
-                application:set_env(gen_rpc, port_discovery, stateless),
-                ok;
-           (_) ->
-                ok
-        end,
-
-    [ok = rpc:call(Node, application, load, [App]) || App <- [gen_rpc, emqx]],
-    ok = rpc:call(Node, emqx_ct_helpers, start_apps, [[emqx], EnvHandler]),
-    rpc:call(Node, ekka, join, [node()]),
-
-    %% Sanity check. Assert that `gen_rpc' is set up correctly:
-    ?assertEqual( Node
-                , gen_rpc:call(Node, erlang, node, [])
-                ),
-    ?assertEqual( node()
-                , gen_rpc:call(Node, gen_rpc, call, [node(), erlang, node, []])
-                ),
-    ok.
+    Listeners = [#{listen_on => {{127,0,0,1}, Port},
+                   name => "internal",
+                   opts => [{zone,internal}],
+                   proto => tcp}],
+    emqx_node_helpers:start_slave(Name, #{listeners => Listeners}).

From 9435b6aa8227f7744630c0bfdb704f05687be655 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sat, 13 Aug 2022 14:07:55 +0200
Subject: [PATCH 24/52] docs: Update CHANGES-4.3.md

---
 CHANGES-4.3.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index 10304fc70..ab74b877a 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -56,7 +56,6 @@ File format:
 - HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450]
 - Add check_conf cli to check config format. [#8486]
 - Optimize performance of shared subscription
-- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553)
 
 ## v4.3.16
 

From 698b1b6df67e0b44629aae5f93cedd8983714c99 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Thu, 11 Aug 2022 16:45:03 +0800
Subject: [PATCH 25/52] fix: different default plugins start between ee and ce

---
 src/emqx_plugins.erl | 50 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl
index 4088988c2..1c8849745 100644
--- a/src/emqx_plugins.erl
+++ b/src/emqx_plugins.erl
@@ -217,22 +217,50 @@ load_plugin_conf(AppName, PluginDir) ->
 ensure_file(File) ->
     case filelib:is_file(File) of
         false ->
-            DefaultPlugins = [ {emqx_management, true}
-                             , {emqx_dashboard, true}
-                             , {emqx_modules, true}
-                             , {emqx_recon, true}
-                             , {emqx_retainer, true}
-                             , {emqx_telemetry, true}
-                             , {emqx_rule_engine, true}
-                             , {emqx_bridge_mqtt, false}
-                             , {emqx_eviction_agent, true}
-                             , {emqx_node_rebalance, true}
-                             ],
+            DefaultPlugins = default_plugins(),
             write_loaded(DefaultPlugins);
         true ->
             ok
     end.
 
+-ifndef(EMQX_ENTERPRISE).
+%% default plugins see rebar.config.erl
+default_plugins() ->
+    [
+        {emqx_management, true},
+        {emqx_dashboard, true},
+        %% emqx_modules is not a plugin, but a normal application starting when boots.
+        {emqx_modules, false},
+        {emqx_retainer, true},
+        {emqx_recon, true},
+        {emqx_telemetry, true},
+        {emqx_rule_engine, true},
+        {emqx_bridge_mqtt, false},
+        {emqx_eviction_agent, true},
+        {emqx_node_rebalance, true}
+    ].
+
+-else.
+
+default_plugins() ->
+    [
+        {emqx_management, true},
+        {emqx_dashboard, true},
+        %% enterprise version of emqx_modules is a plugin
+        {emqx_modules, true},
+        %% retainer is managed by emqx_modules.
+        %% default is true in data/load_modules. **NOT HERE**
+        {emqx_retainer, false},
+        {emqx_recon, true},
+        {emqx_telemetry, true},
+        {emqx_rule_engine, true},
+        {emqx_bridge_mqtt, false},
+        {emqx_eviction_agent, true},
+        {emqx_node_rebalance, true}
+    ].
+
+-endif.
+
 with_loaded_file(File, SuccFun) ->
     case read_loaded(File) of
         {ok, Names0} ->

From e6a98524b5755050f72f42e70d696b50edbb8360 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Thu, 11 Aug 2022 23:20:41 +0800
Subject: [PATCH 26/52] chore: improve retainer ct init between ce and ee

---
 .../test/emqx_retainer_SUITE.erl              |  6 +--
 .../test/emqx_retainer_cli_SUITE.erl          |  5 +-
 .../test/emqx_retainer_ct_helper.erl          | 46 ++++++++++++++++++
 .../test/mqtt_protocol_v5_SUITE.erl           |  7 +--
 test/emqx_plugins_SUITE.erl                   | 47 ++++++++++++++-----
 5 files changed, 90 insertions(+), 21 deletions(-)
 create mode 100644 apps/emqx_retainer/test/emqx_retainer_ct_helper.erl

diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl
index 6f29eaedf..0e38548bf 100644
--- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl
+++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl
@@ -31,11 +31,12 @@ all() -> emqx_ct:all(?MODULE).
 %%--------------------------------------------------------------------
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_retainer]),
+    emqx_retainer_ct_helper:ensure_start(),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_retainer]).
+    emqx_retainer_ct_helper:ensure_stop(),
+    ok.
 
 init_per_testcase(TestCase, Config) ->
     emqx_retainer:clean(<<"#">>),
@@ -207,4 +208,3 @@ receive_messages(Count, Msgs) ->
     after 2000 ->
             Msgs
     end.
-
diff --git a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl
index 452a4f3d9..995e3508c 100644
--- a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl
+++ b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl
@@ -24,11 +24,12 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_retainer]),
+    emqx_retainer_ct_helper:ensure_start(),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_retainer]).
+    emqx_retainer_ct_helper:ensure_stop(),
+    ok.
 
 init_per_testcase(_TestCase, Config) ->
     Config.
diff --git a/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl b/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl
new file mode 100644
index 000000000..fd12a4e01
--- /dev/null
+++ b/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl
@@ -0,0 +1,46 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_retainer_ct_helper).
+
+
+%% API
+-export([ensure_start/0, ensure_stop/0]).
+-ifdef(EMQX_ENTERPRISE).
+ensure_start() ->
+    application:stop(emqx_modules),
+    init_conf(),
+    emqx_ct_helpers:start_apps([emqx_retainer]),
+    ok.
+
+-else.
+
+ensure_start() ->
+    init_conf(),
+    emqx_ct_helpers:start_apps([emqx_retainer]),
+    ok.
+
+-endif.
+
+ensure_stop() ->
+    emqx_ct_helpers:stop_apps([emqx_retainer]).
+
+
+init_conf() ->
+    application:set_env(emqx_retainer, expiry_interval, 0),
+    application:set_env(emqx_retainer, max_payload_size, 1024000),
+    application:set_env(emqx_retainer, max_retained_messages, 0),
+    application:set_env(emqx_retainer, storage_type, ram),
+    ok.
diff --git a/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl b/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl
index ef0b111e9..17a31e04b 100644
--- a/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl
+++ b/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl
@@ -27,12 +27,13 @@ init_per_suite(Config) ->
     %% Meck emqtt
     ok = meck:new(emqtt, [non_strict, passthrough, no_history, no_link]),
     %% Start Apps
-    emqx_ct_helpers:start_apps([emqx_retainer]),
+    emqx_retainer_ct_helper:ensure_start(),
     Config.
 
 end_per_suite(_Config) ->
     ok = meck:unload(emqtt),
-    emqx_ct_helpers:stop_apps([emqx_retainer]).
+    emqx_retainer_ct_helper:ensure_stop().
+
 
 %%--------------------------------------------------------------------
 %% Helpers
@@ -107,7 +108,7 @@ t_publish_message_expiry_interval(_) ->
     Msgs = receive_messages(4),
     ?assertEqual(2, length(Msgs)),  %% [MQTT-3.3.2-5]
 
-    L = lists:map(fun(Msg) -> MessageExpiryInterval = maps:get('Message-Expiry-Interval', maps:get(properties, Msg)), MessageExpiryInterval < 10 end, Msgs), 
+    L = lists:map(fun(Msg) -> MessageExpiryInterval = maps:get('Message-Expiry-Interval', maps:get(properties, Msg)), MessageExpiryInterval < 10 end, Msgs),
     ?assertEqual(2, length(L)),  %% [MQTT-3.3.2-6]
 
     ok = emqtt:disconnect(Client1),
diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl
index b95030889..aa548d766 100644
--- a/test/emqx_plugins_SUITE.erl
+++ b/test/emqx_plugins_SUITE.erl
@@ -89,6 +89,38 @@ t_load(_) ->
     ?assertEqual(ignore, emqx_plugins:load()),
     ?assertEqual(ignore, emqx_plugins:unload()).
 
+-ifndef(EMQX_ENTERPRISE).
+default_plugins() ->
+    [
+        {emqx_bridge_mqtt, false},
+        {emqx_dashboard, true},
+        {emqx_eviction_agent, true},
+        {emqx_management, true},
+        {emqx_modules, false},
+        {emqx_node_rebalance, true},
+        {emqx_recon, true},
+        {emqx_retainer, true},
+        {emqx_rule_engine, true},
+        {emqx_telemetry, true}
+    ].
+
+-else.
+
+default_plugins() ->
+    [
+        {emqx_bridge_mqtt, false},
+        {emqx_dashboard, true},
+        {emqx_eviction_agent, true},
+        {emqx_management, true},
+        {emqx_modules, true},
+        {emqx_node_rebalance, true},
+        {emqx_recon, true},
+        {emqx_retainer, false},
+        {emqx_rule_engine, true},
+        {emqx_telemetry, true}
+    ].
+
+-endif.
 t_ensure_default_loaded_plugins_file(Config) ->
     %% this will trigger it to write the default plugins to the
     %% inexistent file; but it won't truly load them in this test
@@ -96,19 +128,8 @@ t_ensure_default_loaded_plugins_file(Config) ->
     TmpFilepath = ?config(tmp_filepath, Config),
     ok = emqx_plugins:load(),
     {ok, Contents} = file:consult(TmpFilepath),
-    ?assertEqual(
-       [ {emqx_bridge_mqtt, false}
-       , {emqx_dashboard, true}
-       , {emqx_eviction_agent, true}
-       , {emqx_management, true}
-       , {emqx_modules, true}
-       , {emqx_node_rebalance, true}
-       , {emqx_recon, true}
-       , {emqx_retainer, true}
-       , {emqx_rule_engine, true}
-       , {emqx_telemetry, true}
-       ],
-       lists:sort(Contents)),
+    DefaultPlugins = default_plugins(),
+    ?assertEqual(DefaultPlugins, lists:sort(Contents)),
     ok.
 
 t_init_config(_) ->

From b02a3b94293b2c9ec063bc3cd915958bb2d25122 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Mon, 1 Aug 2022 17:14:42 +0800
Subject: [PATCH 27/52] fix(channel): Adjust the timing of the
 `client.connected` event

---
 src/emqx_channel.erl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl
index 852c171fe..f422bfcf9 100644
--- a/src/emqx_channel.erl
+++ b/src/emqx_channel.erl
@@ -293,7 +293,7 @@ handle_in(?CONNECT_PACKET(ConnPkt), Channel) ->
                                         },
             case enhanced_auth(?CONNECT_PACKET(NConnPkt), NChannel1) of
                 {ok, Properties, NChannel2} ->
-                    process_connect(Properties, ensure_connected(NChannel2));
+                    process_connect(Properties, NChannel2);
                 {continue, Properties, NChannel2} ->
                     handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, Properties}, NChannel2);
                 {error, ReasonCode, NChannel2} ->
@@ -309,7 +309,7 @@ handle_in(Packet = ?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION, _Properties),
         {ok, NProperties, NChannel} ->
             case ConnState of
                 connecting ->
-                    process_connect(NProperties, ensure_connected(NChannel));
+                    process_connect(NProperties, NChannel);
                 connected ->
                     handle_out(auth, {?RC_SUCCESS, NProperties}, NChannel);
                 _ ->
@@ -499,14 +499,14 @@ process_connect(AckProps, Channel = #channel{conninfo = ConnInfo,
     case emqx_cm:open_session(CleanStart, ClientInfo, ConnInfo) of
         {ok, #{session := Session, present := false}} ->
             NChannel = Channel#channel{session = Session},
-            handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, NChannel);
+            handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, ensure_connected(NChannel));
         {ok, #{session := Session, present := true, pendings := Pendings}} ->
             Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())),
             NChannel = Channel#channel{session  = Session,
                                        resuming = true,
                                        pendings = Pendings1
                                       },
-            handle_out(connack, {?RC_SUCCESS, sp(true), AckProps}, NChannel);
+            handle_out(connack, {?RC_SUCCESS, sp(true), AckProps}, ensure_connected(NChannel));
         {error, client_id_unavailable} ->
             handle_out(connack, ?RC_CLIENT_IDENTIFIER_NOT_VALID, Channel);
         {error, Reason} ->

From 93a97e0df2f826439955fa9679b44c1ca293b420 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 16 Aug 2022 00:56:50 +0200
Subject: [PATCH 28/52] chore: make a no-op change in emqx_retainer_sup

Just to keep the appup in sync with enterprise edition
---
 apps/emqx_retainer/src/emqx_retainer.app.src  |  2 +-
 .../emqx_retainer/src/emqx_retainer.appup.src | 19 +++++++++++++------
 apps/emqx_retainer/src/emqx_retainer_sup.erl  |  5 ++++-
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src
index 05920e985..80ec02c73 100644
--- a/apps/emqx_retainer/src/emqx_retainer.app.src
+++ b/apps/emqx_retainer/src/emqx_retainer.app.src
@@ -1,6 +1,6 @@
 {application, emqx_retainer,
  [{description, "EMQ X Retainer"},
-  {vsn, "4.3.3"}, % strict semver, bump manually!
+  {vsn, "4.3.4"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_retainer_sup]},
   {applications, [kernel,stdlib]},
diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src
index 45ec6420c..18f047119 100644
--- a/apps/emqx_retainer/src/emqx_retainer.appup.src
+++ b/apps/emqx_retainer/src/emqx_retainer.appup.src
@@ -1,14 +1,21 @@
 %% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.2",
-    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
+  [{"4.3.3",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[0-1]">>,
-    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.2",
-    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
+  [{"4.3.3",[{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[0-1]">>,
-    [{load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_retainer_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_retainer_cli,brutal_purge,soft_purge,[]},
      {load_module,emqx_retainer,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]}.
diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl
index 22af32504..d3a979788 100644
--- a/apps/emqx_retainer/src/emqx_retainer_sup.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl
@@ -32,5 +32,8 @@ init([Env]) ->
              restart  => permanent,
              shutdown => 5000,
              type     => worker,
-             modules  => [emqx_retainer]}]}}.
+             modules  => [emqx_retainer]} || not is_managed_by_modules()]}}.
 
+is_managed_by_modules() ->
+    %% always flase for opensource edition
+    false.

From 303249e2e8a8a5d70a7c0fb604690450a7c9dd54 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 16 Aug 2022 07:40:16 +0200
Subject: [PATCH 29/52] chore: move rebalance and eviction code to ee

---
 apps/emqx_eviction_agent/.gitignore           |  19 -
 apps/emqx_eviction_agent/README.md            |   9 -
 .../etc/emqx_eviction_agent.conf              |   3 -
 .../priv/emqx_eviction_agent.schema           |   0
 apps/emqx_eviction_agent/rebar.config         |   2 -
 .../src/emqx_eviction_agent.app.src           |  18 -
 .../src/emqx_eviction_agent.erl               | 291 ------------
 .../src/emqx_eviction_agent_api.erl           |  36 --
 .../src/emqx_eviction_agent_app.erl           |  36 --
 .../src/emqx_eviction_agent_channel.erl       | 299 -------------
 .../src/emqx_eviction_agent_cli.erl           |  42 --
 .../src/emqx_eviction_agent_conn_sup.erl      |  33 --
 .../src/emqx_eviction_agent_sup.erl           |  43 --
 .../test/emqx_eviction_agent_SUITE.erl        | 232 ----------
 .../test/emqx_eviction_agent_api_SUITE.erl    |  64 ---
 .../emqx_eviction_agent_channel_SUITE.erl     | 155 -------
 .../test/emqx_eviction_agent_cli_SUITE.erl    |  47 --
 .../test/emqx_eviction_agent_test_helpers.erl |  55 ---
 apps/emqx_node_rebalance/.gitignore           |  19 -
 apps/emqx_node_rebalance/README.md            |   9 -
 .../etc/emqx_node_rebalance.conf              |   3 -
 .../include/emqx_node_rebalance.hrl           |  31 --
 .../priv/emqx_node_rebalance.schema           |   0
 apps/emqx_node_rebalance/rebar.config         |   2 -
 .../src/emqx_node_rebalance.app.src           |  19 -
 .../src/emqx_node_rebalance.erl               | 414 ------------------
 .../src/emqx_node_rebalance_agent.erl         | 127 ------
 .../src/emqx_node_rebalance_api.erl           | 243 ----------
 .../src/emqx_node_rebalance_app.erl           |  34 --
 .../src/emqx_node_rebalance_cli.erl           | 265 -----------
 .../src/emqx_node_rebalance_evacuation.erl    | 298 -------------
 ...emqx_node_rebalance_evacuation_persist.erl | 109 -----
 .../src/emqx_node_rebalance_status.erl        | 225 ----------
 .../src/emqx_node_rebalance_sup.erl           |  44 --
 .../test/emqx_node_rebalance_SUITE.erl        | 183 --------
 .../test/emqx_node_rebalance_agent_SUITE.erl  | 163 -------
 .../test/emqx_node_rebalance_api_SUITE.erl    | 321 --------------
 .../test/emqx_node_rebalance_cli_SUITE.erl    | 199 ---------
 .../emqx_node_rebalance_evacuation_SUITE.erl  | 194 --------
 ...ode_rebalance_evacuation_persist_SUITE.erl | 107 -----
 data/loaded_plugins.tmpl                      |   2 -
 rebar.config.erl                              |   2 -
 src/emqx_plugins.erl                          |   4 +-
 test/emqx_plugins_SUITE.erl                   |   2 -
 44 files changed, 1 insertion(+), 4402 deletions(-)
 delete mode 100644 apps/emqx_eviction_agent/.gitignore
 delete mode 100644 apps/emqx_eviction_agent/README.md
 delete mode 100644 apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf
 delete mode 100644 apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema
 delete mode 100644 apps/emqx_eviction_agent/rebar.config
 delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src
 delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent.erl
 delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl
 delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl
 delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl
 delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl
 delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl
 delete mode 100644 apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl
 delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
 delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl
 delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl
 delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl
 delete mode 100644 apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl
 delete mode 100644 apps/emqx_node_rebalance/.gitignore
 delete mode 100644 apps/emqx_node_rebalance/README.md
 delete mode 100644 apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf
 delete mode 100644 apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl
 delete mode 100644 apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema
 delete mode 100644 apps/emqx_node_rebalance/rebar.config
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance.erl
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl
 delete mode 100644 apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl
 delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl
 delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl
 delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl
 delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl
 delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl
 delete mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl

diff --git a/apps/emqx_eviction_agent/.gitignore b/apps/emqx_eviction_agent/.gitignore
deleted file mode 100644
index f1c455451..000000000
--- a/apps/emqx_eviction_agent/.gitignore
+++ /dev/null
@@ -1,19 +0,0 @@
-.rebar3
-_*
-.eunit
-*.o
-*.beam
-*.plt
-*.swp
-*.swo
-.erlang.cookie
-ebin
-log
-erl_crash.dump
-.rebar
-logs
-_build
-.idea
-*.iml
-rebar3.crashdump
-*~
diff --git a/apps/emqx_eviction_agent/README.md b/apps/emqx_eviction_agent/README.md
deleted file mode 100644
index f9b8037bf..000000000
--- a/apps/emqx_eviction_agent/README.md
+++ /dev/null
@@ -1,9 +0,0 @@
-emqx_eviction_agent
-=====
-
-An OTP library
-
-Build
------
-
-    $ rebar3 compile
diff --git a/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf b/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf
deleted file mode 100644
index 011b7fb0f..000000000
--- a/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf
+++ /dev/null
@@ -1,3 +0,0 @@
-##--------------------------------------------------------------------
-## EMQX Eviction Agent Plugin
-##--------------------------------------------------------------------
diff --git a/apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema b/apps/emqx_eviction_agent/priv/emqx_eviction_agent.schema
deleted file mode 100644
index e69de29bb..000000000
diff --git a/apps/emqx_eviction_agent/rebar.config b/apps/emqx_eviction_agent/rebar.config
deleted file mode 100644
index 2656fd554..000000000
--- a/apps/emqx_eviction_agent/rebar.config
+++ /dev/null
@@ -1,2 +0,0 @@
-{erl_opts, [debug_info]}.
-{deps, []}.
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src b/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src
deleted file mode 100644
index ffa28af61..000000000
--- a/apps/emqx_eviction_agent/src/emqx_eviction_agent.app.src
+++ /dev/null
@@ -1,18 +0,0 @@
-{application, emqx_eviction_agent,
- [{description, "EMQX Eviction Agent"},
-  {vsn, "4.3.0"},
-  {registered, [emqx_eviction_agent_sup,
-                emqx_eviction_agent,
-                emqx_eviction_agent_conn_sup]},
-  {applications,
-   [kernel,
-    stdlib
-   ]},
-  {mod, {emqx_eviction_agent_app,[]}},
-  {env,[]},
-  {modules, []},
-  {maintainers, ["EMQX Team <contact@emqx.io>"]},
-  {links, [{"Homepage", "https://emqx.io/"},
-           {"Github", "https://github.com/emqx"}
-          ]}
- ]}.
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl
deleted file mode 100644
index a8cf442cf..000000000
--- a/apps/emqx_eviction_agent/src/emqx_eviction_agent.erl
+++ /dev/null
@@ -1,291 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent).
-
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx/include/types.hrl").
-
--include_lib("stdlib/include/qlc.hrl").
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
--export([start_link/0,
-         enable/2,
-         disable/1,
-         status/0,
-         connection_count/0,
-         session_count/0,
-         session_count/1,
-         evict_connections/1,
-         evict_sessions/2,
-         evict_sessions/3,
-         evict_session_channel/3
-        ]).
-
--behaviour(gen_server).
-
--export([init/1,
-         handle_call/3,
-         handle_info/2,
-         handle_cast/2,
-         code_change/3
-        ]).
-
--export([on_connect/2,
-         on_connack/3]).
-
--export([hook/0,
-         unhook/0]).
-
--export_type([server_reference/0]).
-
-%%--------------------------------------------------------------------
-%% APIs
-%%--------------------------------------------------------------------
-
--type server_reference() :: binary() | undefined.
--type status() :: {enabled, conn_stats()} | disabled.
--type conn_stats() :: #{connections := non_neg_integer(),
-                        sessions := non_neg_integer()}.
--type kind() :: atom().
-
--spec start_link() -> startlink_ret().
-start_link() ->
-    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-
--spec enable(kind(), server_reference()) -> ok_or_error(eviction_agent_busy).
-enable(Kind, ServerReference) ->
-    gen_server:call(?MODULE, {enable, Kind, ServerReference}).
-
--spec disable(kind()) -> ok.
-disable(Kind) ->
-    gen_server:call(?MODULE, {disable, Kind}).
-
--spec status() -> status().
-status() ->
-    case enable_status() of
-        {enabled, _Kind, _ServerReference} ->
-            {enabled, stats()};
-        disabled ->
-           disabled
-    end.
-
--spec evict_connections(pos_integer()) -> ok_or_error(disabled).
-evict_connections(N) ->
-    case enable_status() of
-        {enabled, _Kind, ServerReference} ->
-            ok = do_evict_connections(N, ServerReference);
-        disabled ->
-            {error, disabled}
-    end.
-
--spec evict_sessions(pos_integer(), node() | [node()]) -> ok_or_error(disabled).
-evict_sessions(N, Node) when is_atom(Node) ->
-    evict_sessions(N, [Node]);
-evict_sessions(N, Nodes) when is_list(Nodes) andalso length(Nodes) > 0 ->
-    evict_sessions(N, Nodes, any).
-
--spec evict_sessions(pos_integer(), node() | [node()], atom()) -> ok_or_error(disabled).
-evict_sessions(N, Node, ConnState) when is_atom(Node) ->
-    evict_sessions(N, [Node], ConnState);
-evict_sessions(N, Nodes, ConnState)
-  when is_list(Nodes) andalso length(Nodes) > 0 ->
-    case enable_status() of
-        {enabled, _Kind, _ServerReference} ->
-            ok = do_evict_sessions(N, Nodes, ConnState);
-        disabled ->
-            {error, disabled}
-    end.
-
-%%--------------------------------------------------------------------
-%% gen_server callbacks
-%%--------------------------------------------------------------------
-
-init([]) ->
-    _ = persistent_term:erase(?MODULE),
-    {ok, #{}}.
-
-%% enable
-handle_call({enable, Kind, ServerReference}, _From, St) ->
-    Reply = case enable_status() of
-                disabled ->
-                    ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference});
-                {enabled, Kind, _ServerReference} ->
-                    ok = persistent_term:put(?MODULE, {enabled, Kind, ServerReference});
-                {enabled, _OtherKind, _ServerReference} ->
-                    {error, eviction_agent_busy}
-            end,
-    {reply, Reply, St};
-
-%% disable
-handle_call({disable, Kind}, _From, St) ->
-    Reply = case enable_status() of
-                disabled ->
-                    {error, disabled};
-                {enabled, Kind, _ServerReference} ->
-                    _ = persistent_term:erase(?MODULE),
-                    ok;
-                {enabled, _OtherKind, _ServerReference} ->
-                    {error, eviction_agent_busy}
-            end,
-    {reply, Reply, St}.
-
-handle_info(Msg, St) ->
-    ?LOG(warning, "Unknown Msg: ~p, State: ~p", [Msg, St]),
-    {noreply, St}.
-
-handle_cast(Msg, St) ->
-    ?LOG(warning, "Unknown cast Msg: ~p, State: ~p", [Msg, St]),
-    {noreply, St}.
-
-code_change(_Vsn, State, _Extra) ->
-    {ok, State}.
-
-%%--------------------------------------------------------------------
-%% Hook callbacks
-%%--------------------------------------------------------------------
-
-on_connect(_ConnInfo, _Props) ->
-    case enable_status() of
-        {enabled, _Kind, _ServerReference} ->
-            {stop, {error, ?RC_USE_ANOTHER_SERVER}};
-        disabled ->
-            ignore
-    end.
-
-on_connack(#{proto_name := <<"MQTT">>, proto_ver := ?MQTT_PROTO_V5},
-           use_another_server,
-           Props) ->
-    case enable_status() of
-        {enabled, _Kind, ServerReference} ->
-            {ok, Props#{'Server-Reference' => ServerReference}};
-        disabled ->
-            {ok, Props}
-    end;
-on_connack(_ClientInfo, _Reason, Props) ->
-    {ok, Props}.
-
-%%--------------------------------------------------------------------
-%% Hook funcs
-%%--------------------------------------------------------------------
-
-hook() ->
-    ?tp(debug, eviction_agent_hook, #{}),
-    ok = emqx_hooks:put('client.connack', {?MODULE, on_connack, []}),
-    ok = emqx_hooks:put('client.connect', {?MODULE, on_connect, []}).
-
-unhook() ->
-    ?tp(debug, eviction_agent_unhook, #{}),
-    ok = emqx_hooks:del('client.connect', {?MODULE, on_connect}),
-    ok = emqx_hooks:del('client.connack', {?MODULE, on_connack}).
-
-enable_status() ->
-    persistent_term:get(?MODULE, disabled).
-
-% connection management
-stats() ->
-    #{
-        connections => connection_count(),
-        sessions => session_count()
-     }.
-
-connection_table() ->
-    emqx_cm:live_connection_table().
-
-connection_count() ->
-    table_count(connection_table()).
-
-channel_with_session_table(any) ->
-    qlc:q([{ClientId, ConnInfo, ClientInfo}
-           || {ClientId, _, ConnInfo, ClientInfo} <- emqx_cm:channel_with_session_table()]);
-channel_with_session_table(RequiredConnState) ->
-    qlc:q([{ClientId, ConnInfo, ClientInfo}
-           || {ClientId, ConnState, ConnInfo, ClientInfo} <- emqx_cm:channel_with_session_table(),
-              RequiredConnState =:= ConnState]).
-
-session_count() ->
-    session_count(any).
-
-session_count(ConnState) ->
-    table_count(channel_with_session_table(ConnState)).
-
-table_count(QH) ->
-    qlc:fold(fun(_, Acc) -> Acc + 1 end, 0, QH).
-
-take_connections(N) ->
-    ChanQH = qlc:q([ChanPid || {_ClientId, ChanPid} <- connection_table()]),
-    ChanPidCursor = qlc:cursor(ChanQH),
-    ChanPids = qlc:next_answers(ChanPidCursor, N),
-    ok = qlc:delete_cursor(ChanPidCursor),
-    ChanPids.
-
-take_channel_with_sessions(N, ConnState) ->
-    ChanPidCursor = qlc:cursor(channel_with_session_table(ConnState)),
-    Channels = qlc:next_answers(ChanPidCursor, N),
-    ok = qlc:delete_cursor(ChanPidCursor),
-    Channels.
-
-do_evict_connections(N, ServerReference) when N > 0 ->
-    ChanPids = take_connections(N),
-    ok = lists:foreach(
-           fun(ChanPid) ->
-                   disconnect_channel(ChanPid, ServerReference)
-           end,
-           ChanPids).
-
-do_evict_sessions(N, Nodes, ConnState) when N > 0 ->
-    Channels = take_channel_with_sessions(N, ConnState),
-    ok = lists:foreach(
-           fun({ClientId, ConnInfo, ClientInfo}) ->
-                   evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo)
-           end,
-           Channels).
-
-evict_session_channel(Nodes, ClientId, ConnInfo, ClientInfo) ->
-    Node = select_random(Nodes),
-    ?LOG(info, "Evicting client=~p to node=~p, conninfo=~p, clientinfo=~p",
-         [ClientId, Node, ConnInfo, ClientInfo]),
-    case rpc:call(Node, ?MODULE, evict_session_channel, [ClientId, ConnInfo, ClientInfo]) of
-        {badrpc, Reason} ->
-            ?LOG(error, "RPC error while evicting client=~p to node=~p: ~p",
-                 [ClientId, Node, Reason]),
-            {error, Reason};
-        {error, Reason} = Error ->
-            ?LOG(error, "Error evicting client=~p to node=~p: ~p",
-                 [ClientId, Node, Reason]),
-            Error;
-        Res -> Res
-    end.
-
-evict_session_channel(ClientId, ConnInfo, ClientInfo) ->
-    ?LOG(info, "Taking up client=~p, conninfo=~p, clientinfo=~p",
-         [ClientId, ConnInfo, ClientInfo]),
-    Result = emqx_eviction_agent_channel:start_supervised(
-               #{conninfo => ConnInfo,
-                 clientinfo => ClientInfo}),
-    ?LOG(info, "Taking up client=~p, result=~p",
-         [ClientId, Result]),
-    Result.
-
-disconnect_channel(ChanPid, ServerReference) ->
-    ChanPid ! {disconnect,
-               ?RC_USE_ANOTHER_SERVER,
-               use_another_server,
-               #{'Server-Reference' => ServerReference}}.
-
-select_random(List) when length(List) > 0 ->
-    lists:nth(rand:uniform(length(List)) , List).
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl
deleted file mode 100644
index 426f5978d..000000000
--- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_api.erl
+++ /dev/null
@@ -1,36 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_api).
-
--include_lib("emqx/include/logger.hrl").
-
--rest_api(#{name   => node_eviction_status,
-            method => 'GET',
-            path   => "/node_eviction/status",
-            func   => status,
-            descr  => "Get node eviction status"}).
-
--export([status/2]).
-
-status(_Bindings, _Params) ->
-    case emqx_eviction_agent:status() of
-        disabled ->
-            {ok, #{status => disabled}};
-        {enabled, Stats} ->
-            {ok, #{status => enabled,
-                   stats => Stats}}
-    end.
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl
deleted file mode 100644
index c13fcfb0d..000000000
--- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_app.erl
+++ /dev/null
@@ -1,36 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_app).
-
--behaviour(application).
-
--emqx_plugin(?MODULE).
-
--export([ start/2
-        , stop/1
-        ]).
-
-start(_Type, _Args) ->
-    Env = application:get_all_env(emqx_eviction_agent),
-    ok = emqx_eviction_agent:hook(),
-    {ok, Sup} = emqx_eviction_agent_sup:start_link(Env),
-    ok = emqx_eviction_agent_cli:load(),
-    {ok, Sup}.
-
-stop(_State) ->
-    ok = emqx_eviction_agent:unhook(),
-    ok = emqx_eviction_agent_cli:unload().
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl
deleted file mode 100644
index 1f6fad00f..000000000
--- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl
+++ /dev/null
@@ -1,299 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
-%% MQTT Channel
--module(emqx_eviction_agent_channel).
-
--include_lib("emqx/include/emqx.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx/include/types.hrl").
-
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
--logger_header("[Evicted Channel]").
-
--export([start_link/1,
-         start_supervised/1,
-         call/2,
-         call/3,
-         cast/2,
-         stop/1
-        ]).
-
--export([init/1,
-         handle_call/3,
-         handle_cast/2,
-         handle_info/2,
-         terminate/2,
-         code_change/3
-        ]).
-
--type opts() :: #{conninfo := emqx_types:conninfo(),
-                  clientinfo := emqx_types:clientinfo()}.
-
-%%--------------------------------------------------------------------
-%% API
-%%--------------------------------------------------------------------
-
--spec start_supervised(opts()) -> startlink_ret().
-start_supervised(#{clientinfo := #{clientid := ClientId}} = Opts) ->
-    RandomId = integer_to_binary(erlang:unique_integer([positive])),
-    Id = <<ClientId/binary, "-", RandomId/binary>>,
-    ChildSpec = #{id => Id,
-                  start => {?MODULE, start_link, [Opts]},
-                  restart => temporary,
-                  shutdown => 5000,
-                  type => worker,
-                  modules => [?MODULE]
-                 },
-    supervisor:start_child(
-        emqx_eviction_agent_conn_sup,
-        ChildSpec).
-
--spec start_link(opts()) -> startlink_ret().
-start_link(Opts) ->
-    gen_server:start_link(?MODULE, [Opts], []).
-
--spec cast(pid(), term()) -> ok.
-cast(Pid, Req) ->
-    gen_server:cast(Pid, Req).
-
--spec call(pid(), term()) -> term().
-call(Pid, Req) ->
-    call(Pid, Req, infinity).
-
--spec call(pid(), term(), timeout()) -> term().
-call(Pid, Req, Timeout) ->
-    gen_server:call(Pid, Req, Timeout).
-
--spec stop(pid()) -> ok.
-stop(Pid) ->
-    gen_server:stop(Pid).
-
-%%--------------------------------------------------------------------
-%% gen_server API
-%%--------------------------------------------------------------------
-
-init([#{conninfo := OldConnInfo, clientinfo := #{clientid := ClientId} = OldClientInfo}]) ->
-    process_flag(trap_exit, true),
-    ClientInfo = clientinfo(OldClientInfo),
-    ConnInfo = conninfo(OldConnInfo),
-    case open_session(ConnInfo, ClientInfo) of
-        {ok, Channel0} ->
-            case set_expiry_timer(Channel0) of
-                {ok, Channel1} ->
-                    ?LOG(
-                       info,
-                       "Channel initialized for client=~p on node=~p",
-                       [ClientId, node()]),
-                    {ok, Channel1, hibernate};
-                {error, Reason} ->
-                    {stop, Reason}
-            end;
-        {error, Reason} ->
-            {stop, Reason}
-    end.
-
-handle_call(kick, _From, Channel) ->
-    {stop, kicked, ok, Channel};
-
-handle_call(discard, _From, Channel) ->
-    {stop, discarded, ok, Channel};
-
-handle_call({takeover, 'begin'}, _From, #{session := Session} = Channel) ->
-    {reply, Session, Channel#{takeover => true}};
-
-handle_call({takeover, 'end'}, _From, #{session := Session,
-                                        clientinfo := #{clientid := ClientId},
-                                        pendings := Pendings} = Channel) ->
-    ok = emqx_session:takeover(Session),
-    %% TODO: Should not drain deliver here (side effect)
-    Delivers = emqx_misc:drain_deliver(),
-    AllPendings = lists:append(Delivers, Pendings),
-    ?tp(debug,
-        emqx_channel_takeover_end,
-        #{clientid => ClientId}),
-    {stop, normal, AllPendings, Channel};
-
-handle_call(list_acl_cache, _From, Channel) ->
-    {reply, [], Channel};
-
-handle_call({quota, _Policy}, _From, Channel) ->
-    {reply, ok, Channel};
-
-handle_call(Req, _From, Channel) ->
-    ?LOG(error, "Unexpected call: ~p", [Req]),
-    {reply, ignored, Channel}.
-
-handle_info(Deliver = {deliver, _Topic, _Msg}, Channel) ->
-    Delivers = [Deliver | emqx_misc:drain_deliver()],
-    {noreply, handle_deliver(Delivers, Channel)};
-
-handle_info(expire_session, Channel) ->
-    {stop, expired, Channel};
-
-handle_info(Info, Channel) ->
-    ?LOG(error, "Unexpected info: ~p", [Info]),
-    {noreply, Channel}.
-
-handle_cast(Msg, Channel) ->
-    ?LOG(error, "Unexpected cast: ~p", [Msg]),
-    {noreply, Channel}.
-
-terminate(Reason, #{clientinfo := ClientInfo, session := Session} = Channel) ->
-    ok = cancel_expiry_timer(Channel),
-    emqx_session:terminate(ClientInfo, Reason, Session).
-
-code_change(_OldVsn, Channel, _Extra) ->
-    {ok, Channel}.
-
-%%--------------------------------------------------------------------
-%% Internal functions
-%%--------------------------------------------------------------------
-
-handle_deliver(Delivers,
-               #{takeover := true,
-                 pendings := Pendings,
-                 session := Session,
-                 clientinfo := #{clientid := ClientId} = ClientInfo} = Channel) ->
-    %% NOTE: Order is important here. While the takeover is in
-    %% progress, the session cannot enqueue messages, since it already
-    %% passed on the queue to the new connection in the session state.
-    NPendings = lists:append(
-                  Pendings,
-                  ignore_local(ClientInfo, maybe_nack(Delivers), ClientId, Session)),
-    Channel#{pendings => NPendings};
-
-handle_deliver(Delivers,
-               #{takeover := false,
-                 session := Session,
-                 clientinfo := #{clientid := ClientId} = ClientInfo} = Channel) ->
-    NSession = emqx_session:enqueue(
-                 ClientInfo,
-                 ignore_local(ClientInfo, maybe_nack(Delivers), ClientId, Session),
-                 Session),
-    Channel#{session => NSession}.
-
-cancel_expiry_timer(#{expiry_timer := TRef}) when is_reference(TRef) ->
-    _ = erlang:cancel_timer(TRef),
-    ok;
-cancel_expiry_timer(_) ->
-    ok.
-
-set_expiry_timer(#{conninfo := ConnInfo} = Channel) ->
-    case maps:get(expiry_interval, ConnInfo) of
-        ?UINT_MAX -> {ok, Channel};
-        I when I > 0 ->
-            Timer = erlang:send_after(timer:seconds(I), self(), expire_session),
-            {ok, Channel#{expiry_timer => Timer}};
-        _ ->
-            {error, should_be_expired}
-    end.
-
-open_session(ConnInfo, #{clientid := ClientId} = ClientInfo) ->
-    Channel = channel(ConnInfo, ClientInfo),
-    case emqx_cm:open_session(false, ClientInfo, ConnInfo) of
-        {ok, #{present := false}} ->
-            ?LOG(info, "No session for clientid=~p", [ClientId]),
-            {error, no_session};
-        {ok, #{session := Session, present := true, pendings := Pendings0}} ->
-            ?LOG(info, "Session opened for client=~p on node=~p", [ClientId, node()]),
-            Pendings1 = lists:usort(lists:append(Pendings0, emqx_misc:drain_deliver())),
-            NSession = emqx_session:enqueue(
-                         ClientInfo,
-                         ignore_local(
-                           ClientInfo,
-                           maybe_nack(Pendings1),
-                           ClientId,
-                           Session),
-                         Session),
-            NChannel = Channel#{session => NSession},
-            ok = emqx_cm:insert_channel_info(ClientId, info(NChannel), []),
-            ?LOG(info, "Channel info updated for client=~p on node=~p", [ClientId, node()]),
-            {ok, NChannel};
-        {error, Reason} = Error ->
-            ?LOG(error, "Failed to open session due to ~p", [Reason]),
-            Error
-    end.
-
-conninfo(OldConnInfo) ->
-    DisconnectedAt = maps:get(disconnected_at, OldConnInfo, erlang:system_time(millisecond)),
-    ConnInfo0 = maps:with(
-                  [socktype,
-                   sockname,
-                   peername,
-                   peercert,
-                   clientid,
-                   clean_start,
-                   receive_maximum,
-                   expiry_interval],
-                  OldConnInfo),
-    ConnInfo0#{
-      conn_mod => ?MODULE,
-      connected => false,
-      disconnected_at => DisconnectedAt
-     }.
-
-clientinfo(OldClientInfo) ->
-    maps:with(
-      [zone,
-       protocol,
-       peerhost,
-       sockport,
-       clientid,
-       username,
-       is_bridge,
-       is_superuser,
-       mountpoint],
-      OldClientInfo).
-
-channel(ConnInfo, ClientInfo) ->
-    #{conninfo => ConnInfo,
-      clientinfo => ClientInfo,
-      expiry_timer => undefined,
-      takeover => false,
-      resuming => false,
-      pendings => []
-     }.
-
-info(Channel) ->
-    #{conninfo => maps:get(conninfo, Channel, undefined),
-      clientinfo => maps:get(clientinfo, Channel, undefined),
-      session => maps:get(session, Channel, undefined),
-      conn_state => disconnected
-     }.
-
-ignore_local(ClientInfo, Delivers, Subscriber, Session) ->
-    Subs = emqx_session:info(subscriptions, Session),
-    lists:dropwhile(fun({deliver, Topic, #message{from = Publisher} = Msg}) ->
-                        case maps:find(Topic, Subs) of
-                            {ok, #{nl := 1}} when Subscriber =:= Publisher ->
-                                ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, no_local]),
-                                ok = emqx_metrics:inc('delivery.dropped'),
-                                ok = emqx_metrics:inc('delivery.dropped.no_local'),
-                                true;
-                            _ ->
-                                false
-                        end
-                    end, Delivers).
-
-maybe_nack(Delivers) ->
-    lists:filter(fun not_nacked/1, Delivers).
-
-not_nacked({deliver, _Topic, Msg}) ->
-    not (emqx_shared_sub:is_ack_required(Msg)
-         andalso (ok == emqx_shared_sub:nack_no_connection(Msg))).
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl
deleted file mode 100644
index 632f8e480..000000000
--- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_cli.erl
+++ /dev/null
@@ -1,42 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_cli).
-
-%% APIs
--export([ load/0
-        , unload/0
-        , cli/1
-        ]).
-
-load() ->
-    emqx_ctl:register_command(eviction, {?MODULE, cli}, []).
-
-unload() ->
-    emqx_ctl:unregister_command(eviction).
-
-cli(["status"]) ->
-    case emqx_eviction_agent:status() of
-        disabled ->
-            emqx_ctl:print("Eviction status: disabled~n");
-        {enabled, _Stats} ->
-            emqx_ctl:print("Eviction status: enabled~n")
-    end;
-
-cli(_) ->
-    emqx_ctl:usage(
-      [{"eviction status",
-        "Get current node eviction status"}]).
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl
deleted file mode 100644
index 2b11ce9d1..000000000
--- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_conn_sup.erl
+++ /dev/null
@@ -1,33 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_conn_sup).
-
--behaviour(supervisor).
-
--export([start_link/1]).
-
--export([init/1]).
-
-start_link(Env) ->
-	supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]).
-
-init([_Env]) ->
-    Childs = [],
-    {ok, {
-       {one_for_one, 10, 3600},
-       Childs}
-    }.
diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl
deleted file mode 100644
index 9e28c2251..000000000
--- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_sup.erl
+++ /dev/null
@@ -1,43 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_sup).
-
--behaviour(supervisor).
-
--export([start_link/1]).
-
--export([init/1]).
-
-start_link(Env) ->
-	supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]).
-
-init([_Env]) ->
-    Childs = [child_spec(worker, emqx_eviction_agent, []),
-              child_spec(supervisor, emqx_eviction_agent_conn_sup, [#{}])],
-    {ok, {
-       {one_for_one, 10, 3600},
-       Childs}
-    }.
-
-child_spec(Type, Mod, Args) ->
-    #{id => Mod,
-      start => {Mod, start_link, Args},
-      restart => permanent,
-      shutdown => 5000,
-      type => Type,
-      modules => [Mod]
-     }.
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
deleted file mode 100644
index b77c0dee8..000000000
--- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
+++ /dev/null
@@ -1,232 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
--import(emqx_eviction_agent_test_helpers,
-        [emqtt_connect/0, emqtt_connect/2]).
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent]),
-    Config.
-
-end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_eviction_agent]).
-
-init_per_testcase(t_explicit_session_takeover, Config) ->
-    _ = emqx_eviction_agent:disable(test_eviction),
-    Node = emqx_node_helpers:start_slave(
-             evacuate1,
-             #{start_apps => [emqx, emqx_eviction_agent]}),
-    [{evacuate_node, Node} | Config];
-init_per_testcase(_TestCase, Config) ->
-    _ = emqx_eviction_agent:disable(test_eviction),
-    Config.
-
-end_per_testcase(t_explicit_session_takeover, Config) ->
-    _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)),
-    _ = emqx_eviction_agent:disable(test_eviction);
-end_per_testcase(_TestCase, _Config) ->
-    _ = emqx_eviction_agent:disable(test_eviction).
-
-t_enable_disable(_Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    ?assertMatch(
-       disabled,
-       emqx_eviction_agent:status()),
-
-    {ok, C0} = emqtt_connect(),
-    ok = emqtt:disconnect(C0),
-
-    ok = emqx_eviction_agent:enable(test_eviction, undefined),
-
-    ?assertMatch(
-       {error, eviction_agent_busy},
-       emqx_eviction_agent:enable(bar, undefined)),
-
-    ?assertMatch(
-       ok,
-       emqx_eviction_agent:enable(test_eviction, <<"srv">>)),
-
-    ?assertMatch(
-        {enabled, #{}},
-        emqx_eviction_agent:status()),
-
-    ?assertMatch(
-       {error, {use_another_server, #{}}},
-       emqtt_connect()),
-
-    ?assertMatch(
-        {error, eviction_agent_busy},
-        emqx_eviction_agent:disable(bar)),
-
-    ?assertMatch(
-        ok,
-        emqx_eviction_agent:disable(test_eviction)),
-
-    ?assertMatch(
-        {error, disabled},
-        emqx_eviction_agent:disable(test_eviction)),
-
-    ?assertMatch(
-       disabled,
-       emqx_eviction_agent:status()),
-
-    {ok, C1} = emqtt_connect(),
-    ok = emqtt:disconnect(C1).
-
-
-t_evict_connections_status(_Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    {ok, _C} = emqtt_connect(),
-
-    {error, disabled} = emqx_eviction_agent:evict_connections(1),
-
-    ok = emqx_eviction_agent:enable(test_eviction, undefined),
-
-    ?assertMatch(
-       {enabled, #{connections := 1, sessions := _}},
-       emqx_eviction_agent:status()),
-
-    ok = emqx_eviction_agent:evict_connections(1),
-
-    ct:sleep(100),
-
-    ?assertMatch(
-       {enabled, #{connections := 0, sessions := _}},
-       emqx_eviction_agent:status()),
-
-    ok = emqx_eviction_agent:disable(test_eviction).
-
-
-t_explicit_session_takeover(Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    {ok, C0} = emqtt_connect(<<"client_with_session">>, false),
-    {ok, _, _} = emqtt:subscribe(C0, <<"t1">>),
-
-    ok = emqx_eviction_agent:enable(test_eviction, undefined),
-
-    ?assertEqual(
-       1,
-       emqx_eviction_agent:connection_count()),
-
-    ok = emqx_eviction_agent:evict_connections(1),
-
-    receive
-        {'EXIT', C0, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok
-    after 1000 ->
-              ?assert(false, "Connection not evicted")
-    end,
-
-    ?assertEqual(
-       0,
-       emqx_eviction_agent:connection_count()),
-
-    ?assertEqual(
-       1,
-       emqx_eviction_agent:session_count()),
-
-    %% First, evacuate to the same node
-
-    ?check_trace(
-       ?wait_async_action(
-          emqx_eviction_agent:evict_sessions(1, node()),
-          #{?snk_kind := emqx_channel_takeover_end},
-          1000),
-       fun(_Result, Trace) ->
-        ?assertMatch(
-           [#{clientid := <<"client_with_session">>} | _ ],
-           ?of_kind(emqx_channel_takeover_end, Trace))
-       end),
-
-    ok = emqx_eviction_agent:disable(test_eviction),
-    ok = connect_and_publish(<<"t1">>, <<"MessageToEvictedSession1">>),
-    ok = emqx_eviction_agent:enable(test_eviction, undefined),
-
-    %% Evacuate to another node
-
-    TargetNodeForEvacuation = ?config(evacuate_node, Config),
-    ?check_trace(
-       ?wait_async_action(
-          emqx_eviction_agent:evict_sessions(1, TargetNodeForEvacuation),
-          #{?snk_kind := emqx_channel_takeover_end},
-          1000),
-       fun(_Result, Trace) ->
-        ?assertMatch(
-           [#{clientid := <<"client_with_session">>} | _ ],
-           ?of_kind(emqx_channel_takeover_end, Trace))
-       end),
-
-    ?assertEqual(
-       0,
-       emqx_eviction_agent:session_count()),
-
-    ?assertEqual(
-       1,
-       rpc:call(TargetNodeForEvacuation, emqx_eviction_agent, session_count, [])),
-
-    ok = emqx_eviction_agent:disable(test_eviction),
-
-    ct:pal("evicted chann info: ~p", [emqx_cm:get_chan_info(<<"client_with_session">>)]),
-
-    ok = connect_and_publish(<<"t1">>, <<"MessageToEvictedSession2">>),
-    ct:sleep(100),
-
-    {ok, C2} = emqtt_connect(<<"client_with_session">>, false),
-
-    ok = assert_receive_publish(
-           [#{payload => <<"MessageToEvictedSession1">>, topic => <<"t1">>},
-            #{payload => <<"MessageToEvictedSession2">>, topic => <<"t1">>}]),
-    ok = emqtt:disconnect(C2).
-
-t_disable_on_restart(_Config) ->
-    ok = emqx_eviction_agent:enable(test_eviction, undefined),
-
-    ok = supervisor:terminate_child(emqx_eviction_agent_sup, emqx_eviction_agent),
-    {ok, _} = supervisor:restart_child(emqx_eviction_agent_sup, emqx_eviction_agent),
-
-    ?assertEqual(
-        disabled,
-        emqx_eviction_agent:status()).
-
-assert_receive_publish([]) -> ok;
-assert_receive_publish([#{payload := Msg, topic := Topic} | Rest]) ->
-    receive
-        {publish, #{payload := Msg,
-                    topic := Topic}} ->
-            assert_receive_publish(Rest)
-    after 1000 ->
-              ?assert(false, "Message `" ++ binary_to_list(Msg) ++ "` is lost")
-    end.
-
-connect_and_publish(Topic, Message) ->
-    {ok, C} = emqtt_connect(),
-    emqtt:publish(C, Topic, Message),
-    ok = emqtt:disconnect(C).
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl
deleted file mode 100644
index f9585c0e6..000000000
--- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_api_SUITE.erl
+++ /dev/null
@@ -1,64 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_api_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
-
--import(emqx_mgmt_api_test_helpers,
-        [request_api/3,
-         auth_header_/0,
-         api_path/1]).
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_management]),
-    Config.
-
-end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_management, emqx_eviction_agent]),
-    Config.
-
-t_status(_Config) ->
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"disabled">>}},
-       api_get(["node_eviction", "status"])),
-
-    ok = emqx_eviction_agent:enable(apitest, undefined),
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"enabled">>,
-              <<"stats">> := #{}}},
-        api_get(["node_eviction", "status"])),
-
-    ok = emqx_eviction_agent:disable(apitest),
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"disabled">>}},
-       api_get(["node_eviction", "status"])).
-
-api_get(Path) ->
-    case request_api(get, api_path(Path), auth_header_()) of
-        {ok, ResponseBody} ->
-            {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])};
-        {error, _} = Error -> Error
-    end.
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl
deleted file mode 100644
index cbbf1be5a..000000000
--- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl
+++ /dev/null
@@ -1,155 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_channel_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
--define(CLIENT_ID, <<"client_with_session">>).
-
--import(emqx_eviction_agent_test_helpers,
-        [emqtt_connect/2]).
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent]),
-    Config.
-
-end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_eviction_agent]).
-
-t_start_no_session(_Config) ->
-    Opts = #{clientinfo => #{clientid => ?CLIENT_ID,
-                             zone => internal},
-             conninfo => #{clientid => ?CLIENT_ID,
-                           receive_maximum => 32}},
-    ?assertMatch(
-       {error, {no_session, _}},
-       emqx_eviction_agent_channel:start_supervised(Opts)).
-
-t_start_no_expire(_Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    _ = emqtt_connect(?CLIENT_ID, false),
-
-    Opts = #{clientinfo => #{clientid => ?CLIENT_ID,
-                             zone => internal},
-             conninfo => #{clientid => ?CLIENT_ID,
-                           receive_maximum => 32,
-                           expiry_interval => 0}},
-    ?assertMatch(
-       {error, {should_be_expired, _}},
-       emqx_eviction_agent_channel:start_supervised(Opts)).
-
-t_start_infinite_expire(_Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    _ = emqtt_connect(?CLIENT_ID, false),
-
-    Opts = #{clientinfo => #{clientid => ?CLIENT_ID,
-                             zone => internal},
-             conninfo => #{clientid => ?CLIENT_ID,
-                           receive_maximum => 32,
-                           expiry_interval => ?UINT_MAX}},
-    ?assertMatch(
-       {ok, _},
-       emqx_eviction_agent_channel:start_supervised(Opts)).
-
-t_kick(_Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    _ = emqtt_connect(?CLIENT_ID, false),
-    Opts = evict_session_opts(?CLIENT_ID),
-
-    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts),
-
-    ?assertEqual(
-       ok,
-       emqx_eviction_agent_channel:call(Pid, kick)).
-
-t_discard(_Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    _ = emqtt_connect(?CLIENT_ID, false),
-    Opts = evict_session_opts(?CLIENT_ID),
-
-    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts),
-
-    ?assertEqual(
-       ok,
-       emqx_eviction_agent_channel:call(Pid, discard)).
-
-t_stop(_Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    _ = emqtt_connect(?CLIENT_ID, false),
-    Opts = evict_session_opts(?CLIENT_ID),
-
-    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts),
-
-    ?assertEqual(
-       ok,
-       emqx_eviction_agent_channel:stop(Pid)).
-
-
-t_ignored_calls(_Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    _ = emqtt_connect(?CLIENT_ID, false),
-    Opts = evict_session_opts(?CLIENT_ID),
-
-    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts),
-
-    ok = emqx_eviction_agent_channel:cast(Pid, unknown),
-    Pid ! unknown,
-
-    ?assertEqual(
-       [],
-       emqx_eviction_agent_channel:call(Pid, list_acl_cache)),
-
-    ?assertEqual(
-       ok,
-       emqx_eviction_agent_channel:call(Pid, {quota, quota})),
-
-    ?assertEqual(
-       ignored,
-       emqx_eviction_agent_channel:call(Pid, unknown)).
-
-t_expire(_Config) ->
-    erlang:process_flag(trap_exit, true),
-
-    _ = emqtt_connect(?CLIENT_ID, false),
-    #{conninfo := ConnInfo} = Opts0 = evict_session_opts(?CLIENT_ID),
-    Opts1 = Opts0#{conninfo => ConnInfo#{expiry_interval => 1}},
-
-    {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts1),
-
-    ct:sleep(1500),
-
-    ?assertNot(is_process_alive(Pid)).
-
-evict_session_opts(ClientId) ->
-    maps:with(
-      [conninfo, clientinfo],
-      emqx_cm:get_chan_info(ClientId)).
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl
deleted file mode 100644
index 9c05cdb21..000000000
--- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_cli_SUITE.erl
+++ /dev/null
@@ -1,47 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_cli_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent]),
-    Config.
-
-end_per_suite(Config) ->
-    _ = emqx_eviction_agent:disable(foo),
-    emqx_ct_helpers:stop_apps([emqx_eviction_agent]),
-    Config.
-
-t_status(_Config) ->
-    %% usage
-    ok = emqx_eviction_agent_cli:cli(["foobar"]),
-
-    %% status
-    ok = emqx_eviction_agent_cli:cli(["status"]),
-
-    ok = emqx_eviction_agent:enable(foo, undefined),
-
-    %% status
-    ok = emqx_eviction_agent_cli:cli(["status"]).
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl
deleted file mode 100644
index 68b86f059..000000000
--- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl
+++ /dev/null
@@ -1,55 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_eviction_agent_test_helpers).
-
--export([emqtt_connect/0,
-         emqtt_connect/2,
-         emqtt_connect_many/1,
-         emqtt_try_connect/0]).
-
-emqtt_connect() ->
-    emqtt_connect(<<"client1">>, true).
-
-emqtt_connect(ClientId, CleanStart) ->
-    {ok, C} = emqtt:start_link(
-                [{clientid, ClientId},
-                 {clean_start, CleanStart},
-                 {proto_ver, v5},
-                 {properties, #{'Session-Expiry-Interval' => 600}}
-                ]),
-    case emqtt:connect(C) of
-        {ok, _} -> {ok, C};
-        {error, _} = Error -> Error
-    end.
-
-emqtt_connect_many(Count) ->
-    lists:map(
-      fun(N) ->
-              NBin = integer_to_binary(N),
-              ClientId = <<"client-", NBin/binary>>,
-              {ok, C} = emqtt_connect(ClientId, false),
-              C
-      end,
-      lists:seq(1, Count)).
-
-emqtt_try_connect() ->
-    case emqtt_connect() of
-        {ok, C} ->
-            emqtt:disconnect(C),
-            ok;
-        {error, _} = Error -> Error
-    end.
diff --git a/apps/emqx_node_rebalance/.gitignore b/apps/emqx_node_rebalance/.gitignore
deleted file mode 100644
index f1c455451..000000000
--- a/apps/emqx_node_rebalance/.gitignore
+++ /dev/null
@@ -1,19 +0,0 @@
-.rebar3
-_*
-.eunit
-*.o
-*.beam
-*.plt
-*.swp
-*.swo
-.erlang.cookie
-ebin
-log
-erl_crash.dump
-.rebar
-logs
-_build
-.idea
-*.iml
-rebar3.crashdump
-*~
diff --git a/apps/emqx_node_rebalance/README.md b/apps/emqx_node_rebalance/README.md
deleted file mode 100644
index 2e56f62cd..000000000
--- a/apps/emqx_node_rebalance/README.md
+++ /dev/null
@@ -1,9 +0,0 @@
-emqx_node_rebalance
-=====
-
-An OTP library
-
-Build
------
-
-    $ rebar3 compile
diff --git a/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf b/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf
deleted file mode 100644
index 8ace22435..000000000
--- a/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf
+++ /dev/null
@@ -1,3 +0,0 @@
-##--------------------------------------------------------------------
-## EMQX Node Rebalance Plugin
-##--------------------------------------------------------------------
diff --git a/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl b/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl
deleted file mode 100644
index 1903b87cc..000000000
--- a/apps/emqx_node_rebalance/include/emqx_node_rebalance.hrl
+++ /dev/null
@@ -1,31 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--define(DEFAULT_CONN_EVICT_RATE, 500).
--define(DEFAULT_SESS_EVICT_RATE, 500).
-
--define(DEFAULT_WAIT_HEALTH_CHECK, 60). %% sec
--define(DEFAULT_WAIT_TAKEOVER, 60). %% sec
-
--define(DEFAULT_ABS_CONN_THRESHOLD, 1000).
--define(DEFAULT_ABS_SESS_THRESHOLD, 1000).
-
--define(DEFAULT_REL_CONN_THRESHOLD, 1.1).
--define(DEFAULT_REL_SESS_THRESHOLD, 1.1).
-
--define(EVICT_INTERVAL, 1000).
-
--define(EVACUATION_FILENAME, <<".evacuation">>).
diff --git a/apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema b/apps/emqx_node_rebalance/priv/emqx_node_rebalance.schema
deleted file mode 100644
index e69de29bb..000000000
diff --git a/apps/emqx_node_rebalance/rebar.config b/apps/emqx_node_rebalance/rebar.config
deleted file mode 100644
index 2656fd554..000000000
--- a/apps/emqx_node_rebalance/rebar.config
+++ /dev/null
@@ -1,2 +0,0 @@
-{erl_opts, [debug_info]}.
-{deps, []}.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src
deleted file mode 100644
index 761cd1d6f..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src
+++ /dev/null
@@ -1,19 +0,0 @@
-{application, emqx_node_rebalance,
- [{description, "EMQX Node Rebalance"},
-  {vsn, "4.3.0"},
-  {registered, [emqx_node_rebalance_sup,
-                emqx_node_rebalance,
-                emqx_node_rebalance_agent,
-                emqx_node_rebalance_evacuation]},
-  {applications,
-   [kernel,
-    stdlib
-   ]},
-  {mod, {emqx_node_rebalance_app,[]}},
-  {env,[]},
-  {modules, []},
-  {maintainers, ["EMQX Team <contact@emqx.io>"]},
-  {links, [{"Homepage", "https://emqx.io/"},
-           {"Github", "https://github.com/emqx"}
-          ]}
- ]}.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl
deleted file mode 100644
index 1edbbbe3b..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl
+++ /dev/null
@@ -1,414 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance).
-
--include("emqx_node_rebalance.hrl").
-
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx/include/types.hrl").
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
--export([start/1,
-         status/0,
-         status/1,
-         stop/0
-        ]).
-
--export([start_link/0]).
-
--behavior(gen_statem).
-
--export([init/1,
-         callback_mode/0,
-         handle_event/4,
-         code_change/4
-        ]).
-
--export([is_node_available/0,
-         available_nodes/1,
-         connection_count/0,
-         session_count/0,
-         disconnected_session_count/0]).
-
-%%--------------------------------------------------------------------
-%% APIs
-%%--------------------------------------------------------------------
-
--type start_opts() :: #{conn_evict_rate => pos_integer(),
-                        sess_evict_rate => pos_integer(),
-                        wait_health_check => pos_integer(),
-                        wait_takeover => pos_integer(),
-                        abs_conn_threshold => pos_integer(),
-                        rel_conn_threshold => number(),
-                        abs_sess_threshold => pos_integer(),
-                        rel_sess_threshold => number(),
-                        nodes => [node()]
-                       }.
--type start_error() :: already_started | [{node(), term()}].
-
-
--spec start(start_opts()) -> ok_or_error(start_error()).
-start(StartOpts) ->
-    Opts = maps:merge(default_opts(), StartOpts),
-    gen_statem:call(?MODULE, {start, Opts}).
-
--spec stop() -> ok_or_error(not_started).
-stop() ->
-    gen_statem:call(?MODULE, stop).
-
--spec status() -> disabled | {enabled, map()}.
-status() ->
-    gen_statem:call(?MODULE, status).
-
--spec status(pid()) -> disabled | {enabled, map()}.
-status(Pid) ->
-    gen_statem:call(Pid, status).
-
--spec start_link() -> startlink_ret().
-start_link() ->
-    gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []).
-
--spec available_nodes(list(node())) -> list(node()).
-available_nodes(Nodes) when is_list(Nodes) ->
-    {Available, _} = rpc:multicall(Nodes, ?MODULE, is_node_available, []),
-    lists:filter(fun is_atom/1, Available).
-
-%%--------------------------------------------------------------------
-%% gen_statem callbacks
-%%--------------------------------------------------------------------
-
-callback_mode() -> handle_event_function.
-
-%% states: disabled, wait_health_check, evicting_conns, wait_takeover, evicting_sessions
-
-init([]) ->
-    ?tp(debug, emqx_node_rebalance_started, #{}),
-    {ok, disabled, #{}}.
-
-%% start
-handle_event({call, From},
-             {start, #{wait_health_check := WaitHealthCheck} = Opts},
-             disabled,
-             #{} = Data) ->
-    case enable_rebalance(Data#{opts => Opts}) of
-        {ok, NewData} ->
-            ?LOG(warning, "Node rebalance enabled: ~p", [Opts]),
-            {next_state,
-             wait_health_check,
-             NewData,
-             [{state_timeout, seconds(WaitHealthCheck), evict_conns},
-              {reply, From, ok}]};
-        {error, Reason} ->
-            ?LOG(warning, "Node rebalance enabling failed: ~p", [Reason]),
-            {keep_state_and_data,
-             [{reply, From, {error, Reason}}]}
-    end;
-handle_event({call, From}, {start, _Opts}, _State, #{}) ->
-    {keep_state_and_data,
-     [{reply, From, {error, already_started}}]};
-
-%% stop
-handle_event({call, From}, stop, disabled, #{}) ->
-    {keep_state_and_data,
-     [{reply, From, {error, not_started}}]};
-handle_event({call, From}, stop, _State, Data) ->
-    ok = disable_rebalance(Data),
-    ?LOG(warning, "Node rebalance stopped"),
-    {next_state,
-     disabled,
-     deinit(Data),
-     [{reply, From, ok}]};
-
-%% status
-handle_event({call, From}, status, disabled, #{}) ->
-    {keep_state_and_data,
-     [{reply, From, disabled}]};
-handle_event({call, From}, status, State, Data) ->
-    Stats = get_stats(State, Data),
-    {keep_state_and_data,
-     [{reply, From, {enabled, Stats#{state => State,
-                                     coordinator_node => node()}}}]};
-
-%% conn eviction
-handle_event(state_timeout,
-             evict_conns,
-             wait_health_check,
-             Data) ->
-    ?LOG(warning, "Node rebalance wait_health_check over"),
-    {next_state,
-     evicting_conns,
-     Data,
-     [{state_timeout, 0, evict_conns}]};
-
-handle_event(state_timeout,
-             evict_conns,
-             evicting_conns,
-             #{opts := #{wait_takeover := WaitTakeover,
-                         evict_interval := EvictInterval}} = Data) ->
-    case evict_conns(Data) of
-        ok ->
-            ?LOG(warning, "Node rebalance evict_conns over"),
-            {next_state,
-             wait_takeover,
-             Data,
-             [{state_timeout, seconds(WaitTakeover), evict_sessions}]};
-        {continue, NewData} ->
-            {keep_state,
-             NewData,
-             [{state_timeout, EvictInterval, evict_conns}]}
-    end;
-
-handle_event(state_timeout,
-             evict_sessions,
-             wait_takeover,
-             Data) ->
-    ?LOG(warning, "Node rebalance wait_takeover over"),
-    {next_state,
-     evicting_sessions,
-     Data,
-     [{state_timeout, 0, evict_sessions}]};
-
-handle_event(state_timeout,
-             evict_sessions,
-             evicting_sessions,
-             #{opts := #{evict_interval := EvictInterval}} = Data) ->
-    case evict_sessions(Data) of
-        ok ->
-            ?tp(debug, emqx_node_rebalance_evict_sess_over, #{}),
-            ?LOG(warning, "Node rebalance evict_sess over"),
-            ok = disable_rebalance(Data),
-            ?LOG(warning, "Rebalance finished successfully"),
-            {next_state,
-             disabled,
-             deinit(Data)};
-        {continue, NewData} ->
-            {keep_state,
-             NewData,
-             [{state_timeout, EvictInterval, evict_sessions}]}
-    end;
-
-handle_event({call, From}, Msg, State, Data) ->
-    ?LOG(warning, "Unknown call: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
-    {keep_state_and_data,
-     [{reply, From, ignored}]};
-
-handle_event(info, Msg, State, Data) ->
-    ?LOG(warning, "Unknown Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
-    keep_state_and_data;
-
-handle_event(cast, Msg, State, Data) ->
-    ?LOG(warning, "Unknown cast Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
-    keep_state_and_data.
-
-code_change(_Vsn, State, Data, _Extra) ->
-    {ok, State, Data}.
-
-%%--------------------------------------------------------------------
-%% internal funs
-%%--------------------------------------------------------------------
-
-enable_rebalance(#{opts := Opts} = Data) ->
-    Nodes = maps:get(nodes, Opts),
-    ConnCounts = multicall(Nodes, {?MODULE, connection_count, []}),
-    SessCounts = multicall(Nodes, {?MODULE, session_count, []}),
-    {_, Counts} = lists:unzip(ConnCounts),
-    Avg = avg(Counts),
-    {DonorCounts,
-     RecipientCounts} = lists:partition(
-                          fun({_Node, Count}) ->
-                                  Count >= Avg
-                          end,
-                          ConnCounts),
-    ?LOG(warning, "Enabling rebalance: ConnCounts=~p, DonorCounts=~p, RecipientCounts=~p",
-         [ConnCounts, DonorCounts, RecipientCounts]),
-    {DonorNodes, _} = lists:unzip(DonorCounts),
-    {RecipientNodes, _} = lists:unzip(RecipientCounts),
-    case need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) of
-        false -> {error, nothing_to_balance};
-        true ->
-            _ = multicall(DonorNodes, {emqx_node_rebalance_agent, enable, [self()]}),
-            {ok, Data#{donors => DonorNodes,
-                       recipients => RecipientNodes,
-                       initial_conn_counts => maps:from_list(ConnCounts),
-                       initial_sess_counts => maps:from_list(SessCounts)}}
-    end.
-
-disable_rebalance(#{donors := DonorNodes}) ->
-    _ = multicall(DonorNodes, {emqx_node_rebalance_agent, disable, [self()]}),
-    ok.
-
-evict_conns(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) ->
-    DonorNodeCounts = multicall(DonorNodes, {?MODULE, connection_count, []}),
-    {_, DonorCounts} = lists:unzip(DonorNodeCounts),
-    RecipientNodeCounts = multicall(RecipientNodes, {?MODULE, connection_count, []}),
-    {_, RecipientCounts} = lists:unzip(RecipientNodeCounts),
-
-    DonorAvg = avg(DonorCounts),
-    RecipientAvg = avg(RecipientCounts),
-    Thresholds = thresholds(conn, Opts),
-    NewData = Data#{donor_conn_avg => DonorAvg,
-                    recipient_conn_avg => RecipientAvg,
-                    donor_conn_counts => maps:from_list(DonorNodeCounts),
-                    recipient_conn_counts => maps:from_list(RecipientNodeCounts)},
-    case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of
-        true -> ok;
-        false ->
-            ConnEvictRate = maps:get(conn_evict_rate, Opts),
-            NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts),
-            ?LOG(warning, "Node rebalance, evict_conns, nodes=~p, counts=~p",
-                 [NodesToEvict, ConnEvictRate]),
-            _ = multicall(NodesToEvict, {emqx_eviction_agent, evict_connections, [ConnEvictRate]}),
-            {continue, NewData}
-    end.
-
-evict_sessions(#{donors := DonorNodes, recipients := RecipientNodes, opts := Opts} = Data) ->
-    DonorNodeCounts = multicall(DonorNodes, {?MODULE, disconnected_session_count, []}),
-    {_, DonorCounts} = lists:unzip(DonorNodeCounts),
-    RecipientNodeCounts = multicall(RecipientNodes, {?MODULE, disconnected_session_count, []}),
-    {_, RecipientCounts} = lists:unzip(RecipientNodeCounts),
-
-    DonorAvg = avg(DonorCounts),
-    RecipientAvg = avg(RecipientCounts),
-    Thresholds = thresholds(sess, Opts),
-    NewData = Data#{donor_sess_avg => DonorAvg,
-                    recipient_sess_avg => RecipientAvg,
-                    donor_sess_counts => maps:from_list(DonorNodeCounts),
-                    recipient_sess_counts => maps:from_list(RecipientNodeCounts)},
-    case within_thresholds(DonorAvg, RecipientAvg, Thresholds) of
-        true -> ok;
-        false ->
-            SessEvictRate = maps:get(sess_evict_rate, Opts),
-            NodesToEvict = nodes_to_evict(RecipientAvg, DonorNodeCounts),
-            ?LOG(warning, "Node rebalance, evict_sessions, nodes=~p, counts=~p",
-                 [NodesToEvict, SessEvictRate]),
-            _ = multicall(NodesToEvict,
-                          {emqx_eviction_agent,
-                           evict_sessions,
-                           [SessEvictRate, RecipientNodes, disconnected]}),
-            {continue, NewData}
-    end.
-
-need_rebalance([] = _DonorNodes, _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> false;
-need_rebalance(_DonorNodes, [] = _RecipientNodes, _ConnCounts, _SessCounts, _Opts) -> false;
-need_rebalance(DonorNodes, RecipientNodes, ConnCounts, SessCounts, Opts) ->
-    DonorConnAvg = avg_for_nodes(DonorNodes, ConnCounts),
-    RecipientConnAvg = avg_for_nodes(RecipientNodes, ConnCounts),
-    DonorSessAvg = avg_for_nodes(DonorNodes, SessCounts),
-    RecipientSessAvg = avg_for_nodes(RecipientNodes, SessCounts),
-    Result = (not within_thresholds(DonorConnAvg, RecipientConnAvg, thresholds(conn, Opts)))
-        orelse (not within_thresholds(DonorSessAvg, RecipientSessAvg, thresholds(sess, Opts))),
-    ?tp(debug, emqx_node_rebalance_need_rebalance,
-        #{donors => DonorNodes,
-          recipients => RecipientNodes,
-          conn_counts => ConnCounts,
-          sess_counts => SessCounts,
-          opts => Opts,
-          result => Result
-         }),
-    Result.
-
-avg_for_nodes(Nodes, Counts) ->
-    avg(maps:values(maps:with(Nodes, maps:from_list(Counts)))).
-
-within_thresholds(Value, GoalValue, {AbsThres, RelThres}) ->
-    (Value =< GoalValue + AbsThres) orelse (Value =< GoalValue * RelThres).
-
-thresholds(conn, #{abs_conn_threshold := Abs, rel_conn_threshold := Rel}) ->
-    {Abs, Rel};
-thresholds(sess, #{abs_sess_threshold := Abs, rel_sess_threshold := Rel}) ->
-    {Abs, Rel}.
-
-nodes_to_evict(Goal, NodeCounts) ->
-    {Nodes, _} = lists:unzip(
-                   lists:filter(
-                     fun({_Node, Count}) ->
-                             Count > Goal
-                     end,
-                     NodeCounts)),
-    Nodes.
-
-get_stats(disabled, _Data) -> #{};
-get_stats(_State, Data) -> Data.
-
-avg(List) when length(List) >= 1 ->
-    lists:sum(List) / length(List).
-
-multicall(Nodes, {M, F, A}) ->
-    case rpc:multicall(Nodes, M, F, A) of
-        {Results, []} ->
-            case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of
-                {OkResults, []} ->
-                    [{Node, ok_result(Result)} || {Node, Result} <- OkResults];
-                {_, BadResults} ->
-                    error({bad_nodes, BadResults})
-            end;
-        {_, [_BadNode | _] = BadNodes} ->
-            error({bad_nodes, BadNodes})
-    end.
-
-is_ok({_Node, {ok, _}}) -> true;
-is_ok({_Node, ok}) -> true;
-is_ok(_) -> false.
-
-ok_result({ok, Result}) -> Result;
-ok_result(ok) -> ok.
-
-connection_count() ->
-    {ok, emqx_eviction_agent:connection_count()}.
-
-session_count() ->
-    {ok, emqx_eviction_agent:session_count()}.
-
-disconnected_session_count() ->
-    {ok, emqx_eviction_agent:session_count(disconnected)}.
-
-default_opts() ->
-    #{
-      conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE,
-      abs_conn_threshold => ?DEFAULT_ABS_CONN_THRESHOLD,
-      rel_conn_threshold => ?DEFAULT_REL_CONN_THRESHOLD,
-
-      sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE,
-      abs_sess_threshold => ?DEFAULT_ABS_SESS_THRESHOLD,
-      rel_sess_threshold => ?DEFAULT_REL_SESS_THRESHOLD,
-
-      wait_health_check => ?DEFAULT_WAIT_HEALTH_CHECK,
-      wait_takeover => ?DEFAULT_WAIT_TAKEOVER,
-
-      evict_interval => ?EVICT_INTERVAL,
-
-      nodes => all_nodes()
-     }.
-
-
-deinit(Data) ->
-    Keys = [recipient_conn_avg, recipient_sess_avg, donor_conn_avg, donor_sess_avg,
-            recipient_conn_counts, recipient_sess_counts, donor_conn_counts, donor_sess_counts,
-            initial_conn_counts, initial_sess_counts,
-            opts],
-    maps:without(Keys, Data).
-
-is_node_available() ->
-    true = is_pid(whereis(emqx_node_rebalance_agent)),
-    disabled = emqx_eviction_agent:status(),
-    node().
-
-all_nodes() ->
-    ekka_mnesia:cluster_nodes(all).
-
-seconds(Sec) ->
-    round(timer:seconds(Sec)).
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl
deleted file mode 100644
index ade043d6b..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_agent.erl
+++ /dev/null
@@ -1,127 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_agent).
-
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx/include/types.hrl").
-
--include_lib("stdlib/include/qlc.hrl").
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
--export([start_link/0,
-         enable/1,
-         disable/1,
-         status/0
-        ]).
-
--export([init/1,
-         handle_call/3,
-         handle_info/2,
-         handle_cast/2,
-         code_change/3
-        ]).
-
--define(ENABLE_KIND, emqx_node_rebalance).
-
-%%--------------------------------------------------------------------
-%% APIs
-%%--------------------------------------------------------------------
-
--type status() :: {enabled, pid()} | disabled.
-
--spec start_link() -> startlink_ret().
-start_link() ->
-    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-
--spec enable(pid()) -> ok_or_error(already_enabled | eviction_agent_busy).
-enable(CoordinatorPid) ->
-    gen_server:call(?MODULE, {enable, CoordinatorPid}).
-
--spec disable(pid()) -> ok_or_error(already_disabled | invalid_coordinator).
-disable(CoordinatorPid) ->
-    gen_server:call(?MODULE, {disable, CoordinatorPid}).
-
--spec status() -> status().
-status() ->
-    gen_server:call(?MODULE, status).
-
-%%--------------------------------------------------------------------
-%% gen_server callbacks
-%%--------------------------------------------------------------------
-
-init([]) ->
-    {ok, #{}}.
-
-handle_call({enable, CoordinatorPid}, _From, St) ->
-    case St of
-        #{coordinator_pid := _Pid} ->
-            {reply, {error, already_enabled}, St};
-        _ ->
-            true = link(CoordinatorPid),
-            EvictionAgentPid = whereis(emqx_eviction_agent),
-            true = link(EvictionAgentPid),
-            case emqx_eviction_agent:enable(?ENABLE_KIND, undefined) of
-                ok ->
-                    {reply, ok, #{coordinator_pid => CoordinatorPid,
-                                  eviction_agent_pid => EvictionAgentPid}};
-                {error, eviction_agent_busy} ->
-                    true = unlink(EvictionAgentPid),
-                    true = unlink(CoordinatorPid),
-                    {reply, {error, eviction_agent_busy}, St}
-            end
-    end;
-
-handle_call({disable, CoordinatorPid}, _From, St) ->
-    case St of
-        #{coordinator_pid := CoordinatorPid,
-          eviction_agent_pid := EvictionAgentPid} ->
-            _ = emqx_eviction_agent:disable(?ENABLE_KIND),
-            true = unlink(EvictionAgentPid),
-            true = unlink(CoordinatorPid),
-            NewSt = maps:without(
-                      [coordinator_pid, eviction_agent_pid],
-                      St),
-            {reply, ok, NewSt};
-        #{coordinator_pid := _CoordinatorPid} ->
-            {reply, {error, invalid_coordinator}, St};
-        #{} ->
-            {reply, {error, already_disabled}, St}
-    end;
-
-handle_call(status, _From, St) ->
-    case St of
-        #{coordinator_pid := Pid} ->
-            {reply, {enabled, Pid}, St};
-        _ ->
-            {reply, disabled, St}
-    end;
-
-handle_call(Msg, _From, St) ->
-    ?LOG(warning, "Unknown call: ~p, State: ~p", [Msg, St]),
-    {reply, ignored, St}.
-
-handle_info(Msg, St) ->
-    ?LOG(warning, "Unknown Msg: ~p, State: ~p", [Msg, St]),
-    {noreply, St}.
-
-handle_cast(Msg, St) ->
-    ?LOG(warning, "Unknown cast Msg: ~p, State: ~p", [Msg, St]),
-    {noreply, St}.
-
-code_change(_Vsn, State, _Extra) ->
-    {ok, State}.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
deleted file mode 100644
index 6ba221cd8..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
+++ /dev/null
@@ -1,243 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_api).
-
--import(minirest,  [return/1]).
-
--rest_api(#{name   => load_rebalance_status,
-            method => 'GET',
-            path   => "/load_rebalance/status",
-            func   => status,
-            descr  => "Get load rebalance status"}).
-
--rest_api(#{name   => load_rebalance_global_status,
-            method => 'GET',
-            path   => "/load_rebalance/global_status",
-            func   => global_status,
-            descr  => "Get status of all rebalance/evacuation processes across the cluster"}).
-
--rest_api(#{name   => load_rebalance_availability_check,
-            method => 'GET',
-            path   => "/load_rebalance/availability_check",
-            func   => availability_check,
-            descr  => "Node rebalance availability check"}).
-
--rest_api(#{name   => load_rebalance_start,
-            method => 'POST',
-            path   => "/load_rebalance/:bin:node/start",
-            func   => rebalance_start,
-            descr  => "Start rebalancing with the node as coordinator"}).
-
--rest_api(#{name   => load_rebalance_stop,
-            method => 'POST',
-            path   => "/load_rebalance/:bin:node/stop",
-            func   => rebalance_stop,
-            descr  => "Stop rebalancing coordinated by the node"}).
-
--rest_api(#{name   => load_rebalance_evacuation_start,
-            method => 'POST',
-            path   => "/load_rebalance/:bin:node/evacuation/start",
-            func   => rebalance_evacuation_start,
-            descr  => "Start evacuation on a node "}).
-
--rest_api(#{name   => load_rebalance_evacuation_stop,
-            method => 'POST',
-            path   => "/load_rebalance/:bin:node/evacuation/stop",
-            func   => rebalance_evacuation_stop,
-            descr  => "Stop evacuation on the node"}).
-
--export([status/2,
-         availability_check/2,
-         global_status/2,
-
-         rebalance_evacuation_start/2,
-         rebalance_evacuation_stop/2,
-         rebalance_start/2,
-         rebalance_stop/2
-        ]).
-
-status(_Bindings, _Params) ->
-    case emqx_node_rebalance_status:local_status() of
-        disabled ->
-            {ok, #{status => disabled}};
-        {rebalance, Stats} ->
-            {ok, format_status(rebalance, Stats)};
-        {evacuation, Stats} ->
-            {ok, format_status(evacuation, Stats)}
-    end.
-
-global_status(_Bindings, _Params) ->
-    #{evacuations := Evacuations,
-      rebalances := Rebalances} = emqx_node_rebalance_status:global_status(),
-      {ok, #{evacuations => maps:from_list(Evacuations),
-             rebalances => maps:from_list(Rebalances)}}.
-
-availability_check(_Bindings, _Params) ->
-    case emqx_eviction_agent:status() of
-        disabled ->
-            {200, #{}};
-        {enabled, _Stats} ->
-            {503, #{}}
-    end.
-
-rebalance_evacuation_start(#{node := NodeBin}, Params) ->
-    validated(
-      fun() ->
-              {Node, Opts} = validate_evacuation(NodeBin, params(Params)),
-              rpc(Node, emqx_node_rebalance_evacuation, start, [Opts])
-      end).
-
-rebalance_evacuation_stop(#{node := NodeBin}, _Params) ->
-    validated(
-      fun() ->
-              Node = parse_node(NodeBin),
-              rpc(Node, emqx_node_rebalance_evacuation, stop, [])
-      end).
-
-rebalance_start(#{node := NodeBin}, Params) ->
-    validated(
-      fun() ->
-              {Node, Opts} = validate_rebalance(NodeBin, params(Params)),
-              rpc(Node, emqx_node_rebalance, start, [Opts])
-      end).
-
-rebalance_stop(#{node := NodeBin}, _Params) ->
-    validated(
-      fun() ->
-              Node = parse_node(NodeBin),
-              rpc(Node, emqx_node_rebalance, stop, [])
-      end).
-
-rpc(Node, M, F, A) ->
-    case rpc:call(Node, M, F, A) of
-        ok -> return({ok, []});
-        {error, Error} ->
-            return({error, 400, io_lib:format("~p", [Error])});
-        {badrpc, _} ->
-            return({error, 400, io_lib:format("Error communicating with node ~p", [Node])});
-        Unknown ->
-            return({error, 400, io_lib:format("Unrecognized rpc result from node ~p: ~p",
-                                              [Node, Unknown])})
-    end.
-
-format_status(Process, Stats) ->
-    Stats#{process => Process, status => enabled}.
-
-validate_evacuation(Node, Params) ->
-    NodeToEvacuate = parse_node(Node),
-    OptList = lists:map(
-                fun validate_evacuation_param/1,
-                Params),
-    {NodeToEvacuate, maps:from_list(OptList)}.
-
-validate_rebalance(Node, Params) ->
-    CoordinatorNode = parse_node(Node),
-    OptList = lists:map(
-                fun validate_rebalance_param/1,
-                Params),
-    {CoordinatorNode, maps:from_list(OptList)}.
-
-validate_evacuation_param({<<"conn_evict_rate">>, Value}) ->
-    validate_pos_int(conn_evict_rate, Value);
-validate_evacuation_param({<<"sess_evict_rate">>, Value}) ->
-    validate_pos_int(sess_evict_rate, Value);
-validate_evacuation_param({<<"redirect_to">>, Value}) ->
-    validate_binary(server_reference, Value);
-validate_evacuation_param({<<"wait_takeover">>, Value}) ->
-    validate_pos_int(wait_takeover, Value);
-validate_evacuation_param({<<"migrate_to">>, Value}) ->
-    validate_nodes(migrate_to, Value);
-validate_evacuation_param(Value) ->
-    validation_error(io_lib:format("Unknown evacuation param: ~p", [Value])).
-
-validate_rebalance_param({<<"wait_health_check">>, Value}) ->
-    validate_pos_int(wait_health_check, Value);
-validate_rebalance_param({<<"conn_evict_rate">>, Value}) ->
-    validate_pos_int(conn_evict_rate, Value);
-validate_rebalance_param({<<"sess_evict_rate">>, Value}) ->
-    validate_pos_int(sess_evict_rate, Value);
-validate_rebalance_param({<<"abs_conn_threshold">>, Value}) ->
-    validate_pos_int(abs_conn_threshold, Value);
-validate_rebalance_param({<<"rel_conn_threshold">>, Value}) ->
-    validate_fraction(rel_conn_threshold, Value);
-validate_rebalance_param({<<"abs_sess_threshold">>, Value}) ->
-    validate_pos_int(abs_sess_threshold, Value);
-validate_rebalance_param({<<"rel_sess_threshold">>, Value}) ->
-    validate_fraction(rel_sess_threshold, Value);
-validate_rebalance_param({<<"wait_takeover">>, Value}) ->
-    validate_pos_int(wait_takeover, Value);
-validate_rebalance_param({<<"nodes">>, Value}) ->
-    validate_nodes(nodes, Value);
-validate_rebalance_param(Value) ->
-    validation_error(io_lib:format("Unknown rebalance param: ~p", [Value])).
-
-validate_binary(Name, Value) when is_binary(Value) ->
-    {Name, Value};
-validate_binary(Name, _Value) ->
-    validation_error("invalid string in " ++ atom_to_list(Name)).
-
-validate_pos_int(Name, Value) ->
-    case is_integer(Value) andalso Value > 0 of
-        true -> {Name, Value};
-        false ->
-            validation_error("invalid " ++  atom_to_list(Name) ++ " value")
-    end.
-
-validate_fraction(Name, Value) ->
-    case is_number(Value) andalso Value > 1.0 of
-        true -> {Name, Value};
-        false ->
-            validation_error("invalid " ++  atom_to_list(Name) ++ " value")
-    end.
-
-validate_nodes(Name, NodeList) when is_list(NodeList) ->
-    Nodes = lists:map(
-              fun parse_node/1,
-              NodeList),
-    case emqx_node_rebalance_evacuation:available_nodes(Nodes) of
-        [] ->
-            validation_error(io_lib:format("no available nodes list in ~p: ~p", [Name, Nodes]));
-        Nodes ->
-            {Name, Nodes};
-        OtherNodes ->
-            validation_error(
-              io_lib:format("unavailable nodes in ~p: ~p",
-                            [Name, Nodes -- OtherNodes]))
-    end;
-validate_nodes(Name, Nodes) ->
-    validation_error(io_lib:format("invalid node list in ~p: ~p", [Name, Nodes])).
-
-validated(Fun) ->
-    try
-        Fun()
-    catch throw:{validation_error, Error} ->
-              return({error, 400, iolist_to_binary(Error)})
-    end.
-
-validation_error(Error) ->
-    throw({validation_error, Error}).
-
-parse_node(Bin) when is_binary(Bin) ->
-    try
-        binary_to_existing_atom(Bin)
-    catch
-        error:badarg ->
-            validation_error("invalid node: " ++ [Bin])
-    end.
-
-params([{}]) -> [];
-params(Params) -> Params.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl
deleted file mode 100644
index 3cba331de..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_app.erl
+++ /dev/null
@@ -1,34 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_app).
-
--behaviour(application).
-
--emqx_plugin(?MODULE).
-
--export([ start/2
-        , stop/1
-        ]).
-
-start(_Type, _Args) ->
-    Env = application:get_all_env(emqx_node_rebalance),
-    {ok, Sup} = emqx_node_rebalance_sup:start_link(Env),
-    ok = emqx_node_rebalance_cli:load(),
-    {ok, Sup}.
-
-stop(_State) ->
-    emqx_node_rebalance_cli:unload().
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl
deleted file mode 100644
index 10d0912c4..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl
+++ /dev/null
@@ -1,265 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_cli).
-
-%% APIs
--export([ load/0
-        , unload/0
-        , cli/1
-        ]).
-
-load() ->
-    emqx_ctl:register_command(rebalance, {?MODULE, cli}, []).
-
-unload() ->
-    emqx_ctl:unregister_command(rebalance).
-
-cli(["start" | StartArgs]) ->
-    case start_args(StartArgs) of
-        {evacuation, Opts} ->
-            case emqx_node_rebalance_evacuation:status() of
-                disabled ->
-                    ok = emqx_node_rebalance_evacuation:start(Opts),
-                    emqx_ctl:print("Rebalance(evacuation) started~n"),
-                    true;
-                {enabled, _} ->
-                    emqx_ctl:print("Rebalance is already enabled~n"),
-                    false
-            end;
-        {rebalance, Opts} ->
-            case emqx_node_rebalance:start(Opts) of
-                ok ->
-                    emqx_ctl:print("Rebalance started~n"),
-                    true;
-                {error, Reason} ->
-                    emqx_ctl:print("Rebalance start error: ~p~n", [Reason]),
-                    false
-            end;
-        {error, Error} ->
-            emqx_ctl:print("Rebalance start error: ~s~n", [Error]),
-            false
-    end;
-cli(["node-status", NodeStr]) ->
-    Node = list_to_atom(NodeStr),
-    node_status(emqx_node_rebalance_status:local_status(Node));
-cli(["node-status"]) ->
-    node_status(emqx_node_rebalance_status:local_status());
-cli(["status"]) ->
-    #{evacuations := Evacuations,
-      rebalances := Rebalances} = emqx_node_rebalance_status:global_status(),
-    lists:foreach(
-      fun({Node, Status}) ->
-        emqx_ctl:print("--------------------------------------------------------------------~n"),
-        emqx_ctl:print("Node ~p: evacuation~n~s",
-                       [Node, emqx_node_rebalance_status:format_local_status(Status)])
-      end,
-      Evacuations),
-    lists:foreach(
-      fun({Node, Status}) ->
-        emqx_ctl:print("--------------------------------------------------------------------~n"),
-        emqx_ctl:print("Node ~p: rebalance coordinator~n~s",
-                       [Node, emqx_node_rebalance_status:format_coordinator_status(Status)])
-      end,
-      Rebalances);
-cli(["stop"]) ->
-    case emqx_node_rebalance_evacuation:status() of
-        {enabled, _} ->
-            ok = emqx_node_rebalance_evacuation:stop(),
-            emqx_ctl:print("Rebalance(evacuation) stopped~n"),
-            true;
-        disabled ->
-            case emqx_node_rebalance:status() of
-                {enabled, _} ->
-                    ok = emqx_node_rebalance:stop(),
-                    emqx_ctl:print("Rebalance stopped~n"),
-                    true;
-                disabled ->
-                    emqx_ctl:print("Rebalance is already disabled~n"),
-                    false
-            end
-    end;
-cli(_) ->
-    emqx_ctl:usage(
-      [{"rebalance start --evacuation \\\n"
-        "    [--redirect-to \"Host1:Port1 Host2:Port2 ...\"] \\\n"
-        "    [--conn-evict-rate CountPerSec] \\\n"
-        "    [--migrate-to \"node1@host1 node2@host2 ...\"] \\\n"
-        "    [--wait-takeover Secs] \\\n"
-        "    [--sess-evict-rate CountPerSec]",
-        "Start current node evacuation with optional server redirect to the specified servers"},
-
-       {"rebalance start \\\n"
-        "    [--nodes \"node1@host1 node2@host2\"] \\\n"
-        "    [--wait-health-check Secs] \\\n"
-        "    [--conn-evict-rate ConnPerSec] \\\n"
-        "    [--abs-conn-threshold Count] \\\n"
-        "    [--rel-conn-threshold Fraction] \\\n"
-        "    [--conn-evict-rate ConnPerSec] \\\n"
-        "    [--wait-takeover Secs] \\\n"
-        "    [--sess-evict-rate CountPerSec] \\\n"
-        "    [--abs-sess-threshold Count] \\\n"
-        "    [--rel-sess-threshold Fraction]",
-        "Start current node evacuation with optional server redirect to the specified servers"},
-
-       {"rebalance node-status",
-        "Get current node rebalance status"},
-
-       {"rebalance node-status \"node1@host1\"",
-        "Get remote node rebalance status"},
-
-       {"rebalance status",
-        "Get statuses of all current rebalance/evacuation processes across the cluster"},
-
-       {"rebalance stop",
-        "Stop node rebalance"}]).
-
-node_status(NodeStatus) ->
-    case NodeStatus of
-        {Process, Status} when Process =:= evacuation orelse Process =:= rebalance ->
-            emqx_ctl:print("Rebalance type: ~p~n~s~n",
-                           [Process, emqx_node_rebalance_status:format_local_status(Status)]);
-        disabled ->
-            emqx_ctl:print("Rebalance disabled~n");
-        Other ->
-            emqx_ctl:print("Error detecting rebalance status: ~p~n", [Other])
-    end.
-
-start_args(Args) ->
-    case collect_args(Args, #{}) of
-        {ok, #{"--evacuation" := true} = Collected} ->
-            case validate_evacuation(maps:to_list(Collected), #{}) of
-                {ok, Validated} ->
-                    {evacuation, Validated};
-                {error, _} = Error -> Error
-            end;
-        {ok, #{} = Collected} ->
-            case validate_rebalance(maps:to_list(Collected), #{}) of
-                {ok, Validated} ->
-                    {rebalance, Validated};
-                {error, _} = Error -> Error
-            end;
-        {error, _} = Error -> Error
-    end.
-
-collect_args([], Map) -> {ok, Map};
-
-%% evacuation
-collect_args(["--evacuation" | Args], Map) ->
-    collect_args(Args, Map#{"--evacuation" => true});
-collect_args(["--redirect-to", ServerReference | Args], Map) ->
-    collect_args(Args, Map#{"--redirect-to" => ServerReference});
-collect_args(["--migrate-to", MigrateTo | Args], Map) ->
-    collect_args(Args, Map#{"--migrate-to" => MigrateTo});
-%% rebalance
-collect_args(["--nodes", Nodes | Args], Map) ->
-    collect_args(Args, Map#{"--nodes" => Nodes});
-collect_args(["--wait-health-check", WaitHealthCheck | Args], Map) ->
-    collect_args(Args, Map#{"--wait-health-check" => WaitHealthCheck});
-collect_args(["--abs-conn-threshold", AbsConnThres | Args], Map) ->
-    collect_args(Args, Map#{"--abs-conn-threshold" => AbsConnThres});
-collect_args(["--rel-conn-threshold", RelConnThres | Args], Map) ->
-    collect_args(Args, Map#{"--rel-conn-threshold" => RelConnThres});
-collect_args(["--abs-sess-threshold", AbsSessThres | Args], Map) ->
-    collect_args(Args, Map#{"--abs-sess-threshold" => AbsSessThres});
-collect_args(["--rel-sess-threshold", RelSessThres | Args], Map) ->
-    collect_args(Args, Map#{"--rel-sess-threshold" => RelSessThres});
-%% common
-collect_args(["--conn-evict-rate", ConnEvictRate | Args], Map) ->
-    collect_args(Args, Map#{"--conn-evict-rate" => ConnEvictRate});
-collect_args(["--wait-takeover", WaitTakeover | Args], Map) ->
-    collect_args(Args, Map#{"--wait-takeover" => WaitTakeover});
-collect_args(["--sess-evict-rate", SessEvictRate | Args], Map) ->
-    collect_args(Args, Map#{"--sess-evict-rate" => SessEvictRate});
-%% fallback
-collect_args(Args, _Map) ->
-    {error, io_lib:format("unknown arguments: ~p", [Args])}.
-
-validate_evacuation([], Map) ->
-    {ok, Map};
-validate_evacuation([{"--evacuation", _} | Rest], Map) ->
-    validate_evacuation(Rest, Map);
-validate_evacuation([{"--redirect-to", ServerReference} | Rest], Map) ->
-    validate_evacuation(Rest, Map#{server_reference => list_to_binary(ServerReference)});
-validate_evacuation([{"--conn-evict-rate", _} | _] = Opts, Map) ->
-    validate_pos_int(conn_evict_rate, Opts, Map, fun validate_evacuation/2);
-validate_evacuation([{"--sess-evict-rate", _} | _] = Opts, Map) ->
-    validate_pos_int(sess_evict_rate, Opts, Map, fun validate_evacuation/2);
-validate_evacuation([{"--wait-takeover", _} | _] = Opts, Map) ->
-    validate_pos_int(wait_takeover, Opts, Map, fun validate_evacuation/2);
-validate_evacuation([{"--migrate-to", MigrateTo} | Rest], Map) ->
-    Nodes = lists:map(fun list_to_atom/1, string:tokens(MigrateTo, ", ")),
-    case emqx_node_rebalance_evacuation:available_nodes(Nodes) of
-        [] ->
-            {error, "invalid --migrate-to, no nodes"};
-        Nodes ->
-            validate_evacuation(Rest, Map#{migrate_to => Nodes});
-        OtherNodes ->
-            {error,
-             io_lib:format("invalid --migrate-to, unavailable nodes: ~p",
-                           [Nodes -- OtherNodes])}
-    end;
-validate_evacuation(Rest, _Map) ->
-    {error, io_lib:format("unknown evacuation arguments: ~p", [Rest])}.
-
-validate_rebalance([], Map) ->
-    {ok, Map};
-validate_rebalance([{"--wait-health-check", _} | _] = Opts, Map) ->
-    validate_pos_int(wait_health_check, Opts, Map, fun validate_rebalance/2);
-validate_rebalance([{"--conn-evict-rate", _} | _] = Opts, Map) ->
-    validate_pos_int(conn_evict_rate, Opts, Map, fun validate_rebalance/2);
-validate_rebalance([{"--sess-evict-rate", _} | _] = Opts, Map) ->
-    validate_pos_int(sess_evict_rate, Opts, Map, fun validate_rebalance/2);
-validate_rebalance([{"--abs-conn-threshold", _} | _] = Opts, Map) ->
-    validate_pos_int(abs_conn_threshold, Opts, Map, fun validate_rebalance/2);
-validate_rebalance([{"--rel-conn-threshold", _} | _] = Opts, Map) ->
-    validate_fraction(rel_conn_threshold, Opts, Map, fun validate_rebalance/2);
-validate_rebalance([{"--abs-sess-threshold", _} | _] = Opts, Map) ->
-    validate_pos_int(abs_sess_threshold, Opts, Map, fun validate_rebalance/2);
-validate_rebalance([{"--rel-sess-threshold", _} | _] = Opts, Map) ->
-    validate_fraction(rel_sess_threshold, Opts, Map, fun validate_rebalance/2);
-validate_rebalance([{"--wait-takeover", _} | _] = Opts, Map) ->
-    validate_pos_int(wait_takeover, Opts, Map, fun validate_rebalance/2);
-validate_rebalance([{"--nodes", NodeStr} | Rest], Map) ->
-    Nodes = lists:map(fun list_to_atom/1, string:tokens(NodeStr, ", ")),
-    case emqx_node_rebalance:available_nodes(Nodes) of
-        [] ->
-            {error, "invalid --nodes, no nodes"};
-        Nodes ->
-            validate_rebalance(Rest, Map#{nodes => Nodes});
-        OtherNodes ->
-            {error,
-             io_lib:format("invalid --nodes, unavailable nodes: ~p",
-                           [Nodes -- OtherNodes])}
-    end;
-validate_rebalance(Rest, _Map) ->
-    {error, io_lib:format("unknown rebalance arguments: ~p", [Rest])}.
-
-validate_fraction(Name, [{OptionName, Value} | Rest], Map, Next) ->
-    case string:to_float(Value) of
-        {Num, ""} when Num > 1.0 ->
-           Next(Rest, Map#{Name => Num});
-        _ ->
-            {error, "invalid " ++ OptionName ++ " value"}
-    end.
-
-validate_pos_int(Name, [{OptionName, Value} | Rest], Map, Next) ->
-    case string:to_integer(Value) of
-        {Int, ""} when Int > 0 ->
-           Next(Rest, Map#{Name => Int});
-        _ ->
-            {error, "invalid " ++ OptionName ++ " value"}
-    end.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl
deleted file mode 100644
index 5dc99d736..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl
+++ /dev/null
@@ -1,298 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_evacuation).
-
--include("emqx_node_rebalance.hrl").
-
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx/include/types.hrl").
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
--export([start/1,
-         status/0,
-         stop/0
-        ]).
-
--export([start_link/0]).
-
--behavior(gen_statem).
-
--export([init/1,
-         callback_mode/0,
-         handle_event/4,
-         code_change/4
-        ]).
-
--export([is_node_available/0,
-         available_nodes/1]).
-
--ifdef(TEST).
--export([migrate_to/1]).
--endif.
-
-%%--------------------------------------------------------------------
-%% APIs
-%%--------------------------------------------------------------------
-
--define(EVICT_INTERVAL_NO_NODES, 30000).
-
--type migrate_to() :: [node()] | undefined.
-
--type start_opts() :: #{server_reference => emqx_eviction_agent:server_reference(),
-                        conn_evict_rate => pos_integer(),
-                        sess_evict_rate => pos_integer(),
-                        wait_takeover => pos_integer(),
-                        migrate_to => migrate_to()
-                       }.
--type start_error() :: already_started | eviction_agent_busy.
--type stats() :: #{
-                   initial_conns := non_neg_integer(),
-                   initial_sessions := non_neg_integer(),
-                   current_conns := non_neg_integer(),
-                   current_sessions := non_neg_integer(),
-                   conn_evict_rate := pos_integer(),
-                   sess_evict_rate := pos_integer(),
-                   server_reference := emqx_eviction_agent:server_reference(),
-                   migrate_to := migrate_to()
-                  }.
--type status() :: {started, stats()} | stopped.
-
--spec start(start_opts()) -> ok_or_error(start_error()).
-start(StartOpts) ->
-    Opts = maps:merge(default_opts(), StartOpts),
-    gen_statem:call(?MODULE, {start, Opts}).
-
--spec stop() -> ok_or_error(not_started).
-stop() ->
-    gen_statem:call(?MODULE, stop).
-
--spec status() -> status().
-status() ->
-    gen_statem:call(?MODULE, status).
-
--spec start_link() -> startlink_ret().
-start_link() ->
-    gen_statem:start_link({local, ?MODULE}, ?MODULE, [], []).
-
--spec available_nodes(list(node())) -> list(node()).
-available_nodes(Nodes) when is_list(Nodes) ->
-    {Available, _} = rpc:multicall(Nodes, ?MODULE, is_node_available, []),
-    lists:filter(fun is_atom/1, Available).
-
-%%--------------------------------------------------------------------
-%% gen_statem callbacks
-%%--------------------------------------------------------------------
-
-callback_mode() -> handle_event_function.
-
-%% states: disabled, evicting_conns, waiting_takeover, evicting_sessions, prohibiting
-
-init([]) ->
-    case emqx_node_rebalance_evacuation_persist:read(default_opts()) of
-        {ok, #{server_reference := ServerReference} = Opts} ->
-            ?LOG(warning, "Restoring evacuation state: ~p", [Opts]),
-            case emqx_eviction_agent:enable(?MODULE, ServerReference) of
-                ok ->
-                    Data = init_data(#{}, Opts),
-                    ok = warn_enabled(),
-                    {ok, evicting_conns, Data, [{state_timeout, 0, evict_conns}]};
-                {error, eviction_agent_busy} ->
-                    emqx_node_rebalance_evacuation_persist:clear(),
-                    {ok, disabled, #{}}
-            end;
-        none ->
-            {ok, disabled, #{}}
-    end.
-
-%% start
-handle_event({call, From},
-             {start, #{server_reference := ServerReference} = Opts},
-             disabled,
-             #{} = Data) ->
-    case emqx_eviction_agent:enable(?MODULE, ServerReference) of
-        ok ->
-            NewData = init_data(Data, Opts),
-            ok = emqx_node_rebalance_evacuation_persist:save(Opts),
-            ?LOG(warning, "Node evacuation started"),
-            {next_state,
-             evicting_conns,
-             NewData,
-             [{state_timeout, 0, evict_conns},
-              {reply, From, ok}]};
-        {error, eviction_agent_busy} ->
-            {keep_state_and_data,
-              [{reply, From, {error, eviction_agent_busy}}]}
-    end;
-handle_event({call, From}, {start, _Opts}, _State, #{}) ->
-    {keep_state_and_data,
-     [{reply, From, {error, already_started}}]};
-
-%% stop
-handle_event({call, From}, stop, disabled, #{}) ->
-    {keep_state_and_data,
-     [{reply, From, {error, not_started}}]};
-handle_event({call, From}, stop, _State, Data) ->
-    ok = emqx_node_rebalance_evacuation_persist:clear(),
-    _ = emqx_eviction_agent:disable(?MODULE),
-    ?LOG(warning, "Node evacuation stopped"),
-    {next_state,
-     disabled,
-     deinit(Data),
-     [{reply, From, ok}]};
-
-%% status
-handle_event({call, From}, status, disabled, #{}) ->
-    {keep_state_and_data,
-     [{reply, From, disabled}]};
-handle_event({call, From}, status, State, #{migrate_to := MigrateTo} = Data) ->
-    Stats = maps:with(
-              [initial_conns, current_conns,
-               initial_sessions, current_sessions,
-               server_reference, conn_evict_rate, sess_evict_rate],
-              Data),
-    {keep_state_and_data,
-     [{reply, From, {enabled, Stats#{state => State, migrate_to => migrate_to(MigrateTo)}}}]};
-
-%% conn eviction
-handle_event(state_timeout,
-             evict_conns,
-             evicting_conns,
-             #{conn_evict_rate := ConnEvictRate,
-               wait_takeover := WaitTakeover} = Data) ->
-    case emqx_eviction_agent:status() of
-        {enabled, #{connections := Conns}} when Conns > 0 ->
-            ok = emqx_eviction_agent:evict_connections(ConnEvictRate),
-            ?tp(debug, node_evacuation_evict_conn, #{conn_evict_rate => ConnEvictRate}),
-            ?LOG(warning, "Node evacuation evict_conns, count=~p, conn_evict_rate=~p",
-                 [Conns, ConnEvictRate]),
-            NewData = Data#{current_conns => Conns},
-            {keep_state,
-             NewData,
-             [{state_timeout, ?EVICT_INTERVAL, evict_conns}]};
-        {enabled, #{connections := 0}} ->
-            NewData = Data#{current_conns => 0},
-            ?LOG(warning, "Node evacuation evict_conns over"),
-            {next_state,
-             waiting_takeover,
-             NewData,
-             [{state_timeout, timer:seconds(WaitTakeover), evict_sessions}]}
-    end;
-
-handle_event(state_timeout,
-             evict_sessions,
-             waiting_takeover,
-             Data) ->
-            ?LOG(warning, "Node evacuation wait_takeover over"),
-            {next_state,
-             evicting_sessions,
-             Data,
-             [{state_timeout, 0, evict_sessions}]};
-
-%% session eviction
-handle_event(state_timeout,
-             evict_sessions,
-             evicting_sessions,
-             #{sess_evict_rate := SessEvictRate,
-               migrate_to := MigrateTo,
-               current_sessions := CurrSessCount} = Data) ->
-    case emqx_eviction_agent:status() of
-        {enabled, #{sessions := SessCount}} when SessCount > 0 ->
-            case migrate_to(MigrateTo) of
-                [] ->
-                    ?LOG(warning,
-                         "No nodes are available to evacuate sessions, session_count=~p",
-                         [CurrSessCount]),
-                    {keep_state_and_data,
-                     [{state_timeout, ?EVICT_INTERVAL_NO_NODES, evict_sessions}]};
-                Nodes ->
-                    ok = emqx_eviction_agent:evict_sessions(SessEvictRate, Nodes),
-                    ?LOG(warning, "Node evacuation evict_sessions, count=~p, sess_evict_rate=~p,"
-                         "target_nodes=~p", [SessCount, SessEvictRate, Nodes]),
-                    NewData = Data#{current_sessions => SessCount},
-                    {keep_state,
-                     NewData,
-                     [{state_timeout, ?EVICT_INTERVAL, evict_sessions}]}
-            end;
-        {enabled, #{sessions := 0}} ->
-            ?tp(debug, node_evacuation_evict_sess_over, #{}),
-            ?LOG(warning, "Node evacuation evict_sessions over"),
-            NewData = Data#{current_sessions => 0},
-            {next_state,
-             prohibiting,
-             NewData}
-    end;
-
-handle_event({call, From}, Msg, State, Data) ->
-    ?LOG(warning, "Unknown call: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
-    {keep_state_and_data,
-     [{reply, From, ignored}]};
-
-handle_event(info, Msg, State, Data) ->
-    ?LOG(warning, "Unknown Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
-    keep_state_and_data;
-
-handle_event(cast, Msg, State, Data) ->
-    ?LOG(warning, "Unknown cast Msg: ~p, State: ~p, Data: ~p", [Msg, State, Data]),
-    keep_state_and_data.
-
-code_change(_Vsn, State, Data, _Extra) ->
-    {ok, State, Data}.
-
-%%--------------------------------------------------------------------
-%% internal funs
-%%--------------------------------------------------------------------
-
-default_opts() ->
-    #{
-      server_reference => undefined,
-      conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE,
-      sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE,
-      wait_takeover => ?DEFAULT_WAIT_TAKEOVER,
-      migrate_to => undefined
-     }.
-
-init_data(Data0, Opts) ->
-    Data1 = maps:merge(Data0, Opts),
-    {enabled, #{connections := ConnCount, sessions := SessCount}} = emqx_eviction_agent:status(),
-    Data1#{
-      initial_conns => ConnCount,
-      current_conns => ConnCount,
-      initial_sessions => SessCount,
-      current_sessions => SessCount
-     }.
-
-deinit(Data) ->
-    Keys = [initial_conns, current_conns, initial_sessions, current_sessions]
-           ++ maps:keys(default_opts()),
-    maps:without(Keys, Data).
-
-warn_enabled() ->
-    Msg = "Node evacuation is enabled. The node will not receive connections.",
-    ?LOG(warning, Msg),
-    io:format(standard_error, "~s~n", [Msg]).
-
-migrate_to(undefined) ->
-    migrate_to(all_nodes());
-migrate_to(Nodes) when is_list(Nodes) ->
-    available_nodes(Nodes).
-
-is_node_available() ->
-    disabled = emqx_eviction_agent:status(),
-    node().
-
-all_nodes() ->
-    ekka_mnesia:cluster_nodes(all) -- [node()].
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl
deleted file mode 100644
index 06d8800da..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl
+++ /dev/null
@@ -1,109 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_evacuation_persist).
-
--export([save/1,
-         clear/0,
-         read/1]).
-
-
--ifdef(TEST).
--export([evacuation_filepath/0]).
--endif.
-
--include("emqx_node_rebalance.hrl").
--include_lib("emqx/include/types.hrl").
-
-%%--------------------------------------------------------------------
-%% APIs
-%%--------------------------------------------------------------------
-
-%% do not persist `migrate_to`:
-%% * after restart there is nothing to migrate
-%% * this value may be invalid after node was offline
--type start_opts() :: #{server_reference => emqx_eviction_agent:server_reference(),
-                        conn_evict_rate => pos_integer(),
-                        sess_evict_rate => pos_integer(),
-                        wait_takeover => pos_integer()
-                       }.
-
--spec save(start_opts()) -> ok_or_error(term()).
-save(#{server_reference := ServerReference,
-       conn_evict_rate := ConnEvictRate,
-       sess_evict_rate := SessEvictRate,
-       wait_takeover := WaitTakeover} = Data)
-  when (is_binary(ServerReference) orelse ServerReference =:= undefined) andalso
-       is_integer(ConnEvictRate) andalso ConnEvictRate > 0 andalso
-       is_integer(SessEvictRate) andalso SessEvictRate > 0 andalso
-       is_integer(WaitTakeover) andalso WaitTakeover >= 0 ->
-    Filepath = evacuation_filepath(),
-    case filelib:ensure_dir(Filepath) of
-        ok ->
-            JsonData = emqx_json:encode(
-                         prepare_for_encode(maps:with(persist_keys(), Data)),
-                         [pretty]),
-            file:write_file(Filepath, JsonData);
-        {error, _} = Error -> Error
-    end.
-
--spec clear() -> ok.
-clear() ->
-    file:delete(evacuation_filepath()).
-
--spec read(start_opts()) -> {ok, start_opts()} | none.
-read(DefaultOpts) ->
-    case file:read_file(evacuation_filepath()) of
-        {ok, Data} ->
-            case emqx_json:safe_decode(Data, [return_maps]) of
-                {ok, Map} when is_map(Map) ->
-                    {ok, map_to_opts(DefaultOpts, Map)};
-                _NotAMap ->
-                    {ok, DefaultOpts}
-            end;
-        {error, _} ->
-            none
-    end.
-
-%%--------------------------------------------------------------------
-%% Internal funcs
-%%--------------------------------------------------------------------
-
-persist_keys() ->
-    [server_reference,
-     conn_evict_rate,
-     sess_evict_rate,
-     wait_takeover].
-
-prepare_for_encode(#{server_reference := undefined} = Data) ->
-    Data#{server_reference => null};
-prepare_for_encode(Data) -> Data.
-
-format_after_decode(#{server_reference := null} = Data) ->
-    Data#{server_reference => undefined};
-format_after_decode(Data) -> Data.
-
-map_to_opts(DefaultOpts, Map) ->
-    format_after_decode(
-      map_to_opts(
-        maps:to_list(DefaultOpts), Map, #{})).
-
-map_to_opts([], _Map, Opts) -> Opts;
-map_to_opts([{Key, DefaultVal} | Rest], Map, Opts) ->
-    map_to_opts(Rest, Map, Opts#{Key => maps:get(atom_to_binary(Key), Map, DefaultVal)}).
-
-evacuation_filepath() ->
-    filename:join([emqx:get_env(data_dir), ?EVACUATION_FILENAME]).
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl
deleted file mode 100644
index 4002d7c13..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl
+++ /dev/null
@@ -1,225 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_status).
-
--export([local_status/0,
-         local_status/1,
-         global_status/0,
-         format_local_status/1,
-         format_coordinator_status/1]).
-
-%% For RPC
--export([evacuation_status/0,
-         rebalance_status/0]).
-
-%%--------------------------------------------------------------------
-%% APIs
-%%--------------------------------------------------------------------
-
--spec local_status() -> disabled | {evacuation, map()} | {rebalance, map()}.
-local_status() ->
-    case emqx_node_rebalance_evacuation:status() of
-        {enabled, Status} ->
-            {evacuation, evacuation(Status)};
-        disabled ->
-            case emqx_node_rebalance_agent:status() of
-                {enabled, CoordinatorPid} ->
-                    case emqx_node_rebalance:status(CoordinatorPid) of
-                        {enabled, Status} ->
-                            local_rebalance(Status, node());
-                        disabled ->
-                            disabled
-                    end;
-                disabled ->
-                    disabled
-            end
-    end.
-
--spec local_status(node()) -> disabled | {evacuation, map()} | {rebalance, map()}.
-local_status(Node) ->
-    rpc:call(Node, ?MODULE, ?FUNCTION_NAME, []).
-
--spec format_local_status(map()) -> iodata().
-format_local_status(Status) ->
-    format_status(Status, local_status_field_format_order()).
-
--spec global_status() -> #{rebalances := [{node(), map()}], evacuations := [{node(), map()}]}.
-global_status() ->
-    Nodes = ekka_mnesia:cluster_nodes(all),
-    {RebalanceResults, _} = rpc:multicall(Nodes, ?MODULE, rebalance_status, []),
-    Rebalances = [{Node, coordinator_rebalance(Status)} || {Node, {enabled, Status}} <- RebalanceResults],
-    {EvacuatioResults, _} = rpc:multicall(Nodes, ?MODULE, evacuation_status, []),
-    Evacuations = [{Node, evacuation(Status)} || {Node, {enabled, Status}} <- EvacuatioResults],
-    #{rebalances => Rebalances, evacuations => Evacuations}.
-
--spec format_coordinator_status(map()) -> iodata().
-format_coordinator_status(Status) ->
-    format_status(Status, coordinator_status_field_format_order()).
-
-%%--------------------------------------------------------------------
-%% Internal functions
-%%--------------------------------------------------------------------
-
-evacuation(Status) ->
-    #{
-      state => maps:get(state, Status),
-      connection_eviction_rate => maps:get(conn_evict_rate, Status),
-      session_eviction_rate => maps:get(sess_evict_rate, Status),
-      connection_goal => 0,
-      session_goal => 0,
-      session_recipients => maps:get(migrate_to, Status),
-      stats => #{
-        initial_connected => maps:get(initial_conns, Status),
-        current_connected => maps:get(current_conns, Status),
-        initial_sessions => maps:get(initial_sessions, Status),
-        current_sessions => maps:get(current_sessions, Status)
-      }
-     }.
-
-local_rebalance(#{donors := Donors} = Stats, Node) ->
-    case lists:member(Node, Donors) of
-        true -> {rebalance, donor_rebalance(Stats, Node)};
-        false -> disabled
-    end.
-
-donor_rebalance(Status, Node) ->
-    Opts = maps:get(opts, Status),
-    InitialConnCounts = maps:get(initial_conn_counts, Status),
-    InitialSessCounts = maps:get(initial_sess_counts, Status),
-
-    CurrentStats = #{
-                     initial_connected => maps:get(Node, InitialConnCounts),
-                     initial_sessions => maps:get(Node, InitialSessCounts),
-                     current_connected => emqx_eviction_agent:connection_count(),
-                     current_sessions => emqx_eviction_agent:session_count(),
-                     current_disconnected_sessions => emqx_eviction_agent:session_count(
-                                                       disconnected)
-                    },
-    maps:from_list(
-      [
-       {state, maps:get(state, Status)},
-       {coordinator_node, maps:get(coordinator_node, Status)},
-       {connection_eviction_rate, maps:get(conn_evict_rate, Opts)},
-       {session_eviction_rate, maps:get(sess_evict_rate, Opts)},
-       {recipients, maps:get(recipients, Status)},
-       {stats, CurrentStats}
-      ] ++
-      [{connection_goal, maps:get(recipient_conn_avg, Status)}
-       || maps:is_key(recipient_conn_avg, Status)
-      ] ++
-      [{disconnected_session_goal, maps:get(recipient_sess_avg, Status)}
-       || maps:is_key(recipient_sess_avg, Status)
-      ]).
-
-coordinator_rebalance(Status) ->
-    Opts = maps:get(opts, Status),
-    maps:from_list(
-      [
-       {state, maps:get(state, Status)},
-       {coordinator_node, maps:get(coordinator_node, Status)},
-       {connection_eviction_rate, maps:get(conn_evict_rate, Opts)},
-       {session_eviction_rate, maps:get(sess_evict_rate, Opts)},
-       {recipients, maps:get(recipients, Status)},
-       {donors, maps:get(donors, Status)}
-      ] ++
-      [{connection_goal, maps:get(recipient_conn_avg, Status)}
-       || maps:is_key(recipient_conn_avg, Status)
-      ] ++
-      [{disconnected_session_goal, maps:get(recipient_sess_avg, Status)}
-       || maps:is_key(recipient_sess_avg, Status)
-      ] ++
-      [{donor_conn_avg, maps:get(donor_conn_avg, Status)}
-       || maps:is_key(donor_conn_avg, Status)
-      ] ++
-      [{donor_sess_avg, maps:get(donor_sess_avg, Status)}
-       || maps:is_key(donor_sess_avg, Status)
-      ]).
-
-local_status_field_format_order() ->
-    [state,
-     coordinator_node,
-     connection_eviction_rate,
-     session_eviction_rate,
-     connection_goal,
-     session_goal,
-     disconnected_session_goal,
-     session_recipients,
-     recipients,
-     stats].
-
-coordinator_status_field_format_order() ->
-    [state,
-     coordinator_node,
-     donors,
-     recipients,
-     connection_eviction_rate,
-     session_eviction_rate,
-     connection_goal,
-     disconnected_session_goal,
-     donor_conn_avg,
-     donor_sess_avg].
-
-format_status(Status, FieldOrder) ->
-    Fields = lists:flatmap(
-               fun(FieldName) ->
-                       maps:to_list(maps:with([FieldName], Status))
-               end,
-               FieldOrder),
-    lists:map(
-      fun format_local_status_field/1,
-      Fields).
-
-format_local_status_field({state, State}) ->
-    io_lib:format("Rebalance state: ~p~n", [State]);
-format_local_status_field({coordinator_node, Node}) ->
-    io_lib:format("Coordinator node: ~p~n", [Node]);
-format_local_status_field({connection_eviction_rate, ConnEvictRate}) ->
-    io_lib:format("Connection eviction rate: ~p connections/second~n", [ConnEvictRate]);
-format_local_status_field({session_eviction_rate, SessEvictRate}) ->
-    io_lib:format("Session eviction rate: ~p sessions/second~n", [SessEvictRate]);
-format_local_status_field({connection_goal, ConnGoal}) ->
-    io_lib:format("Connection goal: ~p~n", [ConnGoal]);
-format_local_status_field({session_goal, SessGoal}) ->
-    io_lib:format("Session goal: ~p~n", [SessGoal]);
-format_local_status_field({disconnected_session_goal, DisconnSessGoal}) ->
-    io_lib:format("Disconnected session goal: ~p~n", [DisconnSessGoal]);
-format_local_status_field({session_recipients, SessionRecipients}) ->
-    io_lib:format("Session recipient  nodes: ~p~n", [SessionRecipients]);
-format_local_status_field({recipients, Recipients}) ->
-    io_lib:format("Recipient nodes: ~p~n", [Recipients]);
-format_local_status_field({donors, Donors}) ->
-    io_lib:format("Donor nodes: ~p~n", [Donors]);
-format_local_status_field({donor_conn_avg, DonorConnAvg}) ->
-    io_lib:format("Current average donor node connection count: ~p~n", [DonorConnAvg]);
-format_local_status_field({donor_sess_avg, DonorSessAvg}) ->
-    io_lib:format("Current average donor node disconnected session count: ~p~n", [DonorSessAvg]);
-format_local_status_field({stats, Stats}) ->
-    format_local_stats(Stats).
-
-format_local_stats(Stats) ->
-    ["Channel statistics:\n" |
-     lists:map(
-       fun({Name, Value}) ->
-               io_lib:format("  ~p: ~p~n", [Name, Value])
-       end,
-       maps:to_list(Stats))].
-
-evacuation_status() ->
-    {node(), emqx_node_rebalance_evacuation:status()}.
-
-rebalance_status() ->
-    {node(), emqx_node_rebalance:status()}.
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl
deleted file mode 100644
index e677eb22e..000000000
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_sup.erl
+++ /dev/null
@@ -1,44 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_sup).
-
--behaviour(supervisor).
-
--export([start_link/1]).
-
--export([init/1]).
-
-start_link(Env) ->
-	supervisor:start_link({local, ?MODULE}, ?MODULE, [Env]).
-
-init([_Env]) ->
-    Childs = [child_spec(emqx_node_rebalance_evacuation, []),
-              child_spec(emqx_node_rebalance_agent, []),
-              child_spec(emqx_node_rebalance, [])],
-    {ok, {
-       {one_for_one, 10, 3600},
-       Childs}
-    }.
-
-child_spec(Mod, Args) ->
-    #{id => Mod,
-      start => {Mod, start_link, Args},
-      restart => permanent,
-      shutdown => 5000,
-      type => worker,
-      modules => [Mod]
-     }.
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl
deleted file mode 100644
index c5c27aec0..000000000
--- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_SUITE.erl
+++ /dev/null
@@ -1,183 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("emqx/include/emqx.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
--import(emqx_eviction_agent_test_helpers,
-        [emqtt_connect_many/1]).
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
-    Config.
-
-end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
-    Config.
-
-init_per_testcase(_Case, Config) ->
-    _ = emqx_node_rebalance:stop(),
-    Node = emqx_node_helpers:start_slave(
-             recipient1,
-             #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}),
-    [{recipient_node, Node} | Config].
-
-end_per_testcase(_Case, Config) ->
-    _ = emqx_node_helpers:stop_slave(?config(recipient_node, Config)),
-    _ = emqx_node_rebalance:stop().
-
-t_rebalance(Config) ->
-    process_flag(trap_exit, true),
-    RecipientNode = ?config(recipient_node, Config),
-
-    Nodes = [node(), RecipientNode],
-
-    _Conns = emqtt_connect_many(500),
-
-    Opts = #{conn_evict_rate => 10,
-             sess_evict_rate => 10,
-             evict_interval => 10,
-             abs_conn_threshold => 50,
-             abs_sess_threshold => 50,
-             rel_conn_threshold => 1.0,
-             rel_sess_threshold => 1.0,
-             wait_health_check => 0.01,
-             wait_takeover => 0.01,
-             nodes => Nodes
-            },
-
-    ?check_trace(
-       ?wait_async_action(
-          emqx_node_rebalance:start(Opts),
-          #{?snk_kind := emqx_node_rebalance_evict_sess_over},
-          10000),
-       fun({ok, _}, Trace) ->
-        ?assertMatch(
-           [_ | _],
-           ?of_kind(emqx_node_rebalance_evict_sess_over, Trace))
-       end),
-
-    DonorConnCount = emqx_eviction_agent:connection_count(),
-    DonorSessCount = emqx_eviction_agent:session_count(),
-    DonorDSessCount = emqx_eviction_agent:session_count(disconnected),
-
-    RecipientConnCount = rpc:call(RecipientNode, emqx_eviction_agent, connection_count, []),
-    RecipientSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, []),
-    RecipientDSessCount = rpc:call(RecipientNode, emqx_eviction_agent, session_count, [disconnected]),
-
-    ct:pal("Donor: conn=~p, sess=~p, dsess=~p",
-             [DonorConnCount, DonorSessCount, DonorDSessCount]),
-    ct:pal("Recipient: conn=~p, sess=~p, dsess=~p",
-             [RecipientConnCount, RecipientSessCount, RecipientDSessCount]),
-
-    ?assert(DonorConnCount - 50 =< RecipientConnCount),
-    ?assert(DonorDSessCount - 50 =< RecipientDSessCount).
-
-t_rebalance_node_crash(Config) ->
-    process_flag(trap_exit, true),
-    RecipientNode = ?config(recipient_node, Config),
-
-    Nodes = [node(), RecipientNode],
-
-    _Conns = emqtt_connect_many(50),
-
-    Opts = #{conn_evict_rate => 10,
-             sess_evict_rate => 10,
-             evict_interval => 10,
-             abs_conn_threshold => 50,
-             abs_sess_threshold => 50,
-             rel_conn_threshold => 1.0,
-             rel_sess_threshold => 1.0,
-             wait_health_check => 0.01,
-             wait_takeover => 0.01,
-             nodes => Nodes
-            },
-
-    ok = emqx_node_rebalance:start(Opts),
-
-    ?check_trace(
-       ?wait_async_action(
-          emqx_node_helpers:stop_slave(?config(recipient_node, Config)),
-          #{?snk_kind := emqx_node_rebalance_started},
-          1000),
-       fun(_Result, _Trace) -> ok end),
-
-    ?assertEqual(
-       disabled,
-       emqx_node_rebalance:status()).
-
-t_no_need_to_rebalance(_Config) ->
-    process_flag(trap_exit, true),
-
-    ?assertEqual(
-        {error, nothing_to_balance},
-        emqx_node_rebalance:start(#{})),
-
-    _Conns = emqtt_connect_many(50),
-
-    ?assertEqual(
-        {error, nothing_to_balance},
-        emqx_node_rebalance:start(#{})).
-
-t_unknown_mesages(Config) ->
-    process_flag(trap_exit, true),
-    RecipientNode = ?config(recipient_node, Config),
-
-    Nodes = [node(), RecipientNode],
-
-    _Conns = emqtt_connect_many(500),
-
-    Opts = #{wait_health_check => 100,
-             abs_conn_threshold => 50,
-             nodes => Nodes
-            },
-
-    Pid = whereis(emqx_node_rebalance),
-
-    Pid ! unknown,
-    ok = gen_server:cast(Pid, unknown),
-    ?assertEqual(
-       ignored,
-       gen_server:call(Pid, unknown)),
-
-    ok = emqx_node_rebalance:start(Opts),
-
-    Pid ! unknown,
-    ok = gen_server:cast(Pid, unknown),
-    ?assertEqual(
-       ignored,
-       gen_server:call(Pid, unknown)).
-
-t_available_nodes(Config) ->
-    rpc:call(?config(recipient_node, Config),
-             emqx_eviction_agent,
-             enable,
-             [test_rebalance, undefined]),
-    ?assertEqual(
-       [node()],
-       emqx_node_rebalance:available_nodes(
-         [node(), ?config(recipient_node, Config)])).
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl
deleted file mode 100644
index 30edd51fe..000000000
--- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_agent_SUITE.erl
+++ /dev/null
@@ -1,163 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_agent_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("emqx/include/emqx.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
-    Config.
-
-end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
-    Config.
-
-init_per_testcase(_Case, Config) ->
-    _ = emqx_node_rebalance_evacuation:stop(),
-    Node = emqx_node_helpers:start_slave(
-             evacuate1,
-             #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}),
-    [{evacuate_node, Node} | Config].
-
-end_per_testcase(_Case, Config) ->
-    _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)),
-    _ = emqx_node_rebalance_evacuation:stop().
-
-t_enable_disable(_Config) ->
-    ?assertEqual(
-       disabled,
-       emqx_node_rebalance_agent:status()),
-
-    ?assertEqual(
-       ok,
-       emqx_node_rebalance_agent:enable(self())),
-
-    ?assertEqual(
-       {error, already_enabled},
-       emqx_node_rebalance_agent:enable(self())),
-
-    ?assertEqual(
-       {enabled, self()},
-       emqx_node_rebalance_agent:status()),
-
-    ?assertEqual(
-       {error, invalid_coordinator},
-       emqx_node_rebalance_agent:disable(spawn_link(fun() -> ok end))),
-
-    ?assertEqual(
-       ok,
-       emqx_node_rebalance_agent:disable(self())),
-
-    ?assertEqual(
-       {error, already_disabled},
-       emqx_node_rebalance_agent:disable(self())),
-
-    ?assertEqual(
-       disabled,
-       emqx_node_rebalance_agent:status()).
-
-t_enable_egent_busy(_Config) ->
-    ok = emqx_eviction_agent:enable(rebalance_test, undefined),
-
-    ?assertEqual(
-       {error, eviction_agent_busy},
-       emqx_node_rebalance_agent:enable(self())),
-
-    ok = emqx_eviction_agent:disable(rebalance_test).
-
-% The following tests verify that emqx_node_rebalance_agent correctly links
-% coordinator process with emqx_eviction_agent-s.
-
-t_rebalance_agent_coordinator_fail(Config) ->
-    process_flag(trap_exit, true),
-
-    Node = ?config(evacuate_node, Config),
-
-
-    CoordinatorPid = spawn_link(
-                       fun() ->
-                               receive
-                                done -> ok
-                               end
-                       end),
-
-    ?assertEqual(
-       disabled,
-       rpc:call(Node, emqx_eviction_agent, status, [])),
-
-    ?assertEqual(
-       ok,
-       rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid])),
-
-    ?assertMatch(
-       {enabled, _},
-       rpc:call(Node, emqx_eviction_agent, status, [])),
-
-    EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]),
-    true = link(EvictionAgentPid),
-
-    true = exit(CoordinatorPid, kill),
-
-    receive
-        {'EXIT', EvictionAgentPid, _} -> true
-    after
-        1000 -> ?assert(false, "emqx_eviction_agent did not exit")
-    end.
-
-t_rebalance_agent_fail(Config) ->
-    process_flag(trap_exit, true),
-
-    Node = ?config(evacuate_node, Config),
-
-    CoordinatorPid = spawn_link(
-                       fun() ->
-                               receive
-                                done -> ok
-                               end
-                       end),
-
-    ?assertEqual(
-       ok,
-       rpc:call(Node, emqx_node_rebalance_agent, enable, [CoordinatorPid])),
-
-    EvictionAgentPid = rpc:call(Node, erlang, whereis, [emqx_eviction_agent]),
-    true = exit(EvictionAgentPid, kill),
-
-    receive
-        {'EXIT', CoordinatorPid, _} -> true
-    after
-        1000 -> ?assert(false, "emqx_eviction_agent did not exit")
-    end.
-
-t_unknown_messages(_Config) ->
-    Pid = whereis(emqx_node_rebalance_agent),
-
-    ok = gen_server:cast(Pid, unknown),
-
-    Pid ! unknown,
-
-    ignored = gen_server:call(Pid, unknown).
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl
deleted file mode 100644
index a15b8e8ab..000000000
--- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl
+++ /dev/null
@@ -1,321 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_api_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
-
--import(emqx_mgmt_api_test_helpers,
-        [request_api/3,
-         request_api/5,
-         auth_header_/0,
-         api_path/1]).
-
--import(emqx_eviction_agent_test_helpers,
-        [emqtt_connect_many/1]).
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance, emqx_management]),
-    Config.
-
-end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_management, emqx_node_rebalance, emqx_eviction_agent]),
-    Config.
-
-init_per_testcase(Case, Config)
-  when Case =:= t_start_evacuation_validation
-       orelse Case =:= t_start_rebalance_validation
-       orelse Case =:= t_start_stop_rebalance ->
-    _ = emqx_node_rebalance:stop(),
-    _ = emqx_node_rebalance_evacuation:stop(),
-    Node = emqx_node_helpers:start_slave(
-             recipient1,
-             #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}),
-    [{recipient_node, Node} | Config];
-init_per_testcase(_Case, Config) ->
-    _ = emqx_node_rebalance:stop(),
-    _ = emqx_node_rebalance_evacuation:stop(),
-    Config.
-
-end_per_testcase(Case, Config)
-  when Case =:= t_start_evacuation_validation
-       orelse Case =:= t_start_rebalance_validation
-       orelse Case =:= t_start_stop_rebalance ->
-    _ = emqx_node_helpers:stop_slave(?config(recipient_node, Config)),
-    _ = emqx_node_rebalance:stop(),
-    _ = emqx_node_rebalance_evacuation:stop();
-end_per_testcase(_Case, _Config) ->
-    _ = emqx_node_rebalance:stop(),
-    _ = emqx_node_rebalance_evacuation:stop().
-
-t_start_evacuation_validation(Config) ->
-    BadOpts = [#{conn_evict_rate => <<"conn">>},
-               #{sess_evict_rate => <<"sess">>},
-               #{redirect_to => 123},
-               #{wait_takeover => <<"wait">>},
-               #{migrate_to => []},
-               #{migrate_to => <<"migrate_to">>},
-               #{migrate_to => [<<"bad_node">>]},
-               #{migrate_to => [<<"bad_node">>, atom_to_binary(node())]},
-               #{unknown => <<"Value">>}
-              ],
-    lists:foreach(
-      fun(Opts) ->
-              ?assertMatch(
-                 {ok, #{}},
-                 api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"],
-                          Opts)),
-
-              ?assertMatch(
-                 {ok, #{<<"status">> := <<"disabled">>}},
-                 api_get(["load_rebalance", "status"]))
-
-      end,
-      BadOpts),
-    ?assertMatch(
-       {ok, #{}},
-       api_post(["load_rebalance", "bad@node", "evacuation", "start"],
-                #{})),
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"disabled">>}},
-       api_get(["load_rebalance", "status"])),
-
-    ?assertMatch(
-       {ok, #{}},
-       api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"],
-                #{conn_evict_rate => 10,
-                  sess_evict_rate => 10,
-                  wait_takeover => 10,
-                  redirect_to => <<"srv">>,
-                  migrate_to => [atom_to_binary(?config(recipient_node, Config))]})),
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"enabled">>}},
-       api_get(["load_rebalance", "status"])).
-
-
-t_start_rebalance_validation(Config) ->
-    BadOpts = [#{conn_evict_rate => <<"conn">>},
-               #{sess_evict_rate => <<"sess">>},
-               #{abs_conn_threshold => <<"act">>},
-               #{rel_conn_threshold => <<"rct">>},
-               #{abs_sess_threshold => <<"act">>},
-               #{rel_sess_threshold => <<"rct">>},
-               #{wait_takeover => <<"wait">>},
-               #{wait_health_check => <<"wait">>},
-               #{nodes => <<"nodes">>},
-               #{nodes => []},
-               #{nodes => [<<"bad_node">>]},
-               #{nodes => [<<"bad_node">>, atom_to_binary(node())]},
-               #{unknown => <<"Value">>}
-              ],
-    lists:foreach(
-      fun(Opts) ->
-              ?assertMatch(
-                 {ok, #{}},
-                 api_post(["load_rebalance", atom_to_list(node()), "start"],
-                          Opts)),
-
-              ?assertMatch(
-                 {ok, #{<<"status">> := <<"disabled">>}},
-                 api_get(["load_rebalance", "status"]))
-
-      end,
-      BadOpts),
-    ?assertMatch(
-       {ok, #{}},
-       api_post(["load_rebalance", "bad@node", "start"],
-                #{})),
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"disabled">>}},
-       api_get(["load_rebalance", "status"])),
-
-    _Conns = emqtt_connect_many(50),
-
-    ?assertMatch(
-       {ok, #{}},
-       api_post(["load_rebalance", atom_to_list(node()), "start"],
-                #{conn_evict_rate => 10,
-                  sess_evict_rate => 10,
-                  wait_takeover => 10,
-                  wait_health_check => 10,
-                  abs_conn_threshold => 10,
-                  rel_conn_threshold => 1.001,
-                  abs_sess_threshold => 10,
-                  rel_sess_threshold => 1.001,
-                  nodes => [atom_to_binary(?config(recipient_node, Config)),
-                            atom_to_binary(node())]})),
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"enabled">>}},
-       api_get(["load_rebalance", "status"])).
-
-t_start_stop_evacuation(_Config) ->
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"disabled">>}},
-        api_get(["load_rebalance", "status"])),
-
-    ?assertMatch(
-       {ok, #{}},
-        api_post(["load_rebalance", atom_to_list(node()), "evacuation", "start"],
-                 #{conn_evict_rate => 10,
-                   sess_evict_rate => 20})),
-
-    ?assertMatch(
-       {ok, #{<<"state">> := _,
-              <<"process">> := <<"evacuation">>,
-              <<"connection_eviction_rate">> := 10,
-              <<"session_eviction_rate">> := 20,
-              <<"connection_goal">> := 0,
-              <<"session_goal">> := 0,
-              <<"stats">> := #{
-                               <<"initial_connected">> := _,
-                               <<"current_connected">> := _,
-                               <<"initial_sessions">> := _,
-                               <<"current_sessions">> := _
-                              }}},
-        api_get(["load_rebalance", "status"])),
-
-    ?assertMatch(
-       {ok, #{<<"rebalances">> := #{},
-              <<"evacuations">> :=
-              #{<<"test@127.0.0.1">> := #{<<"state">> := _,
-                                          <<"connection_eviction_rate">> := 10,
-                                          <<"session_eviction_rate">> := 20,
-                                          <<"connection_goal">> := 0,
-                                          <<"session_goal">> := 0,
-                                          <<"stats">> := #{
-                                                           <<"initial_connected">> := _,
-                                                           <<"current_connected">> := _,
-                                                           <<"initial_sessions">> := _,
-                                                           <<"current_sessions">> := _
-                                                          }
-                                         }
-               }}},
-        api_get(["load_rebalance", "global_status"])),
-
-    ?assertMatch(
-       {ok, #{}},
-        api_post(["load_rebalance", atom_to_list(node()), "evacuation", "stop"],
-                 #{})),
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"disabled">>}},
-        api_get(["load_rebalance", "status"])),
-
-    ?assertMatch(
-       {ok, #{<<"evacuations">> := #{}, <<"rebalances">> := #{}}},
-        api_get(["load_rebalance", "global_status"])).
-
-t_start_stop_rebalance(Config) ->
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"disabled">>}},
-        api_get(["load_rebalance", "status"])),
-
-    _Conns = emqtt_connect_many(100),
-
-    ?assertMatch(
-       {ok, #{}},
-        api_post(["load_rebalance", atom_to_list(node()), "start"],
-                 #{conn_evict_rate => 10,
-                   sess_evict_rate => 20,
-                   abs_conn_threshold => 10})),
-
-    ?assertMatch(
-       {ok, #{<<"state">> := _,
-              <<"process">> := <<"rebalance">>,
-              <<"coordinator_node">> := _,
-              <<"connection_eviction_rate">> := 10,
-              <<"session_eviction_rate">> := 20,
-              <<"stats">> := #{
-                               <<"initial_connected">> := _,
-                               <<"current_connected">> := _,
-                               <<"initial_sessions">> := _,
-                               <<"current_sessions">> := _
-                              }}},
-        api_get(["load_rebalance", "status"])),
-
-    DonorNode = atom_to_binary(node()),
-    RecipientNode = atom_to_binary(?config(recipient_node, Config)),
-
-    ?assertMatch(
-       {ok, #{<<"evacuations">> := #{},
-              <<"rebalances">> :=
-              #{<<"test@127.0.0.1">> := #{<<"state">> := _,
-                                          <<"coordinator_node">> := _,
-                                          <<"connection_eviction_rate">> := 10,
-                                          <<"session_eviction_rate">> := 20,
-                                          <<"donors">> := [DonorNode],
-                                          <<"recipients">> := [RecipientNode]
-                                         }
-               }}},
-        api_get(["load_rebalance", "global_status"])),
-
-    ?assertMatch(
-       {ok, #{}},
-        api_post(["load_rebalance", atom_to_list(node()), "stop"],
-                 #{})),
-
-    ?assertMatch(
-       {ok, #{<<"status">> := <<"disabled">>}},
-        api_get(["load_rebalance", "status"])),
-
-    ?assertMatch(
-       {ok, #{<<"evacuations">> := #{}, <<"rebalances">> := #{}}},
-        api_get(["load_rebalance", "global_status"])).
-
-t_availability_check(_Config) ->
-
-    ?assertMatch(
-       {ok, #{}},
-       api_get(["load_rebalance", "availability_check"])),
-
-    ok = emqx_node_rebalance_evacuation:start(#{}),
-
-    ?assertMatch(
-       {error, {_, 503, _}},
-       api_get(["load_rebalance", "availability_check"])),
-
-    ok = emqx_node_rebalance_evacuation:stop(),
-
-    ?assertMatch(
-       {ok, #{}},
-       api_get(["load_rebalance", "availability_check"])).
-
-api_get(Path) ->
-    case request_api(get, api_path(Path), auth_header_()) of
-        {ok, ResponseBody} ->
-            {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])};
-        {error, _} = Error -> Error
-    end.
-
-api_post(Path, Data) ->
-    case request_api(post, api_path(Path), [], auth_header_(), Data) of
-        {ok, ResponseBody} ->
-            {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])};
-        {error, _} = Error -> Error
-    end.
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl
deleted file mode 100644
index 943746298..000000000
--- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_cli_SUITE.erl
+++ /dev/null
@@ -1,199 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_cli_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
-
--import(emqx_eviction_agent_test_helpers,
-        [emqtt_connect_many/1]).
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
-    Config.
-
-end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
-    Config.
-
-
-init_per_testcase(t_rebalance, Config) ->
-    _ = emqx_node_rebalance_evacuation:stop(),
-    Node = emqx_node_helpers:start_slave(
-             evacuate1,
-             #{start_apps => [emqx, emqx_eviction_agent, emqx_node_rebalance]}),
-    [{evacuate_node, Node} | Config];
-init_per_testcase(_Case, Config) ->
-    _ = emqx_node_rebalance_evacuation:stop(),
-    _ = emqx_node_rebalance:stop(),
-    Config.
-
-end_per_testcase(t_rebalance, Config) ->
-    _ = emqx_node_rebalance_evacuation:stop(),
-    _ = emqx_node_rebalance:stop(),
-    _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config));
-end_per_testcase(_Case, _Config) ->
-    _ = emqx_node_rebalance_evacuation:stop(),
-    _ = emqx_node_rebalance:stop().
-
-t_evacuation(_Config) ->
-    %% usage
-    ok = emqx_node_rebalance_cli:cli(["foobar"]),
-
-    %% status
-    ok = emqx_node_rebalance_cli:cli(["status"]),
-    ok = emqx_node_rebalance_cli:cli(["node-status"]),
-    ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]),
-
-    %% start with invalid args
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--evacuation", "--foo-bar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--evacuation", "--conn-evict-rate", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--evacuation", "--sess-evict-rate", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--evacuation", "--wait-takeover", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--evacuation",
-                                     "--migrate-to", "nonexistent@node"])),
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--evacuation",
-                                     "--migrate-to", ""])),
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--evacuation",
-                                     "--unknown-arg"])),
-    ?assert(
-        emqx_node_rebalance_cli:cli(["start", "--evacuation",
-                                     "--conn-evict-rate", "10",
-                                     "--sess-evict-rate", "10",
-                                     "--wait-takeover", "10",
-                                     "--migrate-to", atom_to_list(node()),
-                                     "--redirect-to", "srv"])),
-
-    %% status
-    ok = emqx_node_rebalance_cli:cli(["status"]),
-    ok = emqx_node_rebalance_cli:cli(["node-status"]),
-    ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]),
-
-    ?assertMatch(
-       {enabled, #{}},
-       emqx_node_rebalance_evacuation:status()),
-
-    %% already enabled
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--evacuation",
-                                     "--conn-evict-rate", "10",
-                                     "--redirect-to", "srv"])),
-
-    %% stop
-    true = emqx_node_rebalance_cli:cli(["stop"]),
-
-    false = emqx_node_rebalance_cli:cli(["stop"]),
-
-    ?assertEqual(
-       disabled,
-       emqx_node_rebalance_evacuation:status()).
-
-t_rebalance(Config) ->
-    %% start with invalid args
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--foo-bar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--conn-evict-rate", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--abs-conn-threshold", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--rel-conn-threshold", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--sess-evict-rate", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--abs-sess-threshold", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--rel-sess-threshold", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--wait-takeover", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start", "--wait-health-check", "foobar"])),
-
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start",
-                                     "--nodes", "nonexistent@node"])),
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start",
-                                     "--nodes", ""])),
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start",
-                                     "--nodes", atom_to_list(?config(evacuate_node, Config))])),
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start",
-                                     "--unknown-arg"])),
-
-    _ = emqtt_connect_many(20),
-
-    ?assert(
-        emqx_node_rebalance_cli:cli(["start",
-                                     "--conn-evict-rate", "10",
-                                     "--abs-conn-threshold", "10",
-                                     "--rel-conn-threshold", "1.1",
-                                     "--sess-evict-rate", "10",
-                                     "--abs-sess-threshold", "10",
-                                     "--rel-sess-threshold", "1.1",
-                                     "--wait-takeover", "10",
-                                     "--nodes", atom_to_list(node()) ++ ","
-                                                ++ atom_to_list(?config(evacuate_node, Config))
-                                    ])),
-
-    %% status
-    ok = emqx_node_rebalance_cli:cli(["status"]),
-    ok = emqx_node_rebalance_cli:cli(["node-status"]),
-    ok = emqx_node_rebalance_cli:cli(["node-status", atom_to_list(node())]),
-
-    ?assertMatch(
-       {enabled, #{}},
-       emqx_node_rebalance:status()),
-
-    %% already enabled
-    ?assertNot(
-        emqx_node_rebalance_cli:cli(["start"])),
-
-    %% stop
-    true = emqx_node_rebalance_cli:cli(["stop"]),
-
-    false = emqx_node_rebalance_cli:cli(["stop"]),
-
-    ?assertEqual(
-       disabled,
-       emqx_node_rebalance:status()).
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl
deleted file mode 100644
index 23f35f61e..000000000
--- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl
+++ /dev/null
@@ -1,194 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_evacuation_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
-
--import(emqx_eviction_agent_test_helpers,
-        [emqtt_connect/0, emqtt_connect/2, emqtt_try_connect/0]).
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
-    Config.
-
-end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
-    Config.
-
-init_per_testcase(_Case, Config) ->
-    _ = emqx_node_rebalance_evacuation:stop(),
-    Node = emqx_node_helpers:start_slave(
-             evacuate1,
-             #{start_apps => [emqx, emqx_eviction_agent]}),
-    [{evacuate_node, Node} | Config].
-
-end_per_testcase(_Case, Config) ->
-    _ = emqx_node_helpers:stop_slave(?config(evacuate_node, Config)),
-    _ = emqx_node_rebalance_evacuation:stop().
-
-t_agent_busy(Config) ->
-
-    ok = emqx_eviction_agent:enable(other_rebalance, undefined),
-
-    ?assertEqual(
-       {error, eviction_agent_busy},
-       emqx_node_rebalance_evacuation:start(opts(Config))),
-
-    emqx_eviction_agent:disable(other_rebalance).
-
-
-t_already_started(Config) ->
-    ok = emqx_node_rebalance_evacuation:start(opts(Config)),
-
-    ?assertEqual(
-       {error, already_started},
-       emqx_node_rebalance_evacuation:start(opts(Config))),
-
-    ok = emqx_node_rebalance_evacuation:stop().
-
-t_not_started(_Config) ->
-    ?assertEqual(
-       {error, not_started},
-       emqx_node_rebalance_evacuation:stop()).
-
-t_start(Config) ->
-    process_flag(trap_exit, true),
-
-    ok = emqx_node_rebalance_evacuation:start(opts(Config)),
-    ?assertMatch(
-       {error, {use_another_server, #{}}},
-       emqtt_try_connect()),
-    ok = emqx_node_rebalance_evacuation:stop().
-
-t_persistence(Config) ->
-    process_flag(trap_exit, true),
-
-    ok = emqx_node_rebalance_evacuation:start(opts(Config)),
-
-    ?assertMatch(
-       {error, {use_another_server, #{}}},
-       emqtt_try_connect()),
-
-    ok = supervisor:terminate_child(emqx_node_rebalance_sup, emqx_node_rebalance_evacuation),
-    {ok, _} = supervisor:restart_child(emqx_node_rebalance_sup, emqx_node_rebalance_evacuation),
-
-    ?assertMatch(
-       {error, {use_another_server, #{}}},
-       emqtt_try_connect()),
-    ?assertMatch(
-       {enabled, #{conn_evict_rate := 10}},
-       emqx_node_rebalance_evacuation:status()),
-
-    ok = emqx_node_rebalance_evacuation:stop().
-
-t_conn_evicted(Config) ->
-    process_flag(trap_exit, true),
-
-    {ok, C} = emqtt_connect(),
-
-    ?check_trace(
-       ?wait_async_action(
-          emqx_node_rebalance_evacuation:start(opts(Config)),
-          #{?snk_kind := node_evacuation_evict_conn},
-          1000),
-       fun(_Result, _Trace) -> ok end),
-
-    ct:sleep(100),
-
-    ?assertMatch(
-       {error, {use_another_server, #{}}},
-       emqtt_try_connect()),
-
-    ?assertNot(
-       is_process_alive(C)).
-
-t_migrate_to(Config) ->
-    ?assertEqual(
-       [?config(evacuate_node, Config)],
-       emqx_node_rebalance_evacuation:migrate_to(undefined)),
-
-    ?assertEqual(
-       [],
-       emqx_node_rebalance_evacuation:migrate_to(['unknown@node'])),
-
-    rpc:call(?config(evacuate_node, Config),
-             emqx_eviction_agent,
-             enable,
-             [test_rebalance, undefined]),
-
-    ?assertEqual(
-       [],
-       emqx_node_rebalance_evacuation:migrate_to(undefined)).
-
-t_session_evicted(Config) ->
-    process_flag(trap_exit, true),
-
-    {ok, C} = emqtt_connect(<<"client_with_sess">>, false),
-
-    ?check_trace(
-       ?wait_async_action(
-          emqx_node_rebalance_evacuation:start(opts(Config)),
-          #{?snk_kind := node_evacuation_evict_sess_over},
-          5000),
-       fun(_Result, Trace) ->
-        ?assertMatch(
-           [_ | _],
-           ?of_kind(node_evacuation_evict_sess_over, Trace))
-       end),
-
-    receive
-        {'EXIT', C, {disconnected, ?RC_USE_ANOTHER_SERVER, _}} -> ok
-    after 1000 ->
-              ?assert(false, "Connection not evicted")
-    end,
-
-    [ChannelPid] = emqx_cm_registry:lookup_channels(<<"client_with_sess">>),
-
-    ?assertEqual(
-       ?config(evacuate_node, Config),
-       node(ChannelPid)).
-
-t_unknown_messages(Config) ->
-    ok = emqx_node_rebalance_evacuation:start(opts(Config)),
-
-    whereis(emqx_node_rebalance_evacuation) ! unknown,
-
-    gen_server:cast(emqx_node_rebalance_evacuation, unknown),
-
-    ?assertEqual(
-       ignored,
-       gen_server:call(emqx_node_rebalance_evacuation, unknown)),
-
-    ok = emqx_node_rebalance_evacuation:stop().
-
-opts(Config) ->
-    #{
-      server_reference => <<"srv">>,
-      conn_evict_rate => 10,
-      sess_evict_rate => 10,
-      wait_takeover => 1,
-      migrate_to => [?config(evacuate_node, Config)]
-     }.
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl
deleted file mode 100644
index ba1a12775..000000000
--- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_persist_SUITE.erl
+++ /dev/null
@@ -1,107 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_node_rebalance_evacuation_persist_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
-
-all() ->
-    emqx_ct:all(?MODULE).
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_eviction_agent, emqx_node_rebalance]),
-    Config.
-
-end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_node_rebalance, emqx_eviction_agent]),
-    Config.
-
-init_per_testcase(_Case, Config) ->
-    _ = emqx_node_rebalance_evacuation_persist:clear(),
-    Config.
-
-end_per_testcase(_Case, _Config) ->
-    _ = emqx_node_rebalance_evacuation_persist:clear().
-
-t_save_read(_Config) ->
-    DefaultOpts = #{server_reference => <<"default_ref">>,
-                    conn_evict_rate => 2001,
-                    sess_evict_rate => 2002,
-                    wait_takeover => 2003
-                   },
-
-    Opts0 = #{server_reference => <<"ref">>,
-             conn_evict_rate => 1001,
-             sess_evict_rate => 1002,
-             wait_takeover => 1003
-            },
-    ok = emqx_node_rebalance_evacuation_persist:save(Opts0),
-
-    {ok, ReadOpts0} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts),
-    ?assertEqual(Opts0, ReadOpts0),
-
-    Opts1 = Opts0#{server_reference => undefined},
-    ok = emqx_node_rebalance_evacuation_persist:save(Opts1),
-
-    {ok, ReadOpts1} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts),
-    ?assertEqual(Opts1, ReadOpts1).
-
-t_read_default(_Config) ->
-    ok = write_evacuation_file(<<"{}">>),
-
-    DefaultOpts = #{server_reference => <<"ref">>,
-                    conn_evict_rate => 1001,
-                    sess_evict_rate => 1002,
-                    wait_takeover => 1003
-                   },
-
-    {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts),
-    ?assertEqual(DefaultOpts, ReadOpts).
-
-t_read_bad_data(_Config) ->
-    ok = write_evacuation_file(<<"{bad json">>),
-
-    DefaultOpts = #{server_reference => <<"ref">>,
-                    conn_evict_rate => 1001,
-                    sess_evict_rate => 1002,
-                    wait_takeover => 1003
-                   },
-
-    {ok, ReadOpts} = emqx_node_rebalance_evacuation_persist:read(DefaultOpts),
-    ?assertEqual(DefaultOpts, ReadOpts).
-
-t_clear(_Config) ->
-    ok = write_evacuation_file(<<"{}">>),
-
-    ?assertMatch(
-        {ok, _},
-        emqx_node_rebalance_evacuation_persist:read(#{})),
-
-    ok = emqx_node_rebalance_evacuation_persist:clear(),
-
-    ?assertEqual(
-        none,
-        emqx_node_rebalance_evacuation_persist:read(#{})).
-
-write_evacuation_file(Json) ->
-    ok = filelib:ensure_dir(emqx_node_rebalance_evacuation_persist:evacuation_filepath()),
-    ok = file:write_file(
-           emqx_node_rebalance_evacuation_persist:evacuation_filepath(),
-           Json).
diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl
index 62b3205cd..d0dac7fe1 100644
--- a/data/loaded_plugins.tmpl
+++ b/data/loaded_plugins.tmpl
@@ -6,5 +6,3 @@
 {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}.
 {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}.
 {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}.
-{emqx_eviction_agent, true}.
-{emqx_node_rebalance, true}.
diff --git a/rebar.config.erl b/rebar.config.erl
index 04ffaefbd..841399a73 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -297,8 +297,6 @@ relx_plugin_apps(ReleaseType) ->
     , emqx_recon
     , emqx_rule_engine
     , emqx_sasl
-    , emqx_eviction_agent
-    , emqx_node_rebalance
     ]
     ++ [emqx_telemetry || not is_enterprise()]
     ++ relx_plugin_apps_per_rel(ReleaseType)
diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl
index 1c8849745..8b5b7ffb2 100644
--- a/src/emqx_plugins.erl
+++ b/src/emqx_plugins.erl
@@ -254,9 +254,7 @@ default_plugins() ->
         {emqx_recon, true},
         {emqx_telemetry, true},
         {emqx_rule_engine, true},
-        {emqx_bridge_mqtt, false},
-        {emqx_eviction_agent, true},
-        {emqx_node_rebalance, true}
+        {emqx_bridge_mqtt, false}
     ].
 
 -endif.
diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl
index aa548d766..eef5152b0 100644
--- a/test/emqx_plugins_SUITE.erl
+++ b/test/emqx_plugins_SUITE.erl
@@ -110,10 +110,8 @@ default_plugins() ->
     [
         {emqx_bridge_mqtt, false},
         {emqx_dashboard, true},
-        {emqx_eviction_agent, true},
         {emqx_management, true},
         {emqx_modules, true},
-        {emqx_node_rebalance, true},
         {emqx_recon, true},
         {emqx_retainer, false},
         {emqx_rule_engine, true},

From 0287d6c755532df8c01fe1717e5bfec65dc36bb9 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 16 Aug 2022 10:30:05 +0200
Subject: [PATCH 30/52] docs: fix comment typo in emqx_retainer_sup.erl

Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com>
---
 apps/emqx_retainer/src/emqx_retainer_sup.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl
index d3a979788..2028affb6 100644
--- a/apps/emqx_retainer/src/emqx_retainer_sup.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl
@@ -35,5 +35,5 @@ init([Env]) ->
              modules  => [emqx_retainer]} || not is_managed_by_modules()]}}.
 
 is_managed_by_modules() ->
-    %% always flase for opensource edition
+    %% always false for opensource edition
     false.

From 4055b2025981857b5ac00728375e265015128ccb Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 16 Aug 2022 21:32:24 +0800
Subject: [PATCH 31/52] fix: sql compare to undefined values

---
 .../src/emqx_rule_runtime.erl                 |  14 +-
 .../test/emqx_rule_engine_SUITE.erl           | 208 ++++++++++++++++++
 2 files changed, 216 insertions(+), 6 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index 88b2c9143..145e30eec 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -216,10 +216,8 @@ match_conditions({}, _Data) ->
     true.
 
 %% comparing numbers against strings
-compare(Op, undefined, undefined) ->
-    do_compare(Op, undefined, undefined);
-compare(_Op, L, R) when L == undefined; R == undefined ->
-    false;
+compare(Op, L, R) when L == undefined; R == undefined ->
+    do_compare(Op, L, R);
 compare(Op, L, R) when is_number(L), is_binary(R) ->
     do_compare(Op, L, number(R));
 compare(Op, L, R) when is_binary(L), is_number(R) ->
@@ -232,10 +230,14 @@ compare(Op, L, R) ->
     do_compare(Op, L, R).
 
 do_compare('=', L, R) -> L == R;
+do_compare('>', L, R) when L == undefined; R == undefined -> false;
 do_compare('>', L, R) -> L > R;
+do_compare('<', L, R) when L == undefined; R == undefined -> false;
 do_compare('<', L, R) -> L < R;
-do_compare('<=', L, R) -> L =< R;
-do_compare('>=', L, R) -> L >= R;
+do_compare('<=', L, R) ->
+    do_compare('=', L, R) orelse do_compare('<', L, R);
+do_compare('>=', L, R) ->
+    do_compare('=', L, R) orelse do_compare('>', L, R);
 do_compare('<>', L, R) -> L /= R;
 do_compare('!=', L, R) -> L /= R;
 do_compare('=~', T, F) -> emqx_topic:match(T, F).
diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
index 9031c3df8..e1851b76c 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
@@ -126,6 +126,10 @@ groups() ->
        t_sqlparse_array_range_1,
        t_sqlparse_array_range_2,
        t_sqlparse_true_false,
+       t_sqlparse_compare_null_null,
+       t_sqlparse_compare_null_notnull,
+       t_sqlparse_compare_notnull_null,
+       t_sqlparse_compare,
        t_sqlparse_new_map,
        t_sqlparse_invalid_json
       ]},
@@ -2486,6 +2490,210 @@ t_sqlparse_true_false(_Config) ->
                    <<"c">> := [true]
                    }, Res00).
 
+-define(TEST_SQL(SQL),
+    emqx_rule_sqltester:test(
+        #{<<"rawsql">> => SQL,
+          <<"ctx">> => #{<<"payload">> => <<"">>,
+                         <<"topic">> => <<"t/a">>}})).
+t_sqlparse_compare_null_null(_Config) ->
+    %% test undefined == undefined
+    Sql00 = "select "
+            " a = b as c "
+            "from \"t/#\" ",
+    {ok, Res00} = ?TEST_SQL(Sql00),
+    ?assertMatch(#{<<"c">> := true
+                   }, Res00),
+
+    %% test undefined != undefined
+    Sql01 = "select "
+            " a != b as c "
+            "from \"t/#\" ",
+    {ok, Res01} = ?TEST_SQL(Sql01),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res01),
+
+    %% test undefined > undefined
+    Sql02 = "select "
+            " a > b as c "
+            "from \"t/#\" ",
+    {ok, Res02} = ?TEST_SQL(Sql02),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res02),
+
+    %% test undefined < undefined
+    Sql03 = "select "
+            " a < b as c "
+            "from \"t/#\" ",
+    {ok, Res03} = ?TEST_SQL(Sql03),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res03),
+
+    %% test undefined <= undefined
+    Sql04 = "select "
+            " a <= b as c "
+            "from \"t/#\" ",
+    {ok, Res04} = ?TEST_SQL(Sql04),
+    ?assertMatch(#{<<"c">> := true
+                   }, Res04),
+
+    %% test undefined >= undefined
+    Sql05 = "select "
+            " a >= b as c "
+            "from \"t/#\" ",
+    {ok, Res05} = ?TEST_SQL(Sql05),
+    ?assertMatch(#{<<"c">> := true
+                   }, Res05).
+
+t_sqlparse_compare_null_notnull(_Config) ->
+    %% test undefined == b
+    Sql00 = "select "
+            " 'b' as b, a = b as c "
+            "from \"t/#\" ",
+    {ok, Res00} = ?TEST_SQL(Sql00),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res00),
+
+    %% test undefined != b
+    Sql01 = "select "
+            " 'b' as b, a != b as c "
+            "from \"t/#\" ",
+    {ok, Res01} = ?TEST_SQL(Sql01),
+    ?assertMatch(#{<<"c">> := true
+                   }, Res01),
+
+    %% test undefined > b
+    Sql02 = "select "
+            " 'b' as b, a > b as c "
+            "from \"t/#\" ",
+    {ok, Res02} = ?TEST_SQL(Sql02),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res02),
+
+    %% test undefined < b
+    Sql03 = "select "
+            " 'b' as b, a < b as c "
+            "from \"t/#\" ",
+    {ok, Res03} = ?TEST_SQL(Sql03),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res03),
+
+    %% test undefined <= b
+    Sql04 = "select "
+            " 'b' as b, a <= b as c "
+            "from \"t/#\" ",
+    {ok, Res04} = ?TEST_SQL(Sql04),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res04),
+
+    %% test undefined >= b
+    Sql05 = "select "
+            " 'b' as b, a >= b as c "
+            "from \"t/#\" ",
+    {ok, Res05} = ?TEST_SQL(Sql05),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res05).
+
+t_sqlparse_compare_notnull_null(_Config) ->
+    %% test 'a' == undefined
+    Sql00 = "select "
+            " 'a' as a, a = b as c "
+            "from \"t/#\" ",
+    {ok, Res00} = ?TEST_SQL(Sql00),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res00),
+
+    %% test 'a' != undefined
+    Sql01 = "select "
+            " 'a' as a, a != b as c "
+            "from \"t/#\" ",
+    {ok, Res01} = ?TEST_SQL(Sql01),
+    ?assertMatch(#{<<"c">> := true
+                   }, Res01),
+
+    %% test 'a' > undefined
+    Sql02 = "select "
+            " 'a' as a, a > b as c "
+            "from \"t/#\" ",
+    {ok, Res02} = ?TEST_SQL(Sql02),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res02),
+
+    %% test 'a' < undefined
+    Sql03 = "select "
+            " 'a' as a, a < b as c "
+            "from \"t/#\" ",
+    {ok, Res03} = ?TEST_SQL(Sql03),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res03),
+
+    %% test 'a' <= undefined
+    Sql04 = "select "
+            " 'a' as a, a <= b as c "
+            "from \"t/#\" ",
+    {ok, Res04} = ?TEST_SQL(Sql04),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res04),
+
+    %% test 'a' >= undefined
+    Sql05 = "select "
+            " 'a' as a, a >= b as c "
+            "from \"t/#\" ",
+    {ok, Res05} = ?TEST_SQL(Sql05),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res05).
+
+t_sqlparse_compare(_Config) ->
+    Sql00 = "select "
+            " 'a' as a, 'a' as b, a = b as c "
+            "from \"t/#\" ",
+    {ok, Res00} = ?TEST_SQL(Sql00),
+    ?assertMatch(#{<<"c">> := true
+                   }, Res00),
+
+    Sql01 = "select "
+            " is_null(a) as c "
+            "from \"t/#\" ",
+    {ok, Res01} = ?TEST_SQL(Sql01),
+    ?assertMatch(#{<<"c">> := true
+                   }, Res01),
+
+    Sql02 = "select "
+            " 1 as a, 2 as b, a < b as c "
+            "from \"t/#\" ",
+    {ok, Res02} = ?TEST_SQL(Sql02),
+    ?assertMatch(#{<<"c">> := true
+                   }, Res02),
+
+    Sql03 = "select "
+            " 1 as a, 2 as b, a > b as c "
+            "from \"t/#\" ",
+    {ok, Res03} = ?TEST_SQL(Sql03),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res03),
+
+    Sql04 = "select "
+            " 1 as a, 2 as b, a = b as c "
+            "from \"t/#\" ",
+    {ok, Res04} = ?TEST_SQL(Sql04),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res04),
+
+    %% test 'a' >= undefined
+    Sql05 = "select "
+            " 1 as a, 2 as b, a >= b as c "
+            "from \"t/#\" ",
+    {ok, Res05} = ?TEST_SQL(Sql05),
+    ?assertMatch(#{<<"c">> := false
+                   }, Res05),
+
+    %% test 'a' >= undefined
+    Sql06 = "select "
+            " 1 as a, 2 as b, a <= b as c "
+            "from \"t/#\" ",
+    {ok, Res06} = ?TEST_SQL(Sql06),
+    ?assertMatch(#{<<"c">> := true
+                   }, Res06).
+
 t_sqlparse_new_map(_Config) ->
     %% construct a range without 'as'
     Sql00 = "select "

From 582ead1d77cba4bda669b931c22c63e2ca9be4a4 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Tue, 16 Aug 2022 21:38:57 +0800
Subject: [PATCH 32/52] fix: update appup for rule engine

---
 .../src/emqx_rule_engine.appup.src            | 22 +++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index 1c5fb0e7b..792f99310 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -1,10 +1,17 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.3.13",[
+    {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.12",[
+    {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
    {"4.3.11",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
@@ -166,11 +173,18 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.3.13",[
+    {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
+   {"4.3.12",[
+    {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}
+    ]},
    {"4.3.11",
     [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
     [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},

From 67ec6e0e66fe7e41f2176b3ae8a5c967fe10d44e Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 16 Aug 2022 10:24:05 +0800
Subject: [PATCH 33/52] fix: log RuleId for take action failed

---
 .../emqx_rule_engine/include/rule_actions.hrl | 19 ++++++++
 .../src/emqx_rule_actions.erl                 | 18 +++++---
 .../src/emqx_rule_runtime.erl                 | 45 +++++++++----------
 apps/emqx_rule_engine/src/emqx_rule_utils.erl | 34 ++++++++++++++
 .../src/emqx_web_hook_actions.erl             | 10 +++--
 5 files changed, 93 insertions(+), 33 deletions(-)

diff --git a/apps/emqx_rule_engine/include/rule_actions.hrl b/apps/emqx_rule_engine/include/rule_actions.hrl
index e432c4399..4571849d3 100644
--- a/apps/emqx_rule_engine/include/rule_actions.hrl
+++ b/apps/emqx_rule_engine/include/rule_actions.hrl
@@ -1,3 +1,19 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
 -compile({parse_transform, emqx_rule_actions_trans}).
 
 -type selected_data() :: map().
@@ -6,6 +22,9 @@
 
 -define(BINDING_KEYS, '__bindings__').
 
+-define(LOG_RULE_ACTION(Level, Metadata, Fmt, Args),
+        emqx_rule_utils:log_action(Level, Metadata, Fmt, Args)).
+
 -define(bound_v(Key, ENVS0),
     maps:get(Key,
         maps:get(?BINDING_KEYS, ENVS0, #{}))).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index d665a0c96..557ddf423 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -171,9 +171,13 @@ on_action_create_republish(Id, Params = #{
 on_action_republish(_Selected, Envs = #{
             topic := Topic,
             headers := #{republish_by := ActId},
-            ?BINDING_KEYS := #{'Id' := ActId}
+            ?BINDING_KEYS := #{'Id' := ActId},
+            metadata := Metadata
         }) ->
-    ?LOG(error, "[republish] recursively republish detected, msg topic: ~p, target topic: ~p",
+    ?LOG_RULE_ACTION(
+        error,
+        Metadata,
+        "[republish] recursively republish detected, msg topic: ~p, target topic: ~p",
         [Topic, ?bound_v('TargetTopic', Envs)]),
     emqx_rule_metrics:inc_actions_error(?bound_v('Id', Envs)),
     {badact, recursively_republish};
@@ -186,8 +190,9 @@ on_action_republish(Selected, _Envs = #{
                 'TargetQoS' := TargetQoS,
                 'TopicTks' := TopicTks,
                 'PayloadTks' := PayloadTks
-            } = Bindings}) ->
-    ?LOG(debug, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]),
+            } = Bindings,
+            metadata := Metadata}) ->
+    ?LOG_RULE_ACTION(debug, Metadata, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]),
     TargetRetain = maps:get('TargetRetain', Bindings, false),
     Message =
         #message{
@@ -210,8 +215,9 @@ on_action_republish(Selected, _Envs = #{
                 'TargetQoS' := TargetQoS,
                 'TopicTks' := TopicTks,
                 'PayloadTks' := PayloadTks
-            } = Bindings}) ->
-    ?LOG(debug, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]),
+            } = Bindings,
+            metadata := Metadata}) ->
+    ?LOG_RULE_ACTION(debug, Metadata, "[republish] republish to: ~p, Selected: ~p", [TargetTopic, Selected]),
     TargetRetain = maps:get('TargetRetain', Bindings, false),
     Message =
         #message{
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index 88b2c9143..22fac23c0 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -17,6 +17,7 @@
 -module(emqx_rule_runtime).
 
 -include("rule_engine.hrl").
+-include("rule_actions.hrl").
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
@@ -54,36 +55,37 @@ apply_rules([Rule|More], Input) ->
     apply_rule(Rule, Input),
     apply_rules(More, Input).
 
-apply_rule(Rule = #rule{id = RuleID}, Input) ->
+apply_rule(Rule = #rule{id = RuleId}, Input) ->
     clear_rule_payload(),
-    ok = emqx_rule_metrics:inc_rules_matched(RuleID),
-    try do_apply_rule(Rule, add_metadata(Input, #{rule_id => RuleID}))
+    ok = emqx_rule_metrics:inc_rules_matched(RuleId),
+    %% Add metadata here caused we need support `metadata` and `rule_id` in SQL
+    try do_apply_rule(Rule, emqx_rule_utils:add_metadata(Input, #{rule_id => RuleId}))
     catch
         %% ignore the errors if select or match failed
         _:Reason = {select_and_transform_error, Error} ->
-            emqx_rule_metrics:inc_rules_exception(RuleID),
+            emqx_rule_metrics:inc_rules_exception(RuleId),
             ?LOG(warning, "SELECT clause exception for ~s failed: ~p",
-                 [RuleID, Error]),
+                 [RuleId, Error]),
             {error, Reason};
         _:Reason = {match_conditions_error, Error} ->
-            emqx_rule_metrics:inc_rules_exception(RuleID),
+            emqx_rule_metrics:inc_rules_exception(RuleId),
             ?LOG(warning, "WHERE clause exception for ~s failed: ~p",
-                 [RuleID, Error]),
+                 [RuleId, Error]),
             {error, Reason};
         _:Reason = {select_and_collect_error, Error} ->
-            emqx_rule_metrics:inc_rules_exception(RuleID),
+            emqx_rule_metrics:inc_rules_exception(RuleId),
             ?LOG(warning, "FOREACH clause exception for ~s failed: ~p",
-                 [RuleID, Error]),
+                 [RuleId, Error]),
             {error, Reason};
         _:Reason = {match_incase_error, Error} ->
-            emqx_rule_metrics:inc_rules_exception(RuleID),
+            emqx_rule_metrics:inc_rules_exception(RuleId),
             ?LOG(warning, "INCASE clause exception for ~s failed: ~p",
-                 [RuleID, Error]),
+                 [RuleId, Error]),
             {error, Reason};
         _:Error:StkTrace ->
-            emqx_rule_metrics:inc_rules_exception(RuleID),
+            emqx_rule_metrics:inc_rules_exception(RuleId),
             ?LOG(error, "Apply rule ~s failed: ~p. Stacktrace:~n~p",
-                 [RuleID, Error, StkTrace]),
+                 [RuleId, Error, StkTrace]),
             {error, {Error, StkTrace}}
     end.
 
@@ -245,9 +247,10 @@ number(Bin) ->
     catch error:badarg -> binary_to_float(Bin)
     end.
 
-%% Step3 -> Take actions
+%% %% Step3 -> Take actions
+%% fallback actions already have `rule_id` in `metadata`
 take_actions(Actions, Selected, Envs, OnFailed) ->
-    [take_action(ActInst, Selected, Envs, OnFailed, ?ActionMaxRetry)
+    [take_action(ActInst, Selected, emqx_rule_utils:add_metadata(Envs, ActInst), OnFailed, ?ActionMaxRetry)
      || ActInst <- Actions].
 
 take_action(#action_instance{id = Id, name = ActName, fallbacks = Fallbacks} = ActInst,
@@ -312,12 +315,12 @@ wait_action_on(Id, RetryN) ->
             end
     end.
 
-handle_action_failure(continue, Id, Fallbacks, Selected, Envs, Reason) ->
-    ?LOG(error, "Take action ~p failed, continue next action, reason: ~0p", [Id, Reason]),
+handle_action_failure(continue, _Id, Fallbacks, Selected, Envs = #{metadata := Metadata}, Reason) ->
+    ?LOG_RULE_ACTION(error, Metadata, "Continue next action, reason: ~0p", [Reason]),
     _ = take_actions(Fallbacks, Selected, Envs, continue),
     failed;
-handle_action_failure(stop, Id, Fallbacks, Selected, Envs, Reason) ->
-    ?LOG(error, "Take action ~p failed, skip all actions, reason: ~0p", [Id, Reason]),
+handle_action_failure(stop, Id, Fallbacks, Selected, Envs = #{metadata := Metadata}, Reason) ->
+    ?LOG_RULE_ACTION(error, Metadata, "Skip all actions, reason: ~0p", [Reason]),
     _ = take_actions(Fallbacks, Selected, Envs, continue),
     error({take_action_failed, {Id, Reason}}).
 
@@ -429,10 +432,6 @@ do_apply_func(Name, Args, Input) ->
         Result -> Result
     end.
 
-add_metadata(Input, Metadata) when is_map(Input), is_map(Metadata) ->
-    NewMetadata = maps:merge(maps:get(metadata, Input, #{}), Metadata),
-    Input#{metadata => NewMetadata}.
-
 %%------------------------------------------------------------------------------
 %% Internal Functions
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl
index 137e22128..8bd46a958 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl
@@ -16,6 +16,9 @@
 
 -module(emqx_rule_utils).
 
+-include("rule_engine.hrl").
+-include_lib("emqx/include/logger.hrl").
+
 -export([ replace_var/2
         ]).
 
@@ -59,6 +62,10 @@
         , can_topic_match_oneof/2
         ]).
 
+-export([ add_metadata/2
+        , log_action/4
+        ]).
+
 -compile({no_auto_import,
           [ float/1
           ]}).
@@ -371,3 +378,30 @@ can_topic_match_oneof(Topic, Filters) ->
     lists:any(fun(Fltr) ->
         emqx_topic:match(Topic, Fltr)
     end, Filters).
+
+add_metadata(Envs, Metadata) when is_map(Envs), is_map(Metadata) ->
+    NMetadata = maps:merge(maps:get(metadata, Envs, #{}), Metadata),
+    Envs#{metadata => NMetadata};
+add_metadata(Envs, Action) when is_map(Envs), is_record(Action, action_instance)->
+    Metadata = gen_metadata_from_action(Action),
+    NMetadata = maps:merge(maps:get(metadata, Envs, #{}), Metadata),
+    Envs#{metadata => NMetadata}.
+
+gen_metadata_from_action(#action_instance{name = Name, args = undefined}) ->
+    #{action_name => Name, resource_id => undefined};
+gen_metadata_from_action(#action_instance{name = Name, args = Args})
+  when is_map(Args) ->
+    #{action_name => Name, resource_id => maps:get(<<"$resource">>, Args, undefined)};
+gen_metadata_from_action(#action_instance{name = Name}) ->
+    #{action_name => Name, resource_id => undefined}.
+
+log_action(Level, Metadata, Fmt, Args) ->
+    ?LOG(Level,
+         "Rule: ~p; Action: ~p; Rusource: ~p. " ++ Fmt,
+         metadata_values(Metadata) ++ Args).
+
+metadata_values(Metadata) ->
+    RuleId = maps:get(rule_id, Metadata, undefined),
+    ActionName = maps:get(action_name, Metadata, undefined),
+    ResourceName = maps:get(resource_id, Metadata, undefined),
+    [RuleId, ActionName, ResourceName].
diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl
index 00094d46d..29550e1e9 100644
--- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl
+++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl
@@ -259,25 +259,27 @@ on_action_data_to_webserver(Selected, _Envs =
                                 'BodyTokens' := BodyTokens,
                                 'Pool' := Pool,
                                 'RequestTimeout' := RequestTimeout},
-                              clientid := ClientID}) ->
+                              clientid := ClientID,
+                              metadata := Metadata}) ->
     NBody = format_msg(BodyTokens, clear_user_property_header(Selected)),
     NPath = emqx_rule_utils:proc_tmpl(PathTokens, Selected),
     Req = create_req(Method, NPath, Headers, NBody),
     case ehttpc:request({Pool, ClientID}, Method, Req, RequestTimeout) of
         {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 ->
+            ?LOG_RULE_ACTION(debug, Metadata, "HTTP Request succeeded with path: ~p status code ~p", [NPath, StatusCode]),
             emqx_rule_metrics:inc_actions_success(Id);
         {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 ->
             emqx_rule_metrics:inc_actions_success(Id);
         {ok, StatusCode, _} ->
-            ?LOG(warning, "HTTP request failed with path: ~p status code: ~p", [NPath, StatusCode]),
+            ?LOG_RULE_ACTION(warning, Metadata, "HTTP request failed with path: ~p status code: ~p", [NPath, StatusCode]),
             emqx_rule_metrics:inc_actions_error(Id),
             {badact, StatusCode};
         {ok, StatusCode, _, _} ->
-            ?LOG(warning, "HTTP request failed with path: ~p status code: ~p", [NPath, StatusCode]),
+            ?LOG_RULE_ACTION(warning, Metadata, "HTTP request failed with path: ~p status code: ~p", [NPath, StatusCode]),
             emqx_rule_metrics:inc_actions_error(Id),
             {badact, StatusCode};
         {error, Reason} ->
-            ?LOG(error, "HTTP request failed path: ~p error: ~p", [NPath, Reason]),
+            ?LOG_RULE_ACTION(error, Metadata, "HTTP request failed path: ~p error: ~p", [NPath, Reason]),
             emqx_rule_metrics:inc_actions_error(Id),
             {badact, Reason}
     end.

From 4a89dfe3622060ebfe31365c8b6681e62806693e Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 16 Aug 2022 10:53:39 +0800
Subject: [PATCH 34/52] chore: update CHANGES.md and appup.src

This reverts commit 7af25a82e70845a631be0c8b83ba7f1838d68389.
---
 CHANGES-4.3.md                                | 13 +++----
 .../src/emqx_rule_engine.appup.src            | 34 +++++++++++++++----
 apps/emqx_web_hook/src/emqx_web_hook.app.src  |  2 +-
 .../emqx_web_hook/src/emqx_web_hook.appup.src | 18 +++-------
 4 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index ab74b877a..babc8cb17 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -12,22 +12,17 @@ File format:
 
 ## v4.3.19
 
-### Bug fixes
-- Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655)
-
-## v4.3.19
-
 ### Enhancements
 
 - Improve error message for LwM2M plugin when object ID is not valid [#8654](https://github.com/emqx/emqx/pull/8654).
 - Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671)
-- Add node evacuation and cluster rebalancing features [#8597]
-
-## v4.3.19
+- Add node evacuation and cluster rebalancing features [#8597](https://github.com/emqx/emqx/pull/8597)
+- Refine Rule Engine error log. RuleId will be logged when take action failed. [#8737](https://github.com/emqx/emqx/pull/8737)
 
 ### Bug fixes
 
-- Add a idle timer for ExProto UDP client to avoid client leaking [#8628]
+- Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655)
+- Add a idle timer for ExProto UDP client to avoid client leaking [#8628](https://github.com/emqx/emqx/pull/8628)
 
 ## v4.3.18
 
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index 1c5fb0e7b..b113b2e22 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -1,10 +1,21 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.3.13",
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.12",
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
-    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
@@ -166,10 +177,21 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.13",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
-   {"4.3.12",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+  [{"4.3.13",
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
+   {"4.3.12",
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
-    [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src
index 43e28fe1a..efe41b3bb 100644
--- a/apps/emqx_web_hook/src/emqx_web_hook.app.src
+++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src
@@ -1,6 +1,6 @@
 {application, emqx_web_hook,
  [{description, "EMQ X WebHook Plugin"},
-  {vsn, "4.3.13"}, % strict semver, bump manually!
+  {vsn, "4.3.14"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_web_hook_sup]},
   {applications, [kernel,stdlib,ehttpc]},
diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src
index fff4ec08c..a2f72f0e2 100644
--- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src
+++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src
@@ -1,12 +1,7 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{<<"4\\.3\\.[0-2]">>,
-    [{apply,{application,stop,[emqx_web_hook]}},
-     {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_web_hook,brutal_purge,soft_purge,[]},
-     {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
-   {<<"4\\.3\\.[3-7]">>,
+  [{<<"4\\.3\\.[0-7]">>,
     [{apply,{application,stop,[emqx_web_hook]}},
      {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_web_hook,brutal_purge,soft_purge,[]},
@@ -26,15 +21,10 @@
    {"4.3.11",
     [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]},
-   {"4.3.12",
+   {<<"4\\.3\\.1[2-3]">>,
     [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{<<"4\\.3\\.[0-2]">>,
-    [{apply,{application,stop,[emqx_web_hook]}},
-     {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_web_hook,brutal_purge,soft_purge,[]},
-     {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
-   {<<"4\\.3\\.[3-7]">>,
+  [{<<"4\\.3\\.[0-7]">>,
     [{apply,{application,stop,[emqx_web_hook]}},
      {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_web_hook,brutal_purge,soft_purge,[]},
@@ -54,6 +44,6 @@
    {"4.3.11",
     [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]},
-   {"4.3.12",
+   {<<"4\\.3\\.1[2-3]">>,
     [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]}.

From 0502be6055f8841b5d91734110e945051b6314bc Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 17 Aug 2022 15:36:04 +0800
Subject: [PATCH 35/52] chore(typo): fix typo

---
 apps/emqx_rule_engine/src/emqx_rule_utils.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl
index 8bd46a958..5c9472367 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl
@@ -397,7 +397,7 @@ gen_metadata_from_action(#action_instance{name = Name}) ->
 
 log_action(Level, Metadata, Fmt, Args) ->
     ?LOG(Level,
-         "Rule: ~p; Action: ~p; Rusource: ~p. " ++ Fmt,
+         "Rule: ~p; Action: ~p; Resource: ~p. " ++ Fmt,
          metadata_values(Metadata) ++ Args).
 
 metadata_values(Metadata) ->

From 768ab4eacdd60d2edb1e1c6ec1958ec2a91c54fb Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 17 Aug 2022 16:03:42 +0800
Subject: [PATCH 36/52] fix(bridge): mqtt bridge worker status idle

---
 .../src/emqx_bridge_mqtt.app.src              |  2 +-
 .../src/emqx_bridge_mqtt.appup.src            |  4 ++--
 .../src/emqx_bridge_mqtt_actions.erl          | 23 +++++++++++++++----
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
index 31c795ff5..578671f4e 100644
--- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
+++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
@@ -1,6 +1,6 @@
 {application, emqx_bridge_mqtt,
  [{description, "EMQ X Bridge to MQTT Broker"},
-  {vsn, "4.3.5"}, % strict semver, bump manually!
+  {vsn, "4.3.6"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,replayq,emqtt]},
diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src
index 9e1eda7f4..a72a18658 100644
--- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src
+++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.4",
+  [{<<"4\\.3\\.[4-5]">>,
     [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
     [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]},
@@ -14,7 +14,7 @@
      {load_module,emqx_bridge_worker,brutal_purge,soft_purge,[]},
      {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
-  [{"4.3.4",
+  [{<<"4\\.3\\.[4-5]">>,
     [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
     [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]},
diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl
index 941ad51d0..a2b88352e 100644
--- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl
+++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl
@@ -433,7 +433,7 @@ test_resource_status(PoolName) ->
     try
         Status = [
             receive {Pid, R} -> R
-            after 1000 -> %% get_worker_status/1 should be a quick operation
+            after 10000 -> %% get_worker_status/1 should be a quick operation
                 throw({timeout, Pid})
             end || Pid <- Pids],
         lists:any(fun(St) -> St =:= true end, Status)
@@ -444,13 +444,28 @@ test_resource_status(PoolName) ->
             false
     end.
 
+-define(RETRY_TIMES, 4).
+
 get_worker_status(Worker) ->
+    get_worker_status(Worker, ?RETRY_TIMES).
+
+get_worker_status(_Worker, 0) ->
+    false;
+get_worker_status(Worker, Times) ->
     case ecpool_worker:client(Worker) of
         {ok, Bridge} ->
             try emqx_bridge_worker:status(Bridge) of
-                connected -> true;
-                _ -> false
-            catch _Error:_Reason ->
+                connected ->
+                    true;
+                idle ->
+                    ?LOG(info, "MQTT Bridge get status idle. Should not ignore this."),
+                    timer:sleep(100),
+                    get_worker_status(Worker, Times - 1);
+                ErrorStatus ->
+                    ?LOG(error, "MQTT Bridge get status ~p", [ErrorStatus]),
+                    false
+            catch Error:Reason:ST ->
+                    ?LOG(error, "MQTT Bridge get status error: ~p reason: ~p stacktrace: ~p", [Error, Reason, ST]),
                     false
             end;
         {error, _} ->

From 58e24c2fde0f08bdeadeb23e5533e5a3a2a8cb58 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 17 Aug 2022 16:27:52 +0800
Subject: [PATCH 37/52] chore: bump eredis to 1.2.8 to fix reconnect port leak

---
 rebar.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rebar.config b/rebar.config
index daddc8b97..799631dc7 100644
--- a/rebar.config
+++ b/rebar.config
@@ -41,7 +41,7 @@
     [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
     , {redbug, "2.0.7"}
     , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.2.0"}}}
-    , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.2"}}}
+    , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.3"}}}
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}

From 2a50daa98b4ac7d972d2e5a9ade53cfbd46d6c52 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 17 Aug 2022 11:45:53 +0200
Subject: [PATCH 38/52] chore: sync ee emqx.appup.src to ce

main-v4.3-enterprise had some ee specific changes added to emqx.erl
having emqx module added to reload even there is no change is OK
so we can simply sync the file from ee to ce
---
 src/emqx.appup.src | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index b2be51df5..fe65271fc 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -2,17 +2,18 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.3.19",
-    [{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.18",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
@@ -22,6 +23,7 @@
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
     [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {update,emqx_broker_sup,supervisor},
@@ -39,6 +41,7 @@
      {load_module,emqx_topic,brutal_purge,soft_purge,[]}]},
    {"4.3.15",
     [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {add_module,emqx_calendar},
      {load_module,emqx_topic,brutal_purge,soft_purge,[]},
      {add_module,emqx_exclusive_subscription},
@@ -693,17 +696,18 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.3.19",
-    [{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]}]},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.18",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
@@ -713,6 +717,7 @@
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
     [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
      {update,emqx_broker_sup,supervisor},
@@ -730,6 +735,7 @@
      {delete_module,emqx_exclusive_subscription}]},
    {"4.3.15",
     [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+     {load_module,emqx,brutal_purge,soft_purge,[]},
      {delete_module,emqx_calendar},
      {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
      {delete_module,emqx_exclusive_subscription},

From 719f6cfb2c5ad43a38a04036187afcc2220276ca Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Tue, 16 Aug 2022 10:48:48 +0800
Subject: [PATCH 39/52] fix(jwt): change `request_jwks` to be called after
 initialization

If called in `init/1`, the module-enabled API may have inconsistent state due to timeout
---
 apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl | 60 ++++++++++++--------
 1 file changed, 35 insertions(+), 25 deletions(-)

diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl
index 08c60d8ed..ac07a8640 100644
--- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl
+++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl
@@ -73,8 +73,9 @@ verify(JwsCompacted) when is_binary(JwsCompacted) ->
 init([Options]) ->
     ok = jose:json_module(jiffy),
     _ = ets:new(?TAB, [set, protected, named_table]),
-    {Static, Remote} = do_init_jwks(Options),
-    true = ets:insert(?TAB, [{static, Static}, {remote, Remote}]),
+    Static = do_init_jwks(Options),
+    to_request_jwks(Options),
+    true = ets:insert(?TAB, [{static, Static}, {remote, undefined}]),
     Intv = proplists:get_value(interval, Options, ?INTERVAL),
     {ok, reset_timer(
            #state{
@@ -83,29 +84,13 @@ init([Options]) ->
 
 %% @private
 do_init_jwks(Options) ->
-    K2J = fun(K, F) ->
-              case proplists:get_value(K, Options) of
-                  undefined -> undefined;
-                  V ->
-                     try F(V) of
-                         {error, Reason} ->
-                             ?LOG(warning, "Build ~p JWK ~p failed: {error, ~p}~n",
-                                  [K, V, Reason]),
-                             undefined;
-                         J -> J
-                     catch T:R ->
-                         ?LOG(warning, "Build ~p JWK ~p failed: {~p, ~p}~n",
-                              [K, V, T, R]),
-                         undefined
-                     end
-              end
-          end,
-    OctJwk = K2J(secret, fun(V) ->
-                             jose_jwk:from_oct(list_to_binary(V))
-                         end),
-    PemJwk = K2J(pubkey, fun jose_jwk:from_pem_file/1),
-    Remote = K2J(jwks_addr, fun request_jwks/1),
-    {[J ||J <- [OctJwk, PemJwk], J /= undefined], Remote}.
+    OctJwk = key2jwt_value(secret,
+                           fun(V) ->
+                                   jose_jwk:from_oct(list_to_binary(V))
+                           end,
+                           Options),
+    PemJwk = key2jwt_value(pubkey, fun jose_jwk:from_pem_file/1, Options),
+    [J ||J <- [OctJwk, PemJwk], J /= undefined].
 
 handle_call(_Req, _From, State) ->
     {reply, ok, State}.
@@ -122,6 +107,11 @@ handle_info({timeout, _TRef, refresh}, State = #state{addr = Addr}) ->
              end,
     {noreply, reset_timer(NState)};
 
+handle_info({request_jwks, Options}, State) ->
+    Remote = key2jwt_value(jwks_addr, fun request_jwks/1, Options),
+    true = ets:insert(?TAB, {remote, Remote}),
+    {noreply, State};
+
 handle_info(_Info, State) ->
     {noreply, State}.
 
@@ -249,3 +239,23 @@ do_check_claim([{K, F}|More], Claims) ->
         _ ->
             do_check_claim(More, Claims)
     end.
+
+to_request_jwks(Options) ->
+    erlang:send(self(), {request_jwks, Options}).
+
+key2jwt_value(Key, Func, Options) ->
+    case proplists:get_value(Key, Options) of
+        undefined -> undefined;
+        V ->
+            try Func(V) of
+                {error, Reason} ->
+                    ?LOG(warning, "Build ~p JWK ~p failed: {error, ~p}~n",
+                         [Key, V, Reason]),
+                    undefined;
+                J -> J
+            catch T:R ->
+                    ?LOG(warning, "Build ~p JWK ~p failed: {~p, ~p}~n",
+                         [Key, V, T, R]),
+                    undefined
+            end
+    end.

From b9d26506a64149a989f0b46728011ef1810f164d Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Tue, 16 Aug 2022 10:55:24 +0800
Subject: [PATCH 40/52] chore(jwt): bump version && update appup

---
 apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src   | 2 +-
 apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src
index 32ca1fc14..72e6e749b 100644
--- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src
+++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src
@@ -1,6 +1,6 @@
 {application, emqx_auth_jwt,
  [{description, "EMQ X Authentication with JWT"},
-  {vsn, "4.3.4"}, % strict semver, bump manually!
+  {vsn, "4.3.5"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_jwt_sup]},
   {applications, [kernel,stdlib,jose]},
diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src
index 2538187fd..2364f901e 100644
--- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src
+++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src
@@ -1,9 +1,11 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"4.3.3",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+  [{"4.3.4",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]},
    {<<".*">>,[]}],
-  [{"4.3.3",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+  [{"4.3.4",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
    {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]},
    {<<".*">>,[]}]}.

From 1bcc58e2987acb8ccadfe486bbc7dd8abc32b13d Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Wed, 17 Aug 2022 17:59:16 +0800
Subject: [PATCH 41/52] chore: update CHANGES-4.3.md

---
 CHANGES-4.3.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index babc8cb17..2eb3d93a1 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -18,6 +18,7 @@ File format:
 - Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671)
 - Add node evacuation and cluster rebalancing features [#8597](https://github.com/emqx/emqx/pull/8597)
 - Refine Rule Engine error log. RuleId will be logged when take action failed. [#8737](https://github.com/emqx/emqx/pull/8737)
+- Improved jwt authentication module initialization process.[#8736](https://github.com/emqx/emqx/pull/8736)
 
 ### Bug fixes
 

From e1615e86df2741141ca41d86018252da34643fd1 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 17 Aug 2022 18:14:29 +0800
Subject: [PATCH 42/52] chore: update the change log

---
 CHANGES-4.3.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index ab74b877a..6023772a3 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -13,7 +13,12 @@ File format:
 ## v4.3.19
 
 ### Bug fixes
+
 - Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655)
+- Fix rule SQL compare to null values always returns false. [#8743](https://github.com/emqx/emqx/pull/8743)
+  Before this change, the following SQL failed to match on the WHERE clause (`clientid != foo` returns false):
+  `SELECT 'some_var' as clientid FROM "t" WHERE clientid != foo`.
+  The `foo` variable is a null value, so `clientid != foo` should be evaluated as true.
 
 ## v4.3.19
 

From 1d43c11e5b6ce39bced1048a6c49fa7ef3bc75be Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 17 Aug 2022 20:27:41 +0800
Subject: [PATCH 43/52] chore: bump to v4.3.19-beta.1

---
 include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl
index 305815de1..404532b87 100644
--- a/include/emqx_release.hrl
+++ b/include/emqx_release.hrl
@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.3.18"}).
+-define(EMQX_RELEASE, {opensource, "4.3.19-beta.1"}).
 
 -else.
 

From 117348fb858d2ca368751fe44b60ace8c10d09a4 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 17 Aug 2022 22:05:22 +0800
Subject: [PATCH 44/52] feat(rebalance): remove unannounced plugins from plugin
 lists

---
 src/emqx_plugins.erl | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl
index 8b5b7ffb2..97ec9a88a 100644
--- a/src/emqx_plugins.erl
+++ b/src/emqx_plugins.erl
@@ -235,9 +235,7 @@ default_plugins() ->
         {emqx_recon, true},
         {emqx_telemetry, true},
         {emqx_rule_engine, true},
-        {emqx_bridge_mqtt, false},
-        {emqx_eviction_agent, true},
-        {emqx_node_rebalance, true}
+        {emqx_bridge_mqtt, false}
     ].
 
 -else.

From a583c221f0abacda5dea9c9069462804584981dc Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 17 Aug 2022 22:51:02 +0800
Subject: [PATCH 45/52] fix: list_listener crash with invaild json

---
 CHANGES-4.3.md                                       | 3 ++-
 apps/emqx_management/src/emqx_mgmt_api_listeners.erl | 3 +--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md
index babc8cb17..ea45e0d69 100644
--- a/CHANGES-4.3.md
+++ b/CHANGES-4.3.md
@@ -22,7 +22,8 @@ File format:
 ### Bug fixes
 
 - Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655)
-- Add a idle timer for ExProto UDP client to avoid client leaking [#8628](https://github.com/emqx/emqx/pull/8628)
+- Add an idle timer for ExProto UDP client to avoid client leaking [#8628](https://github.com/emqx/emqx/pull/8628)
+- Fix GET `/listeners/` crashes when listener is not ready. [#8752](https://github.com/emqx/emqx/pull/8752)
 
 ## v4.3.18
 
diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl
index 461c91d84..10ad9f4d4 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl
@@ -72,5 +72,4 @@ format(Listeners) when is_list(Listeners) ->
     [ Info#{listen_on => list_to_binary(esockd:to_string(ListenOn))}
      || Info = #{listen_on := ListenOn} <- Listeners ];
 
-format({error, Reason}) -> [{error, Reason}].
-
+format({error, Reason}) -> [{error, iolist_to_binary(io_lib:format("~p", [Reason]))}].

From 8a78c8a2f548d40d9478e1958c3135bafa725943 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 17 Aug 2022 23:37:50 +0800
Subject: [PATCH 46/52] chore: add emqx_app to 4.3.19

---
 src/emqx.appup.src | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index fe65271fc..5b6b88e61 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -2,7 +2,8 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.3.19",
-    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.18",
@@ -696,7 +697,8 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.3.19",
-    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.18",

From 10cabddb0793599ba4457cbfea0ffb2342e1b4ce Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 17 Aug 2022 16:26:36 +0200
Subject: [PATCH 47/52] ci: update OTP version

---
 .ci/build_packages/Dockerfile                  | 2 +-
 .ci/docker-compose-file/docker-compose.yaml    | 2 +-
 .github/workflows/apps_version_check.yaml      | 2 +-
 .github/workflows/build_packages.yaml          | 4 ++--
 .github/workflows/build_slim_packages.yaml     | 4 ++--
 .github/workflows/check_deps_integrity.yaml    | 2 +-
 .github/workflows/release.yaml                 | 2 +-
 .github/workflows/run_acl_migration_tests.yaml | 2 +-
 .github/workflows/run_fvt_tests.yaml           | 8 ++++----
 .github/workflows/run_test_cases.yaml          | 2 +-
 .tool-versions                                 | 2 +-
 Makefile                                       | 2 +-
 Windows.md                                     | 2 +-
 13 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile
index 3c7e401ae..b56cb5220 100644
--- a/.ci/build_packages/Dockerfile
+++ b/.ci/build_packages/Dockerfile
@@ -1,4 +1,4 @@
-ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+ARG BUILD_FROM=emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 FROM ${BUILD_FROM}
 
 ARG EMQX_NAME=emqx
diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml
index 6b5c1b602..918be18fd 100644
--- a/.ci/docker-compose-file/docker-compose.yaml
+++ b/.ci/docker-compose-file/docker-compose.yaml
@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+    image: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
     env_file:
       - conf.env
     environment:
diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml
index cb2c7a26b..86fe60204 100644
--- a/.github/workflows/apps_version_check.yaml
+++ b/.github/workflows/apps_version_check.yaml
@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - erl23.2.7.2-emqx-3
+        - erl23.3.4.9-3
         os:
         - ubuntu20.04
 
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 19ed368a6..2571746bd 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -18,7 +18,7 @@ on:
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
@@ -125,7 +125,7 @@ jobs:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         erl_otp:
-          - 23.2.7.2-emqx-3
+          - 23.3.4.9-3
         exclude:
           - profile: emqx-edge
         macos:
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 713ccc605..17e99c831 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - erl23.2.7.2-emqx-3
+        - erl23.3.4.9-3
         os:
         - ubuntu20.04
         - centos7
@@ -55,7 +55,7 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - 23.2.7.2-emqx-3
+        - 23.3.4.9-3
         macos:
         - macos-11
     runs-on: ${{ matrix.macos }}
diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml
index e58afcc1a..07ec47754 100644
--- a/.github/workflows/check_deps_integrity.yaml
+++ b/.github/workflows/check_deps_integrity.yaml
@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 8e6b02c21..bdbd7e982 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -7,7 +7,7 @@ on:
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml
index 0f001ac04..bbaed018f 100644
--- a/.github/workflows/run_acl_migration_tests.yaml
+++ b/.github/workflows/run_acl_migration_tests.yaml
@@ -5,7 +5,7 @@ on: workflow_dispatch
 jobs:
     test:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+        container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
         strategy:
             fail-fast: true
         env:
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index ef76e8108..601ab304f 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -15,7 +15,7 @@ jobs:
         - uses: actions/checkout@v1
         - uses: erlef/setup-beam@v1
           with:
-            otp-version: "23.2"
+            otp-version: "23.3.4.9"
         - name: make docker
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
@@ -67,7 +67,7 @@ jobs:
         - uses: actions/checkout@v1
         - uses: erlef/setup-beam@v1
           with:
-            otp-version: "23.2"
+            otp-version: "23.3.4.9"
         - name: prepare
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
@@ -176,7 +176,7 @@ jobs:
 
     relup_test_plan:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+        container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
         outputs:
           profile: ${{ steps.profile-and-versions.outputs.profile }}
           vsn: ${{ steps.profile-and-versions.outputs.vsn }}
@@ -222,7 +222,7 @@ jobs:
     relup_test_build:
         needs: relup_test_plan
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+        container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
         defaults:
           run:
             shell: bash
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 6110c587c..b7869c74c 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -10,7 +10,7 @@ on:
 jobs:
     run_proper_test:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+        container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 
         steps:
         - uses: actions/checkout@v2
diff --git a/.tool-versions b/.tool-versions
index 0b6392b95..757309f18 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1 +1 @@
-erlang 23.2.7.2-emqx-3
+erlang 23.3.4.9-3
diff --git a/Makefile b/Makefile
index 0fb5c63e5..f7f46c48f 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export EMQX_RELUP ?= true
-export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine
+export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.3.4.9-3-alpine
 export EMQX_DEFAULT_RUNNER = alpine:3.12
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 export DOCKERFILE := deploy/docker/Dockerfile
diff --git a/Windows.md b/Windows.md
index 6b6eff60a..ff49365e0 100644
--- a/Windows.md
+++ b/Windows.md
@@ -29,7 +29,7 @@ The second path is for CMD to setup environment variables.
 
 ### Erlang/OTP
 
-Install Erlang/OTP 23.2 from https://www.erlang.org/downloads
+Install Erlang/OTP 23.3 from https://www.erlang.org/downloads
 You may need to edit the `Path` environment variable to allow running
 Erlang commands such as `erl` from CMD.
 

From a645a8870099b7917a9caced1d17e260058cab6f Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 17 Aug 2022 17:01:23 +0200
Subject: [PATCH 48/52] ci: skip relup test from 4.3.13

---
 .ci/build_packages/tests.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh
index d4620e2d3..93844b3e2 100755
--- a/.ci/build_packages/tests.sh
+++ b/.ci/build_packages/tests.sh
@@ -139,6 +139,10 @@ relup_test(){
 
         find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" |
             while read -r pkg; do
+                if [[ "${pkg}" == *4.3.13* ]]; then
+                    echo "skipping upgrade test from 4.3.13 because this release had crypto linked with openssl 1.1.1n, it was in later version rolled back (to default 1.1.1k)."
+                    continue
+                fi
                 packagename=$(basename "${pkg}")
                 unzip -q "$packagename"
                 ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )

From f1ad4cb8cab7f9c8aec6ed852856f1eefe02c821 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 17 Aug 2022 23:40:48 +0200
Subject: [PATCH 49/52] chore: fix emqx_misc ipv6 probe

The new OTP version exports gen_tcp:ipv6_probe/0 which always return
true. So we no longer need the old hacky way to do it.
---
 src/emqx.appup.src | 24 ++++++++++++++++--------
 src/emqx_misc.erl  | 13 +++----------
 2 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/emqx.appup.src b/src/emqx.appup.src
index 5b6b88e61..27c7d3e1a 100644
--- a/src/emqx.appup.src
+++ b/src/emqx.appup.src
@@ -2,17 +2,20 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.3.19",
-    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.18",
-    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
-    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
@@ -23,7 +26,8 @@
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
-    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
@@ -697,17 +701,20 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.3.19",
-    [{load_module,emqx_app,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.3.18",
-    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.3.17",
-    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_exclusive_subscription,brutal_purge,soft_purge,[]},
@@ -718,7 +725,8 @@
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]},
    {"4.3.16",
-    [{load_module,emqx_cm,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_misc,brutal_purge,soft_purge,[]},
+     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl
index 36a3386e3..813ab84e8 100644
--- a/src/emqx_misc.erl
+++ b/src/emqx_misc.erl
@@ -84,16 +84,9 @@ maybe_parse_ip(Host) ->
 
 %% @doc Add `ipv6_probe' socket option if it's supported.
 ipv6_probe(Opts) ->
-    case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of
-        unknown ->
-            %% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64
-            OtpVsn = emqx_vm:get_otp_version(),
-            Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])),
-            _ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool),
-            ipv6_probe(Bool, Opts);
-        Bool ->
-            ipv6_probe(Bool, Opts)
-    end.
+    Bool = try gen_tcp:ipv6_probe()
+           catch _ : _ -> false end,
+    ipv6_probe(Bool, Opts).
 
 ipv6_probe(false, Opts) -> Opts;
 ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts].

From 413612a69de1a8ad8bdcee773775c388b103269c Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Thu, 18 Aug 2022 09:18:41 +0800
Subject: [PATCH 50/52] fix: duplicate appup instructions

---
 .../src/emqx_rule_engine.appup.src            |  1 -
 .../test/emqx_rule_engine_SUITE.erl           | 28 ++++++++++++++++++-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index ff2ddc4c9..f50c1a368 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -190,7 +190,6 @@
    {"4.3.11",
     [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
index e1851b76c..d1e6b26a0 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
@@ -126,6 +126,7 @@ groups() ->
        t_sqlparse_array_range_1,
        t_sqlparse_array_range_2,
        t_sqlparse_true_false,
+       t_sqlparse_compare_undefined,
        t_sqlparse_compare_null_null,
        t_sqlparse_compare_null_notnull,
        t_sqlparse_compare_notnull_null,
@@ -2493,8 +2494,33 @@ t_sqlparse_true_false(_Config) ->
 -define(TEST_SQL(SQL),
     emqx_rule_sqltester:test(
         #{<<"rawsql">> => SQL,
-          <<"ctx">> => #{<<"payload">> => <<"">>,
+          <<"ctx">> => #{<<"payload">> => <<"{}">>,
                          <<"topic">> => <<"t/a">>}})).
+
+t_sqlparse_compare_undefined(_Config) ->
+    Sql00 = "select "
+            " * "
+            "from \"t/#\" "
+            "where dev != undefined ",
+    %% no match
+    ?assertMatch({error, nomatch}, ?TEST_SQL(Sql00)),
+
+    Sql01 = "select "
+            " 'd' as dev "
+            "from \"t/#\" "
+            "where dev != undefined ",
+    {ok, Res01} = ?TEST_SQL(Sql01),
+    %% pass
+    ?assertMatch(#{}, Res01),
+
+    Sql02 = "select "
+            " * "
+            "from \"t/#\" "
+            "where dev != 'undefined' ",
+    {ok, Res02} = ?TEST_SQL(Sql02),
+    %% pass
+    ?assertMatch(#{}, Res02).
+
 t_sqlparse_compare_null_null(_Config) ->
     %% test undefined == undefined
     Sql00 = "select "

From da05a2ad305f8839999172e58e454eaf2b9d39ff Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 18 Aug 2022 07:30:19 +0200
Subject: [PATCH 51/52] test: delete unknown plugin

---
 test/emqx_plugins_SUITE.erl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl
index eef5152b0..f3c37afea 100644
--- a/test/emqx_plugins_SUITE.erl
+++ b/test/emqx_plugins_SUITE.erl
@@ -94,10 +94,8 @@ default_plugins() ->
     [
         {emqx_bridge_mqtt, false},
         {emqx_dashboard, true},
-        {emqx_eviction_agent, true},
         {emqx_management, true},
         {emqx_modules, false},
-        {emqx_node_rebalance, true},
         {emqx_recon, true},
         {emqx_retainer, true},
         {emqx_rule_engine, true},

From b4ea2aefa6e6841c873fbb5354eca2ded8cea7fa Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 18 Aug 2022 07:35:40 +0200
Subject: [PATCH 52/52] test: export ct callbacks

---
 .../test/emqx_auth_mnesia_data_export_import_SUITE.erl      | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl
index 021cf735a..da27fa2ee 100644
--- a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl
+++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl
@@ -70,4 +70,10 @@ remove_all_users_and_acl() ->
     mnesia:delete_table(emqx_user),
     mnesia:delete_table(emqx_acl).
 
+-else.
+
+%% opensource edition
+
+all() -> [].
+
 -endif.