From ca2660d60977d949aa6ee3312719070543ff47e7 Mon Sep 17 00:00:00 2001
From: Spycsh <757407490@qq.com>
Date: Fri, 17 Dec 2021 17:19:40 +0100
Subject: [PATCH 01/25] chore: refactor ?SLOG

---
 apps/emqx/src/emqx_channel.erl                        |  2 +-
 .../src/emqx_limiter/src/emqx_limiter_manager.erl     |  6 +++---
 .../emqx/src/emqx_limiter/src/emqx_limiter_server.erl | 10 +++++-----
 apps/emqx/src/emqx_trace/emqx_trace_handler.erl       |  8 ++++----
 apps/emqx_management/src/emqx_mgmt_api_trace.erl      |  2 +-
 apps/emqx_modules/src/emqx_delayed.erl                |  8 ++++----
 apps/emqx_modules/src/emqx_telemetry.erl              |  8 ++++----
 apps/emqx_modules/src/emqx_topic_metrics.erl          |  4 ++--
 apps/emqx_retainer/src/emqx_retainer.erl              |  6 +++---
 apps/emqx_retainer/src/emqx_retainer_mnesia.erl       | 11 +++++------
 apps/emqx_retainer/src/emqx_retainer_pool.erl         |  8 ++++----
 apps/emqx_slow_subs/src/emqx_slow_subs.erl            |  6 +++---
 12 files changed, 39 insertions(+), 40 deletions(-)

diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index eb71aca58..d167f3924 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -292,7 +292,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
                    fun check_banned/2
                   ], ConnPkt, Channel#channel{conn_state = connecting}) of
         {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
-            ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]),
+            ?SLOG(debug, #{msg => "recv_packet", packet => emqx_packet:format(Packet)}),
             NChannel1 = NChannel#channel{
                                         will_msg = emqx_packet:will_msg(NConnPkt),
                                         alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl
index 3d46590c5..fe2a03e46 100644
--- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl
+++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl
@@ -138,7 +138,7 @@ init([]) ->
           {stop, Reason :: term(), Reply :: term(), NewState :: term()} |
           {stop, Reason :: term(), NewState :: term()}.
 handle_call(Req, _From, State) ->
-    ?LOG(error, "Unexpected call: ~p", [Req]),
+    ?SLOG(error, #{msg => "unexpected_call", call => Req}),
     {reply, ignore, State}.
 
 %%--------------------------------------------------------------------
@@ -153,7 +153,7 @@ handle_call(Req, _From, State) ->
           {noreply, NewState :: term(), hibernate} |
           {stop, Reason :: term(), NewState :: term()}.
 handle_cast(Req, State) ->
-    ?LOG(error, "Unexpected cast: ~p", [Req]),
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Req}),
     {noreply, State}.
 
 %%--------------------------------------------------------------------
@@ -168,7 +168,7 @@ handle_cast(Req, State) ->
           {noreply, NewState :: term(), hibernate} |
           {stop, Reason :: normal | term(), NewState :: term()}.
 handle_info(Info, State) ->
-    ?LOG(error, "Unexpected info: ~p", [Info]),
+    ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl
index 799d623bf..1727e4608 100644
--- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl
+++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl
@@ -98,7 +98,7 @@ connect(Type, BucketName) when is_atom(BucketName) ->
     Path = [emqx_limiter, Type, bucket, BucketName],
     case emqx:get_config(Path, undefined) of
         undefined ->
-            ?LOG(error, "can't find the config of this bucket: ~p~n", [Path]),
+            ?SLOG(error, #{msg => "bucket_config_not_found", path => Path}),
             throw("bucket's config not found");
         #{zone := Zone,
           aggregated := #{rate := AggrRate, capacity := AggrSize},
@@ -113,7 +113,7 @@ connect(Type, BucketName) when is_atom(BucketName) ->
                             emqx_htb_limiter:make_ref_limiter(Cfg, Bucket)
                     end;
                 undefined ->
-                    ?LOG(error, "can't find the bucket:~p~n", [Path]),
+                    ?SLOG(error, #{msg => "bucket_not_found", path => Path}),
                     throw("invalid bucket")
             end
     end;
@@ -182,7 +182,7 @@ init([Type]) ->
           {stop, Reason :: term(), Reply :: term(), NewState :: term()} |
           {stop, Reason :: term(), NewState :: term()}.
 handle_call(Req, _From, State) ->
-    ?LOG(error, "Unexpected call: ~p", [Req]),
+    ?SLOG(error, #{msg => "unexpected_call", call => Req}),
     {reply, ignored, State}.
 
 %%--------------------------------------------------------------------
@@ -197,7 +197,7 @@ handle_call(Req, _From, State) ->
           {noreply, NewState :: term(), hibernate} |
           {stop, Reason :: term(), NewState :: term()}.
 handle_cast(Req, State) ->
-    ?LOG(error, "Unexpected cast: ~p", [Req]),
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Req}),
     {noreply, State}.
 
 %%--------------------------------------------------------------------
@@ -215,7 +215,7 @@ handle_info(oscillate, State) ->
     {noreply, oscillation(State)};
 
 handle_info(Info, State) ->
-    ?LOG(error, "Unexpected info: ~p", [Info]),
+    ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
index c76bf1aa9..4aaa42003 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
@@ -103,7 +103,7 @@ uninstall(Type, Name) ->
 -spec uninstall(HandlerId :: atom()) -> ok | {error, term()}.
 uninstall(HandlerId) ->
     Res = logger:remove_handler(HandlerId),
-    show_prompts(Res, HandlerId, "Stop trace"),
+    show_prompts(Res, HandlerId, "stop_trace"),
     Res.
 
 %% @doc Return all running trace handlers information.
@@ -151,7 +151,7 @@ install_handler(Who = #{name := Name, type := Type}, Level, LogFile) ->
         config => ?CONFIG(LogFile)
     },
     Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
-    show_prompts(Res, Who, "Start trace"),
+    show_prompts(Res, Who, "start_trace"),
     Res.
 
 filters(#{type := clientid, filter := Filter, name := Name}) ->
@@ -223,6 +223,6 @@ ensure_list(Bin) when is_binary(Bin) -> unicode:characters_to_list(Bin, utf8);
 ensure_list(List) when is_list(List) -> List.
 
 show_prompts(ok, Who, Msg) ->
-    ?LOG(info, Msg ++ " ~p " ++ "successfully~n", [Who]);
+    ?SLOG(info, #{msg => "trace_action_succeeded", action => Msg, traced => Who});
 show_prompts({error, Reason}, Who, Msg) ->
-    ?LOG(error, Msg ++ " ~p " ++ "failed with ~p~n", [Who, Reason]).
+    ?SLOG(info, #{msg => "trace_action_failed", action => Msg, traced => Who, reason => Reason}).
diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
index d6902d123..0fa086d98 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
@@ -310,7 +310,7 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) ->
                     _ -> Acc
                 end;
             {error, Node, Reason} ->
-                ?LOG(error, "download trace log error:~p", [{Node, TraceLog, Reason}]),
+                ?SLOG(error, #{msg => "download_trace_log_error", node => Node, log => TraceLog, reason => Reason}),
                 Acc
         end
                 end, [], TraceFiles).
diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl
index 36569d52e..c8d76f9e3 100644
--- a/apps/emqx_modules/src/emqx_delayed.erl
+++ b/apps/emqx_modules/src/emqx_delayed.erl
@@ -97,7 +97,7 @@ on_message_publish(Msg = #message{
     case store(#delayed_message{key = {PubAt, Id}, delayed = Delayed, msg = PubMsg}) of
         ok -> ok;
         {error, Error} ->
-            ?LOG(error, "Store delayed message fail: ~p", [Error])
+            ?SLOG(error, #{msg => "store_delayed_message_fail", error => Error})
     end,
     {stop, PubMsg#message{headers = Headers#{allow_publish => false}}};
 
@@ -230,11 +230,11 @@ handle_call(disable, _From, State) ->
     {reply, ok, State};
 
 handle_call(Req, _From, State) ->
-    ?LOG(error, "Unexpected call: ~p", [Req]),
+    ?SLOG(error, #{msg => "unexpected_call", call => Req}),
     {reply, ignored, State}.
 
 handle_cast(Msg, State) ->
-    ?LOG(error, "Unexpected cast: ~p", [Msg]),
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
     {noreply, State}.
 
 %% Do Publish...
@@ -248,7 +248,7 @@ handle_info(stats, State = #{stats_fun := StatsFun}) ->
     {noreply, State, hibernate};
 
 handle_info(Info, State) ->
-    ?LOG(error, "Unexpected info: ~p", [Info]),
+    ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
 
 terminate(_Reason, #{timer := TRef}) ->
diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl
index 932af8005..195b45a3d 100644
--- a/apps/emqx_modules/src/emqx_telemetry.erl
+++ b/apps/emqx_modules/src/emqx_telemetry.erl
@@ -173,15 +173,15 @@ handle_call(get_telemetry, _From, State) ->
     {reply, {ok, get_telemetry(State)}, State};
 
 handle_call(Req, _From, State) ->
-    ?LOG(error, "Unexpected call: ~p", [Req]),
+    ?SLOG(error, #{msg => "unexpected_call", call => Req}),
     {reply, ignored, State}.
 
 handle_cast(Msg, State) ->
-    ?LOG(error, "Unexpected msg: ~p", [Msg]),
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
     {noreply, State}.
 
 handle_continue(Continue, State) ->
-    ?LOG(error, "Unexpected continue: ~p", [Continue]),
+    ?SLOG(error, #{msg => "unexpected_continue", continue => Continue}),
     {noreply, State}.
 
 handle_info({timeout, TRef, time_to_report_telemetry_data}, State = #state{timer = TRef}) ->
@@ -192,7 +192,7 @@ handle_info({timeout, TRef, time_to_report_telemetry_data}, State = #state{timer
     {noreply, ensure_report_timer(State)};
 
 handle_info(Info, State) ->
-    ?LOG(error, "Unexpected info: ~p", [Info]),
+    ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
 
 terminate(_Reason, _State) ->
diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl
index 7ca14b921..58636870c 100644
--- a/apps/emqx_modules/src/emqx_topic_metrics.erl
+++ b/apps/emqx_modules/src/emqx_topic_metrics.erl
@@ -261,7 +261,7 @@ handle_call({get_rates, Topic, Metric}, _From, State = #state{speeds = Speeds})
     end.
 
 handle_cast(Msg, State) ->
-    ?LOG(error, "Unexpected cast: ~p", [Msg]),
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
     {noreply, State}.
 
 handle_info(ticking, State = #state{speeds = Speeds}) ->
@@ -276,7 +276,7 @@ handle_info(ticking, State = #state{speeds = Speeds}) ->
     {noreply, State#state{speeds = NSpeeds}};
 
 handle_info(Info, State) ->
-    ?LOG(error, "Unexpected info: ~p", [Info]),
+    ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
 
 terminate(_Reason, _State) ->
diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl
index 9be449b60..6e138360b 100644
--- a/apps/emqx_retainer/src/emqx_retainer.erl
+++ b/apps/emqx_retainer/src/emqx_retainer.erl
@@ -217,11 +217,11 @@ handle_call({page_read, Topic, Page, Limit}, _, #{context := Context} = State) -
     {reply, Result, State};
 
 handle_call(Req, _From, State) ->
-    ?LOG(error, "Unexpected call: ~p", [Req]),
+    ?SLOG(error, #{msg => "unexpected_call", call => Req}),
     {reply, ignored, State}.
 
 handle_cast(Msg, State) ->
-    ?LOG(error, "Unexpected cast: ~p", [Msg]),
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
     {noreply, State}.
 
 handle_info(clear_expired, #{context := Context} = State) ->
@@ -248,7 +248,7 @@ handle_info(release_deliver_quota, #{context := Context, wait_quotas := Waits} =
                      wait_quotas := []}};
 
 handle_info(Info, State) ->
-    ?LOG(error, "Unexpected info: ~p", [Info]),
+    ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
 
 terminate(_Reason, #{clear_timer := TRef1, release_quota_timer := TRef2}) ->
diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl
index e5e347fdc..eb08bf6cc 100644
--- a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl
@@ -91,14 +91,13 @@ store_retained(_, Msg =#message{topic = Topic}) ->
                                                          expiry_time = ExpiryTime},
                                                write);
                               [] ->
-                                  ?LOG(error,
-                                       "Cannot retain message(topic=~ts) for table is full!",
-                                       [Topic]),
-                                  ok
+                                  mnesia:abort(table_is_full)
                           end
             end,
-            {atomic, ok} = mria:transaction(?RETAINER_SHARD, Fun),
-            ok
+            case mria:transaction(?RETAINER_SHARD, Fun) of
+                {atomic, ok} ->  ok;
+                {aborted, Reason} -> ?SLOG(error, #{msg => "failed_to_retain_message", topic => Topic, reason => Reason})
+            end
     end.
 
 clear_expired(_) ->
diff --git a/apps/emqx_retainer/src/emqx_retainer_pool.erl b/apps/emqx_retainer/src/emqx_retainer_pool.erl
index 6b48c0453..5a6d8e66b 100644
--- a/apps/emqx_retainer/src/emqx_retainer_pool.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_pool.erl
@@ -84,7 +84,7 @@ init([Pool, Id]) ->
           {stop, Reason :: term(), Reply :: term(), NewState :: term()} |
           {stop, Reason :: term(), NewState :: term()}.
 handle_call(Req, _From, State) ->
-    ?LOG(error, "Unexpected call: ~p", [Req]),
+    ?SLOG(error, #{msg => "unexpected_call", call => Req}),
     {reply, ignored, State}.
 
 %%--------------------------------------------------------------------
@@ -101,12 +101,12 @@ handle_call(Req, _From, State) ->
 handle_cast({async_submit, Task}, State) ->
     try run(Task)
     catch _:Error:Stacktrace ->
-            ?LOG(error, "Error: ~0p, ~0p", [Error, Stacktrace])
+            ?SLOG(error, #{msg => "crashed_handling_async_task", exception => Error, stacktrace => Stacktrace})
     end,
     {noreply, State};
 
 handle_cast(Msg, State) ->
-    ?LOG(error, "Unexpected cast: ~p", [Msg]),
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
     {noreply, State}.
 
 %%--------------------------------------------------------------------
@@ -121,7 +121,7 @@ handle_cast(Msg, State) ->
           {noreply, NewState :: term(), hibernate} |
           {stop, Reason :: normal | term(), NewState :: term()}.
 handle_info(Info, State) ->
-    ?LOG(error, "Unexpected info: ~p", [Info]),
+    ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.erl b/apps/emqx_slow_subs/src/emqx_slow_subs.erl
index acb4ea441..6c15d5b69 100644
--- a/apps/emqx_slow_subs/src/emqx_slow_subs.erl
+++ b/apps/emqx_slow_subs/src/emqx_slow_subs.erl
@@ -153,11 +153,11 @@ handle_call(clear_history, _, State) ->
     {reply, ok, State};
 
 handle_call(Req, _From, State) ->
-    ?LOG(error, "Unexpected call: ~p", [Req]),
+    ?SLOG(error, #{msg => "unexpected_call", call => Req}),
     {reply, ignored, State}.
 
 handle_cast(Msg, State) ->
-    ?LOG(error, "Unexpected cast: ~p", [Msg]),
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
     {noreply, State}.
 
 handle_info(expire_tick, State) ->
@@ -173,7 +173,7 @@ handle_info(notice_tick, State) ->
     {noreply, State#{last_tick_at := ?NOW}};
 
 handle_info(Info, State) ->
-    ?LOG(error, "Unexpected info: ~p", [Info]),
+    ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
 
 terminate(_Reason, _) ->

From 6f28e103d09c6c9b0354932d92dd9f60d24d6f48 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 20 Dec 2021 15:54:09 +0800
Subject: [PATCH 02/25] fix(dashobard): statistical diagram timestamp use UTC
 time

---
 .../emqx_dashboard/src/emqx_dashboard_collection.erl | 12 ++++++------
 .../src/emqx_dashboard_monitor_api.erl               |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_collection.erl b/apps/emqx_dashboard/src/emqx_dashboard_collection.erl
index dc9c894b6..f8937dee9 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_collection.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_collection.erl
@@ -22,7 +22,7 @@
 
 -export([get_collect/0]).
 
--export([get_local_time/0]).
+-export([get_universal_epoch/0]).
 
 -boot_mnesia({mnesia, [boot]}).
 
@@ -108,7 +108,7 @@ handle_info(collect, State = #{count := Count, collect := Collect, temp_collect
 
 handle_info(clear_expire_data, State = #{expire_interval := ExpireInterval}) ->
     timer(?CLEAR_INTERVAL, clear_expire_data),
-    T1 = get_local_time(),
+    T1 = get_universal_epoch(),
     Spec = ets:fun2ms(fun({_, T, _C} = Data) when (T1 - T) > ExpireInterval -> Data end),
     Collects = ets:select(?TAB_COLLECT, Spec),
     lists:foreach(fun(Collect) ->
@@ -161,7 +161,7 @@ flush({Connection, Route, Subscription}, {Received0, Sent0, Dropped0}) ->
                diff(Received, Received0),
                diff(Sent, Sent0),
                diff(Dropped, Dropped0)},
-    Ts = get_local_time(),
+    Ts = get_universal_epoch(),
     {atomic, ok} = mria:transaction(mria:local_content_shard(),
                                     fun mnesia:write/3,
                                     [ ?TAB_COLLECT
@@ -179,8 +179,8 @@ timer(Secs, Msg) ->
     erlang:send_after(Secs, self(), Msg).
 
 get_today_remaining_seconds() ->
-    ?CLEAR_INTERVAL - (get_local_time() rem ?CLEAR_INTERVAL).
+    ?CLEAR_INTERVAL - (get_universal_epoch() rem ?CLEAR_INTERVAL).
 
-get_local_time() ->
-    (calendar:datetime_to_gregorian_seconds(calendar:local_time()) -
+get_universal_epoch() ->
+    (calendar:datetime_to_gregorian_seconds(calendar:universal_time()) -
         calendar:datetime_to_gregorian_seconds({{1970,1,1}, {0,0,0}})).
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
index a05746811..5ada429a3 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
@@ -278,7 +278,7 @@ sampling(Node, Counter) ->
     rpc:call(Node, ?MODULE, sampling, [Node, Counter]).
 
 select_data() ->
-    Time = emqx_dashboard_collection:get_local_time() - 7200000,
+    Time = emqx_dashboard_collection:get_universal_epoch() - 7200000,
     ets:select(?TAB_COLLECT, [{{mqtt_collect,'$1','$2'}, [{'>', '$1', Time}], ['$_']}]).
 
 format(Collects) ->

From 99a5f14301aeb86b2758c3c5527eca5331b79c80 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Tue, 21 Dec 2021 10:44:54 +0800
Subject: [PATCH 03/25] fix(telemetry): use required fields, rolling distro use
 PRETTY_NAME

---
 apps/emqx_modules/src/emqx_telemetry.erl | 49 +++++++++++++-----------
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl
index 932af8005..198e410f0 100644
--- a/apps/emqx_modules/src/emqx_telemetry.erl
+++ b/apps/emqx_modules/src/emqx_telemetry.erl
@@ -220,37 +220,24 @@ os_info() ->
             [{os_name, Name},
              {os_version, Version}];
         {unix, _} ->
-            case file:read_file_info("/etc/os-release") of
+            case file:read_file("/etc/os-release") of
                 {error, _} ->
                     [{os_name, "Unknown"},
                      {os_version, "Unknown"}];
-                {ok, FileInfo} ->
-                    case FileInfo#file_info.access of
-                        Access when Access =:= read orelse Access =:= read_write ->
-                            OSInfo = lists:foldl(fun(Line, Acc) ->
-                                                     [Var, Value] = string:tokens(Line, "="),
-                                                     NValue = case Value of
-                                                                  _ when is_list(Value) ->
-                                                                      lists:nth(1, string:tokens(Value, "\""));
-                                                                  _ ->
-                                                                      Value
-                                                              end,
-                                                     [{Var, NValue} | Acc]
-                                                 end, [], string:tokens(os:cmd("cat /etc/os-release"), "\n")),
-                            [{os_name, get_value("NAME", OSInfo, "Unknown")},
-                             {os_version, get_value("VERSION", OSInfo,
-                                                    get_value("VERSION_ID", OSInfo, "Unknown"))}];
-                        _ ->
-                            [{os_name, "Unknown"},
-                             {os_version, "Unknown"}]
-                    end
+                {ok, FileContent} ->
+                    OSInfo = parse_os_release(FileContent),
+                    [{os_name, get_value("NAME", OSInfo)},
+                     {os_version, get_value("VERSION", OSInfo,
+                                            get_value("VERSION_ID", OSInfo,
+                                                      get_value("PRETTY_NAME", OSInfo)))}]
             end;
         {win32, nt} ->
             Ver = os:cmd("ver"),
             case re:run(Ver, "[a-zA-Z ]+ \\[Version ([0-9]+[\.])+[0-9]+\\]", [{capture, none}]) of
                 match ->
                     [NVer | _] = string:tokens(Ver, "\r\n"),
-                    {match, [Version]} = re:run(NVer, "([0-9]+[\.])+[0-9]+", [{capture, first, list}]),
+                    {match, [Version]} =
+                        re:run(NVer, "([0-9]+[\.])+[0-9]+", [{capture, first, list}]),
                     [Name | _] = string:split(NVer, " [Version "),
                     [{os_name, Name},
                      {os_version, Version}];
@@ -307,7 +294,8 @@ generate_uuid() ->
     <<NTimeHigh:16>> = <<16#01:4, TimeHigh:12>>,
     <<NClockSeq:16>> = <<1:1, 0:1, ClockSeq:14>>,
     <<Node:48>> = <<First:7, 1:1, Last:40>>,
-    list_to_binary(io_lib:format("~.16B-~.16B-~.16B-~.16B-~.16B", [TimeLow, TimeMid, NTimeHigh, NClockSeq, Node])).
+    list_to_binary(io_lib:format( "~.16B-~.16B-~.16B-~.16B-~.16B"
+                                , [TimeLow, TimeMid, NTimeHigh, NClockSeq, Node])).
 
 get_telemetry(#state{uuid = UUID}) ->
     OSInfo = os_info(),
@@ -339,7 +327,22 @@ report_telemetry(State = #state{url = URL}) ->
 httpc_request(Method, URL, Headers, Body) ->
     httpc:request(Method, {URL, Headers, "application/json", Body}, [], []).
 
+parse_os_release(FileContent) ->
+    lists:foldl(fun(Line, Acc) ->
+                        [Var, Value] = string:tokens(Line, "="),
+                        NValue = case Value of
+                                     _ when is_list(Value) ->
+                                         lists:nth(1, string:tokens(Value, "\""));
+                                     _ ->
+                                         Value
+                                 end,
+                        [{Var, NValue} | Acc]
+                end,
+                [], string:tokens(binary:bin_to_list(FileContent), "\n")).
+
 bin(L) when is_list(L) ->
     list_to_binary(L);
+bin(A) when is_atom(A) ->
+    atom_to_binary(A);
 bin(B) when is_binary(B) ->
     B.

From 5f050b149be0f71dee1e6d8b0d916ff8baeba68f Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 22 Dec 2021 10:15:48 +0800
Subject: [PATCH 04/25] fix(rules): the schema for unsubscribe is messing from
 rule_test API

---
 apps/emqx_rule_engine/src/emqx_rule_api_schema.erl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl
index 1caa8da23..d992cdc07 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl
@@ -68,6 +68,7 @@ fields("rule_events") ->
 fields("rule_test") ->
     [ {"context", sc(hoconsc:union([ ref("ctx_pub")
                                    , ref("ctx_sub")
+                                   , ref("ctx_unsub")
                                    , ref("ctx_delivered")
                                    , ref("ctx_acked")
                                    , ref("ctx_dropped")

From 9b4b3d2e8cdf3e01058c746d8335727dd8f18f92 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 22 Dec 2021 10:17:33 +0800
Subject: [PATCH 05/25] fix(rules): make the 'name' field of POST /rules
 mandatory

---
 apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
index ba516bfa7..5d72d5a6d 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
@@ -182,6 +182,7 @@ rule_name() ->
     {"name", sc(binary(),
         #{ desc => "The name of the rule"
          , default => ""
+         , nullable => false
          , example => "foo"
          })}.
 

From cd4227b8511809d3ba2b084153ba38ff47ab7325 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 22 Dec 2021 10:24:52 +0800
Subject: [PATCH 06/25] fix(rules): don't show the module name
 'emqx_rule_outputs' in outputs

---
 apps/emqx_rule_engine/src/emqx_rule_engine_api.erl | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index 205f85488..cbfda16db 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -257,11 +257,16 @@ format_output(Outputs) ->
     [do_format_output(Out) || Out <- Outputs].
 
 do_format_output(#{mod := Mod, func := Func, args := Args}) ->
-    #{function => list_to_binary(lists:concat([Mod,":",Func])),
+    #{function => printable_function_name(Mod, Func),
       args => maps:remove(preprocessed_tmpl, Args)};
 do_format_output(BridgeChannelId) when is_binary(BridgeChannelId) ->
     BridgeChannelId.
 
+printable_function_name(emqx_rule_outputs, Func) ->
+    Func;
+printable_function_name(Mod, Func) ->
+    list_to_binary(lists:concat([Mod,":",Func])).
+
 get_rule_metrics(Id) ->
     Format = fun (Node, #{matched := Matched,
                           rate := Current,

From b11a15fa0008ba9b5d26b3edfcc6bbc1e1b2aaaa Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 22 Dec 2021 14:24:58 +0800
Subject: [PATCH 07/25] fix(banned): create banned with utf8 failed by 500

---
 apps/emqx/src/emqx_trace/emqx_trace.erl              |  1 +
 apps/emqx_dashboard/src/emqx_dashboard_swagger.erl   | 10 ++++++++--
 apps/emqx_gateway/src/emqx_gateway_api_listeners.erl |  2 +-
 apps/emqx_management/src/emqx_mgmt_api_banned.erl    |  6 +++---
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl
index 0d7f66323..42e4d0baf 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace.erl
@@ -194,6 +194,7 @@ format(Traces) ->
               end, Traces).
 
 init([]) ->
+    ok = mria:wait_for_tables([?TRACE]),
     erlang:process_flag(trap_exit, true),
     OriginLogLevel = emqx_logger:get_primary_log_level(),
     ok = filelib:ensure_dir(trace_dir()),
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
index 2dcdba643..9a54be9c5 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
@@ -211,11 +211,16 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
 %%                   {good_nest_2, mk(ref(?MODULE, good_ref), #{})}
 %%                ]}
 %% ]
-check_request_body(#{body := Body}, Spec, _Module, CheckFun, false) ->
+check_request_body(#{body := Body}, Spec, _Module, CheckFun, false)when is_list(Spec) ->
     lists:foldl(fun({Name, Type}, Acc) ->
         Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
         maps:merge(Acc, CheckFun(Schema, Body, #{}))
-                end, #{}, Spec).
+                end, #{}, Spec);
+
+%% requestBody => #{content => #{ 'application/octet-stream' =>
+%% #{schema => #{ type => string, format => binary}}}
+check_request_body(#{body := Body}, Spec, _Module, _CheckFun, false)when is_map(Spec) ->
+    Body.
 
 %% tags, description, summary, security, deprecated
 meta_to_spec(Meta, Module) ->
@@ -287,6 +292,7 @@ trans_desc(Spec, Hocon) ->
         Desc -> Spec#{description => to_bin(Desc)}
     end.
 
+request_body(#{content := _} = Content, _Module) -> {Content, []};
 request_body([], _Module) -> {[], []};
 request_body(Schema, Module) ->
     {{Props, Refs}, Examples} =
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
index fbf923700..ad381ce44 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
@@ -32,7 +32,7 @@
 
 -import(emqx_gateway_api_authn, [schema_authn/0]).
 
-%% minirest/dashbaord_swagger behaviour callbacks
+%% minirest/dashboard_swagger behaviour callbacks
 -export([ api_spec/0
         , paths/0
         , schema/1
diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl
index c9ae1401d..6521a549c 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl
@@ -101,15 +101,15 @@ fields(ban) ->
             desc => <<"Banned type clientid, username, peerhost">>,
             nullable => false,
             example => username})},
-        {who, hoconsc:mk(binary(), #{
+        {who, hoconsc:mk(emqx_schema:unicode_binary(), #{
             desc => <<"Client info as banned type">>,
             nullable => false,
-            example => <<"Badass">>})},
+            example => <<"Badass坏"/utf8>>})},
         {by, hoconsc:mk(binary(), #{
             desc => <<"Commander">>,
             nullable => true,
             example => <<"mgmt_api">>})},
-        {reason, hoconsc:mk(binary(), #{
+        {reason, hoconsc:mk(emqx_schema:unicode_binary(), #{
             desc => <<"Banned reason">>,
             nullable => true,
             example => <<"Too many requests">>})},

From 3fd90614187e84626bf3cabcb488c006fabe0661 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 20 Dec 2021 17:54:29 +0800
Subject: [PATCH 08/25] fix(gw): save coap channel info

---
 apps/emqx_gateway/src/coap/emqx_coap_channel.erl | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl
index ab079b587..60f3d4837 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl
@@ -460,8 +460,9 @@ process_connect(#channel{ctx = Ctx,
         {ok, _Sess} ->
             RandVal = rand:uniform(?TOKEN_MAXIMUM),
             Token = erlang:list_to_binary(erlang:integer_to_list(RandVal)),
+            NResult = Result#{events => [{event, connected}]},
             iter(Iter,
-                 reply({ok, created}, Token, Msg, Result),
+                 reply({ok, created}, Token, Msg, NResult),
                  Channel#channel{token = Token});
         {error, Reason} ->
             ?SLOG(error, #{ msg => "failed_open_session"
@@ -568,7 +569,8 @@ process_out(Outs, Result, Channel, _) ->
                 Reply ->
                     [Reply | Outs2]
             end,
-    {ok, {outgoing, Outs3}, Channel}.
+    Events = maps:get(events, Result, []),
+    {ok, [{outgoing, Outs3}] ++ Events, Channel}.
 
 %% leaf node
 process_nothing(_, _, Channel) ->
@@ -607,4 +609,6 @@ process_reply(Reply, Result, #channel{session = Session} = Channel, _) ->
     Session2 = emqx_coap_session:set_reply(Reply, Session),
     Outs = maps:get(out, Result, []),
     Outs2 = lists:reverse(Outs),
-    {ok, {outgoing, [Reply | Outs2]}, Channel#channel{session = Session2}}.
+    Events = maps:get(events, Result, []),
+    {ok, [{outgoing, [Reply | Outs2]}] ++ Events,
+     Channel#channel{session = Session2}}.

From 6d4aac16007cd97a652e09c1125aaade1418a4d7 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 22 Dec 2021 15:33:33 +0800
Subject: [PATCH 09/25] chore(gw): improve http error messages

---
 apps/emqx_gateway/src/emqx_gateway_api.erl    |  2 +-
 .../src/emqx_gateway_api_clients.erl          | 15 +--
 apps/emqx_gateway/src/emqx_gateway_conf.erl   | 55 ++++++++---
 apps/emqx_gateway/src/emqx_gateway_http.erl   | 92 ++++++++++++-------
 .../src/emqx_gateway_insta_sup.erl            | 12 +--
 apps/emqx_gateway/src/emqx_gateway_utils.erl  |  1 +
 .../src/exproto/emqx_exproto_impl.erl         |  8 +-
 .../src/lwm2m/emqx_lwm2m_impl.erl             | 22 +++--
 .../src/lwm2m/emqx_lwm2m_xml_object_db.erl    | 14 ++-
 .../test/emqx_gateway_conf_SUITE.erl          | 38 +++++---
 10 files changed, 174 insertions(+), 85 deletions(-)

diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl
index 3ee209f19..3a133e340 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api.erl
@@ -83,7 +83,7 @@ gateway(post, Request) ->
                     {ok, NGwConf} ->
                         {201, NGwConf};
                     {error, Reason} ->
-                        return_http_error(500, Reason)
+                        emqx_gateway_http:reason2resp(Reason)
                 end
         end
     catch
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
index 69a06be61..697bccc1d 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
@@ -745,7 +745,8 @@ common_client_props() ->
                        "due to exceeding the length">>})}
     , {awaiting_rel_cnt,
        mk(integer(),
-          #{ desc => <<"Number of awaiting PUBREC packet">>})}
+          %% FIXME: PUBREC ??
+          #{ desc => <<"Number of awaiting acknowledge packet">>})}
     , {awaiting_rel_max,
        mk(integer(),
           #{ desc => <<"Maximum allowed number of awaiting PUBREC "
@@ -755,25 +756,25 @@ common_client_props() ->
           #{ desc => <<"Number of bytes received by EMQ X Broker">>})}
     , {recv_cnt,
        mk(integer(),
-          #{ desc => <<"Number of TCP packets received">>})}
+          #{ desc => <<"Number of socket packets received">>})}
     , {recv_pkt,
        mk(integer(),
-          #{ desc => <<"Number of MQTT packets received">>})}
+          #{ desc => <<"Number of protocol packets received">>})}
     , {recv_msg,
        mk(integer(),
-          #{ desc => <<"Number of PUBLISH packets received">>})}
+          #{ desc => <<"Number of message packets received">>})}
     , {send_oct,
        mk(integer(),
           #{ desc => <<"Number of bytes sent">>})}
     , {send_cnt,
        mk(integer(),
-          #{ desc => <<"Number of TCP packets sent">>})}
+          #{ desc => <<"Number of socket packets sent">>})}
     , {send_pkt,
        mk(integer(),
-          #{ desc => <<"Number of MQTT packets sent">>})}
+          #{ desc => <<"Number of protocol packets sent">>})}
     , {send_msg,
        mk(integer(),
-          #{ desc => <<"Number of PUBLISH packets sent">>})}
+          #{ desc => <<"Number of message packets sent">>})}
     , {mailbox_len,
        mk(integer(),
           #{ desc => <<"Process mailbox size">>})}
diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl
index 987c9720b..cd1f64871 100644
--- a/apps/emqx_gateway/src/emqx_gateway_conf.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl
@@ -248,7 +248,8 @@ update(Req) ->
     res(emqx_conf:update([gateway], Req, #{override_to => cluster})).
 
 res({ok, Result}) -> {ok, Result};
-res({error, {pre_config_update,emqx_gateway_conf,Reason}}) -> {error, Reason};
+res({error, {pre_config_update,?MODULE,Reason}}) -> {error, Reason};
+res({error, {post_config_update,?MODULE,Reason}}) -> {error, Reason};
 res({error, Reason}) -> {error, Reason}.
 
 bin({LType, LName}) ->
@@ -314,12 +315,12 @@ pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) ->
             NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf),
             {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})};
         _ ->
-            {error, already_exist}
+            badres_gateway(already_exist, GwName)
     end;
 pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) ->
     case maps:get(GwName, RawConf, undefined) of
         undefined ->
-            {error, not_found};
+            badres_gateway(not_found, GwName);
         _ ->
             NConf = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf),
             {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})}
@@ -341,13 +342,13 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
                    RawConf,
                    #{GwName => #{<<"listeners">> => NListener}})};
         _ ->
-            {error, already_exist}
+            badres_listener(already_exist, GwName, LType, LName)
     end;
 pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) ->
     case emqx_map_lib:deep_get(
            [GwName, <<"listeners">>, LType, LName], RawConf, undefined) of
         undefined ->
-            {error, not_found};
+            badres_listener(not_found, GwName, LType, LName);
         OldConf ->
             NConf = convert_certs(certs_dir(GwName), Conf, OldConf),
             NListener = #{LType => #{LName => NConf}},
@@ -374,14 +375,14 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
                    RawConf,
                    #{GwName => #{?AUTHN_BIN => Conf}})};
         _ ->
-            {error, already_exist}
+            badres_authn(already_exist, GwName)
     end;
 pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
     case emqx_map_lib:deep_get(
            [GwName, <<"listeners">>, LType, LName],
            RawConf, undefined) of
         undefined ->
-            {error, not_found};
+            badres_listener(not_found, GwName, LType, LName);
         Listener ->
             case maps:get(?AUTHN_BIN, Listener, undefined) of
                 undefined ->
@@ -391,14 +392,14 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
                                    #{LType => #{LName => NListener}}}},
                     {ok, emqx_map_lib:deep_merge(RawConf, NGateway)};
                 _ ->
-                    {error, already_exist}
+                    badres_listener_authn(already_exist, GwName, LType, LName)
             end
     end;
 pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
     case emqx_map_lib:deep_get(
            [GwName, ?AUTHN_BIN], RawConf, undefined) of
         undefined ->
-            {error, not_found};
+            badres_authn(not_found, GwName);
         _ ->
             {ok, emqx_map_lib:deep_merge(
                    RawConf,
@@ -409,11 +410,11 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
            [GwName, <<"listeners">>, LType, LName],
            RawConf, undefined) of
         undefined ->
-            {error, not_found};
+            badres_listener(not_found, GwName, LType, LName);
         Listener ->
             case maps:get(?AUTHN_BIN, Listener, undefined) of
                 undefined ->
-                    {error, not_found};
+                    badres_listener_authn(not_found, GwName, LType, LName);
                 Auth ->
                     NListener = maps:put(
                                   ?AUTHN_BIN,
@@ -437,6 +438,38 @@ pre_config_update(_, UnknownReq, _RawConf) ->
     logger:error("Unknown configuration update request: ~0p", [UnknownReq]),
     {error, badreq}.
 
+badres_gateway(not_found, GwName) ->
+    {error, {badres, #{resource => gateway, gateway => GwName,
+                       reason => not_found}}};
+badres_gateway(already_exist, GwName) ->
+    {error, {badres, #{resource => gateway, gateway => GwName,
+                       reason => already_exist}}}.
+
+badres_listener(not_found, GwName, LType, LName) ->
+    {error, {badres, #{resource => listener, gateway => GwName,
+                       listener => {GwName, LType, LName},
+                       reason => not_found}}};
+badres_listener(already_exist, GwName, LType, LName) ->
+    {error, {badres, #{resource => listener, gateway => GwName,
+                       listener => {GwName, LType, LName},
+                       reason => already_exist}}}.
+
+badres_authn(not_found, GwName) ->
+    {error, {badres, #{resource => authn, gateway => GwName,
+                       reason => not_found}}};
+badres_authn(already_exist, GwName) ->
+    {error, {badres, #{resource => authn, gateway => GwName,
+                       reason => already_exist}}}.
+
+badres_listener_authn(not_found, GwName, LType, LName) ->
+    {error, {badres, #{resource => listener_authn, gateway => GwName,
+                       listener => {GwName, LType, LName},
+                       reason => not_found}}};
+badres_listener_authn(already_exist, GwName, LType, LName) ->
+    {error, {badres, #{resource => listener_authn, gateway => GwName,
+                       listener => {GwName, LType, LName},
+                       reason => already_exist}}}.
+
 -spec post_config_update(list(atom()),
                          emqx_config:update_request(),
                          emqx_config:config(),
diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl
index 810a79987..634ae8252 100644
--- a/apps/emqx_gateway/src/emqx_gateway_http.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_http.erl
@@ -23,6 +23,8 @@
 
 -define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
 
+-import(emqx_gateway_utils, [listener_id/3]).
+
 %% Mgmt APIs - gateway
 -export([ gateways/1
         ]).
@@ -59,10 +61,7 @@
         , with_authn/2
         , with_listener_authn/3
         , checks/2
-        , schema_bad_request/0
-        , schema_not_found/0
-        , schema_internal_error/0
-        , schema_no_content/0
+        , reason2resp/1
         ]).
 
 -type gateway_summary() ::
@@ -131,7 +130,7 @@ current_connections_count(GwName) ->
 get_listeners_status(GwName, Config) ->
     Listeners = emqx_gateway_utils:normalize_config(Config),
     lists:map(fun({Type, LisName, ListenOn, _, _}) ->
-        Name0 = emqx_gateway_utils:listener_id(GwName, Type, LisName),
+        Name0 = listener_id(GwName, Type, LisName),
         Name = {Name0, ListenOn},
         LisO = #{id => Name0, type => Type, name => LisName},
         case catch esockd:listener(Name) of
@@ -223,12 +222,7 @@ remove_authn(GwName, ListenerId) ->
 
 confexp(ok) -> ok;
 confexp({ok, Res}) -> {ok, Res};
-confexp({error, badarg}) ->
-    error({update_conf_error, badarg});
-confexp({error, not_found}) ->
-    error({update_conf_error, not_found});
-confexp({error, already_exist}) ->
-    error({update_conf_error, already_exist}).
+confexp({error, Reason}) -> error(Reason).
 
 %%--------------------------------------------------------------------
 %% Mgmt APIs - clients
@@ -322,6 +316,59 @@ with_channel(GwName, ClientId, Fun) ->
 %% Utils
 %%--------------------------------------------------------------------
 
+-spec reason2resp({atom(), map()} | any()) -> binary() | any().
+reason2resp({badconf, #{key := Key, value := Value, reason := Reason}}) ->
+    fmt400err("Bad config value '~s' for '~s', reason: ~s",
+              [Value, Key, Reason]);
+reason2resp({badres, #{resource := gateway,
+                       gateway := GwName,
+                       reason := not_found}}) ->
+    fmt400err("The ~s gateway is unloaded", [GwName]);
+
+reason2resp({badres, #{resource := gateway,
+                       gateway := GwName,
+                       reason := already_exist}}) ->
+    fmt400err("The ~s gateway has loaded", [GwName]);
+
+reason2resp({badres, #{resource := listener,
+                       listener := {GwName, LType, LName},
+                       reason := not_found}}) ->
+    fmt400err("Listener ~s not found",
+              [listener_id(GwName, LType, LName)]);
+
+reason2resp({badres, #{resource := listener,
+                       listener := {GwName, LType, LName},
+                       reason := already_exist}}) ->
+    fmt400err("The listener ~s of ~s already exist",
+              [listener_id(GwName, LType, LName), GwName]);
+
+reason2resp({badres, #{resource := authn,
+                       gateway := GwName,
+                       reason := not_found}}) ->
+    fmt400err("The authentication not found on ~s", [GwName]);
+
+reason2resp({badres, #{resource := authn,
+                       gateway := GwName,
+                       reason := already_exist}}) ->
+    fmt400err("The authentication already exist on ~s", [GwName]);
+
+reason2resp({badres, #{resource := listener_authn,
+                       listener := {GwName, LType, LName},
+                       reason := not_found}}) ->
+    fmt400err("The authentication not found on ~s",
+              [listener_id(GwName, LType, LName)]);
+
+reason2resp({badres, #{resource := listener_authn,
+                       listener := {GwName, LType, LName},
+                       reason := already_exist}}) ->
+    fmt400err("The authentication already exist on ~s",
+              [listener_id(GwName, LType, LName)]);
+
+reason2resp(R) -> return_http_error(500, R).
+
+fmt400err(Fmt, Args) ->
+    return_http_error(400, io_lib:format(Fmt, Args)).
+
 -spec return_http_error(integer(), any()) -> {integer(), binary()}.
 return_http_error(Code, Msg) ->
     {Code, emqx_json:encode(
@@ -378,19 +425,12 @@ with_gateway(GwName0, Fun) ->
             Path = lists:concat(
                      lists:join(".", lists:map(fun to_list/1, Path0))),
             return_http_error(404, "Resource not found. path: " ++ Path);
-        %% Exceptions from: confexp/1
-        error : {update_conf_error, badarg} ->
-            return_http_error(400, "Bad arguments");
-        error : {update_conf_error, not_found} ->
-            return_http_error(404, "Resource not found");
-        error : {update_conf_error, already_exist} ->
-            return_http_error(400, "Resource already exist");
         Class : Reason : Stk ->
             ?SLOG(error, #{ msg => "uncatched_error"
                           , reason => {Class, Reason}
                           , stacktrace => Stk
                           }),
-            return_http_error(500, {Class, Reason, Stk})
+            reason2resp(Reason)
     end.
 
 -spec checks(list(), map()) -> ok.
@@ -408,20 +448,6 @@ to_list(A) when is_atom(A) ->
 to_list(B) when is_binary(B) ->
     binary_to_list(B).
 
-%%--------------------------------------------------------------------
-%% common schemas
-
-schema_bad_request() ->
-    emqx_mgmt_util:error_schema(
-      <<"Some Params missed">>, ['PARAMETER_MISSED']).
-schema_internal_error() ->
-    emqx_mgmt_util:error_schema(
-      <<"Ineternal Server Error">>, ['INTERNAL_SERVER_ERROR']).
-schema_not_found() ->
-    emqx_mgmt_util:error_schema(<<"Resource Not Found">>).
-schema_no_content() ->
-    #{description => <<"No Content">>}.
-
 %%--------------------------------------------------------------------
 %% Internal funcs
 
diff --git a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
index efb3f6fe6..20db58512 100644
--- a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl
@@ -112,7 +112,7 @@ init([Gateway, Ctx, _GwDscrptr]) ->
         true ->
             case cb_gateway_load(State) of
                 {error, Reason} ->
-                    {stop, {load_gateway_failure, Reason}};
+                    {stop, Reason};
                 {ok, NState} ->
                     {ok, NState}
             end
@@ -360,7 +360,7 @@ cb_gateway_unload(State = #state{name = GwName,
                           , reason => {Class, Reason}
                           , stacktrace => Stk
                           }),
-            {error, {Class, Reason, Stk}}
+            {error, Reason}
     after
         _ = do_deinit_authn(State#state.authns)
     end.
@@ -381,7 +381,7 @@ cb_gateway_load(State = #state{name = GwName,
         case CbMod:on_gateway_load(Gateway, NCtx) of
             {error, Reason} ->
                 do_deinit_authn(AuthnNames),
-                throw({callback_return_error, Reason});
+                {error, Reason};
             {ok, ChildPidOrSpecs, GwState} ->
                 ChildPids = start_child_process(ChildPidOrSpecs),
                 {ok, State#state{
@@ -403,7 +403,7 @@ cb_gateway_load(State = #state{name = GwName,
                           , reason => {Class, Reason1}
                           , stacktrace => Stk
                           }),
-            {error, {Class, Reason1, Stk}}
+            {error, Reason1}
     end.
 
 cb_gateway_update(Config,
@@ -412,7 +412,7 @@ cb_gateway_update(Config,
     try
         #{cbkmod := CbMod} = emqx_gateway_registry:lookup(GwName),
         case CbMod:on_gateway_update(Config, detailed_gateway_info(State), GwState) of
-            {error, Reason} -> throw({callback_return_error, Reason});
+            {error, Reason} -> {error, Reason};
             {ok, ChildPidOrSpecs, NGwState} ->
                 %% XXX: Hot-upgrade ???
                 ChildPids = start_child_process(ChildPidOrSpecs),
@@ -430,7 +430,7 @@ cb_gateway_update(Config,
                           , reason => {Class, Reason1}
                           , stacktrace => Stk
                           }),
-            {error, {Class, Reason1, Stk}}
+            {error, Reason1}
     end.
 
 start_child_process([]) -> [];
diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl
index fa74f9437..8a81584d6 100644
--- a/apps/emqx_gateway/src/emqx_gateway_utils.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl
@@ -90,6 +90,7 @@ childspec(Id, Type, Mod, Args) ->
     -> {ok, pid()}
      | {error, supervisor:startchild_err()}.
 supervisor_ret({ok, Pid, _Info}) -> {ok, Pid};
+supervisor_ret({error, {Reason, _Child}}) -> {error, Reason};
 supervisor_ret(Ret) -> Ret.
 
 -spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id())
diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
index d0ac84322..46e3a1628 100644
--- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
+++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
@@ -75,7 +75,13 @@ stop_grpc_server(GwName) ->
 start_grpc_client_channel(_GwName, undefined) ->
     undefined;
 start_grpc_client_channel(GwName, Options = #{address := Address}) ->
-    {Host, Port} = emqx_gateway_utils:parse_address(Address),
+    {Host, Port} = try emqx_gateway_utils:parse_address(Address)
+                   catch error : badarg ->
+                         throw({badconf, #{key => address,
+                                           value => Address,
+                                           reason => illegal_grpc_address
+                                          }})
+                   end,
     case maps:to_list(maps:get(ssl, Options, #{})) of
         [] ->
             SvrAddr = compose_http_uri(http, Host, Port),
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
index 6e01161bb..ee27d89b1 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
@@ -50,14 +50,20 @@ unreg() ->
 on_gateway_load(_Gateway = #{ name := GwName,
                               config := Config
                             }, Ctx) ->
-    %% Xml registry
-    {ok, RegPid} = emqx_lwm2m_xml_object_db:start_link(maps:get(xml_dir, Config)),
-
-    Listeners = emqx_gateway_utils:normalize_config(Config),
-    ListenerPids = lists:map(fun(Lis) ->
-                                     start_listener(GwName, Ctx, Lis)
-                             end, Listeners),
-    {ok, ListenerPids, _GwState = #{ctx => Ctx, registry => RegPid}}.
+    XmlDir = maps:get(xml_dir, Config),
+    case emqx_lwm2m_xml_object_db:start_link(XmlDir) of
+        {ok, RegPid} ->
+            Listeners = emqx_gateway_utils:normalize_config(Config),
+            ListenerPids = lists:map(fun(Lis) ->
+                                             start_listener(GwName, Ctx, Lis)
+                                     end, Listeners),
+            {ok, ListenerPids, _GwState = #{ctx => Ctx, registry => RegPid}};
+        {error, Reason} ->
+            throw({badconf, #{ key => xml_dir
+                             , value => XmlDir
+                             , reason => Reason
+                             }})
+    end.
 
 on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
     GwName = maps:get(name, Gateway),
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl
index 3cef3c19e..509971b15 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl
@@ -47,6 +47,11 @@
 %% API Function Definitions
 %% ------------------------------------------------------------------
 
+-spec start_link(string())
+    -> {ok, pid()}
+     | ignore
+     | {error, no_xml_files_found}
+     | {error, term()}.
 start_link(XmlDir) ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [XmlDir], []).
 
@@ -85,8 +90,11 @@ stop() ->
 init([XmlDir]) ->
     _ = ets:new(?LWM2M_OBJECT_DEF_TAB, [set, named_table, protected]),
     _ = ets:new(?LWM2M_OBJECT_NAME_TO_ID_TAB, [set, named_table, protected]),
-    load(XmlDir),
-    {ok, #state{}}.
+    case load(XmlDir) of
+        ok ->
+            {ok, #state{}};
+        {error, Reason} -> {stop, Reason}
+    end.
 
 handle_call(_Request, _From, State) ->
     {reply, ignored, State}.
@@ -116,7 +124,7 @@ load(BaseDir) ->
                     Wild
             end,
     case filelib:wildcard(Wild2) of
-        [] -> error(no_xml_files_found, BaseDir);
+        [] -> {error, no_xml_files_found};
         AllXmlFiles -> load_loop(AllXmlFiles)
     end.
 
diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
index f3859532e..459ebe364 100644
--- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
@@ -245,8 +245,9 @@ t_load_unload_gateway(_) ->
                          ?CONF_STOMP_AUTHN_1,
                          ?CONF_STOMP_LISTENER_1),
     {ok, _} = emqx_gateway_conf:load_gateway(stomp, StompConf1),
-    {error, already_exist} =
-        emqx_gateway_conf:load_gateway(stomp, StompConf1),
+    ?assertMatch(
+       {error, {badres, #{reason := already_exist}}},
+       emqx_gateway_conf:load_gateway(stomp, StompConf1)),
     assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
 
     {ok, _} = emqx_gateway_conf:update_gateway(stomp, StompConf2),
@@ -255,8 +256,9 @@ t_load_unload_gateway(_) ->
     ok = emqx_gateway_conf:unload_gateway(stomp),
     ok = emqx_gateway_conf:unload_gateway(stomp),
 
-    {error, not_found} =
-        emqx_gateway_conf:update_gateway(stomp, StompConf2),
+    ?assertMatch(
+       {error, {badres, #{reason := not_found}}},
+       emqx_gateway_conf:update_gateway(stomp, StompConf2)),
 
     ?assertException(error, {config_not_found, [gateway, stomp]},
                      emqx:get_raw_config([gateway, stomp])),
@@ -280,8 +282,9 @@ t_load_remove_authn(_) ->
 
     ok = emqx_gateway_conf:remove_authn(<<"stomp">>),
 
-    {error, not_found} =
-        emqx_gateway_conf:update_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_2),
+    ?assertMatch(
+       {error, {badres, #{reason := not_found}}},
+       emqx_gateway_conf:update_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_2)),
 
     ?assertException(
        error, {config_not_found, [gateway, stomp, authentication]},
@@ -312,9 +315,10 @@ t_load_remove_listeners(_) ->
     ok = emqx_gateway_conf:remove_listener(
            <<"stomp">>, {<<"tcp">>, <<"default">>}),
 
-    {error, not_found} =
-        emqx_gateway_conf:update_listener(
-          <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_2),
+    ?assertMatch(
+       {error, {badres, #{reason := not_found}}},
+       emqx_gateway_conf:update_listener(
+         <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_2)),
 
     ?assertException(
        error, {config_not_found, [gateway, stomp, listeners, tcp, default]},
@@ -352,9 +356,10 @@ t_load_remove_listener_authn(_) ->
     ok = emqx_gateway_conf:remove_authn(
            <<"stomp">>, {<<"tcp">>, <<"default">>}),
 
-    {error, not_found} =
-        emqx_gateway_conf:update_authn(
-          <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_2),
+    ?assertMatch(
+       {error, {badres, #{reason := not_found}}},
+       emqx_gateway_conf:update_authn(
+         <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_2)),
 
     Path = [gateway, stomp, listeners, tcp, default, authentication],
     ?assertException(
@@ -426,9 +431,12 @@ t_add_listener_with_certs_content(_) ->
     ok = emqx_gateway_conf:remove_listener(
            <<"stomp">>, {<<"ssl">>, <<"default">>}),
     assert_ssl_confs_files_deleted(SslConf),
-    {error, not_found} =
-        emqx_gateway_conf:update_listener(
-          <<"stomp">>, {<<"ssl">>, <<"default">>}, ?CONF_STOMP_LISTENER_SSL_2),
+
+    ?assertMatch(
+       {error, {badres, #{reason := not_found}}},
+       emqx_gateway_conf:update_listener(
+         <<"stomp">>, {<<"ssl">>, <<"default">>}, ?CONF_STOMP_LISTENER_SSL_2)),
+
     ?assertException(
        error, {config_not_found, [gateway, stomp, listeners, ssl, default]},
        emqx:get_raw_config([gateway, stomp, listeners, ssl, default])

From 1bcdbf3a06836c884e1ce2de36fe1fa4593507ad Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 22 Dec 2021 16:22:09 +0800
Subject: [PATCH 10/25] chore(gw): make some fields required

---
 apps/emqx_gateway/src/emqx_gateway_schema.erl | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl
index dfdf6ea2a..cc14eaa33 100644
--- a/apps/emqx_gateway/src/emqx_gateway_schema.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl
@@ -118,6 +118,7 @@ fields(mqttsn) ->
     [ {gateway_id,
        sc(integer(),
           #{ default => 1
+           , nullable => false
            , desc =>
 "MQTT-SN Gateway Id.<br>
 When the <code>broadcast</code> option is enabled,
@@ -142,6 +143,7 @@ The client just sends its PUBLISH messages to a GW"
     , {predefined,
        sc(hoconsc:array(ref(mqttsn_predefined)),
           #{ default => []
+           , nullable => {true, recursively}
            , desc =>
 <<"The Pre-defined topic ids and topic names.<br>
 A 'pre-defined' topic id is a topic id whose mapping to a topic name
@@ -217,6 +219,7 @@ fields(lwm2m) ->
     [ {xml_dir,
        sc(binary(),
           #{ default =>"etc/lwm2m_xml"
+           , nullable => false
            , desc => "The Directory for LwM2M Resource defination"
            })}
     , {lifetime_min,
@@ -265,18 +268,21 @@ beyond this time window are temporarily stored in memory."
 fields(exproto) ->
     [ {server,
        sc(ref(exproto_grpc_server),
-          #{ desc => "Configurations for starting the <code>ConnectionAdapter</code> service"
+          #{ nullable => false
+           , desc => "Configurations for starting the <code>ConnectionAdapter</code> service"
            })}
     , {handler,
        sc(ref(exproto_grpc_handler),
-          #{ desc => "Configurations for request to <code>ConnectionHandler</code> service"
+          #{ nullable => false
+           , desc => "Configurations for request to <code>ConnectionHandler</code> service"
            })}
     , {listeners, sc(ref(udp_tcp_listeners))}
     ] ++ gateway_common_options();
 
 fields(exproto_grpc_server) ->
     [ {bind,
-       sc(hoconsc:union([ip_port(), integer()]))}
+       sc(hoconsc:union([ip_port(), integer()]),
+          #{nullable => false})}
     , {ssl,
        sc(ref(ssl_server_opts),
           #{ nullable => {true, recursively}
@@ -284,7 +290,7 @@ fields(exproto_grpc_server) ->
     ];
 
 fields(exproto_grpc_handler) ->
-    [ {address, sc(binary())}
+    [ {address, sc(binary(), #{nullable => false})}
     , {ssl,
        sc(ref(ssl_client_opts),
           #{ nullable => {true, recursively}
@@ -316,11 +322,13 @@ fields(lwm2m_translators) ->
 For each new LwM2M client that succeeds in going online, the gateway creates
 a the subscription relationship to receive downstream commands and send it to
 the LwM2M client"
+           , nullable => false
            })}
     , {response,
        sc(ref(translator),
           #{ desc =>
 "The topic for gateway to publish the acknowledge events from LwM2M client"
+           , nullable => false
            })}
     , {notify,
        sc(ref(translator),
@@ -328,21 +336,24 @@ the LwM2M client"
 "The topic for gateway to publish the notify events from LwM2M client.<br>
 After succeed observe a resource of LwM2M client, Gateway will send the
 notifyevents via this topic, if the client reports any resource changes"
+           , nullable => false
            })}
     , {register,
        sc(ref(translator),
           #{ desc =>
 "The topic for gateway to publish the register events from LwM2M client.<br>"
+           , nullable => false
            })}
     , {update,
        sc(ref(translator),
           #{ desc =>
 "The topic for gateway to publish the update events from LwM2M client.<br>"
+           , nullable => false
            })}
     ];
 
 fields(translator) ->
-    [ {topic, sc(binary())}
+    [ {topic, sc(binary(), #{nullable => false})}
     , {qos, sc(range(0, 2), #{default => 0})}
     ];
 

From af023b16c67da9dac2a98323479fdfd6b0cc711b Mon Sep 17 00:00:00 2001
From: k32 <10274441+k32@users.noreply.github.com>
Date: Wed, 22 Dec 2021 11:16:35 +0100
Subject: [PATCH 11/25] fix(system_monitor): Fix warning spam

---
 rebar.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rebar.config b/rebar.config
index 1894d817d..8bcab9922 100644
--- a/rebar.config
+++ b/rebar.config
@@ -62,7 +62,7 @@
     , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
-    , {system_monitor, {git, "https://github.com/klarna-incubator/system_monitor", {tag, "2.2.0"}}}
+    , {system_monitor, {git, "https://github.com/k32/system_monitor", {tag, "2.2.1"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.16.0"}}}
     , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}}

From 2130d5ca8e44a5655f6743d161b4ba58a33274fd Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 22 Dec 2021 16:18:27 +0800
Subject: [PATCH 12/25] chore(dashboard): update dashboard version

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ad8b97da1..96a056dff 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3
 export EMQX_DEFAULT_RUNNER = alpine:3.14
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v0.8.0
+export EMQX_DASHBOARD_VERSION ?= v0.9.0
 export DOCKERFILE := deploy/docker/Dockerfile
 export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
 ifeq ($(OS),Windows_NT)

From e2804ab29dc2a09d877897c145eaa0ae694fb3f3 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 22 Dec 2021 17:59:18 +0800
Subject: [PATCH 13/25] chore(dashboard): update dashboard version, ignore
 v0.9.0

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 96a056dff..49547a03b 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3
 export EMQX_DEFAULT_RUNNER = alpine:3.14
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v0.9.0
+export EMQX_DASHBOARD_VERSION ?= v0.10.0
 export DOCKERFILE := deploy/docker/Dockerfile
 export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
 ifeq ($(OS),Windows_NT)

From 52502e29c34592809daf3cc44d28f033796a6e8b Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 20 Dec 2021 14:22:06 +0800
Subject: [PATCH 14/25] fix: disconnect the client due to exceed
 receive-maximum packets

As described in the 5.0 specification, we should disconnect clients that
exceed the receive-maximum limit.

> If it receives more than Receive Maximum QoS 1 and QoS 2 PUBLISH packets
where it has not sent a PUBACK or PUBCOMP in response, **the Server uses a
DISCONNECT packet with Reason Code 0x9**

fix: #6447
---
 apps/emqx/src/emqx_channel.erl        | 2 +-
 apps/emqx/test/emqx_channel_SUITE.erl | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index eb71aca58..a6cc6b8d7 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -637,7 +637,7 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2},
                 packet_id => PacketId
             }),
             ok = emqx_metrics:inc('packets.publish.dropped'),
-            handle_out(pubrec, {PacketId, RC}, Channel)
+            handle_out(disconnect, RC, Channel)
     end.
 
 ensure_quota(_, Channel = #channel{quota = undefined}) ->
diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl
index 45b00ff29..1077cc870 100644
--- a/apps/emqx/test/emqx_channel_SUITE.erl
+++ b/apps/emqx/test/emqx_channel_SUITE.erl
@@ -370,7 +370,8 @@ t_handle_in_qos2_publish_with_error_return(_) ->
     {ok, ?PUBREC_PACKET(2, ?RC_NO_MATCHING_SUBSCRIBERS), Channel1} =
         emqx_channel:handle_in(Publish2, Channel),
     Publish3 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 3, <<"payload">>),
-    {ok, ?PUBREC_PACKET(3, ?RC_RECEIVE_MAXIMUM_EXCEEDED), Channel1} =
+    {ok, [{outgoing, ?DISCONNECT_PACKET(?RC_RECEIVE_MAXIMUM_EXCEEDED)},
+          {close, receive_maximum_exceeded}], Channel1} =
         emqx_channel:handle_in(Publish3, Channel1).
 
 t_handle_in_puback_ok(_) ->

From 7c9c7b6a6064122e6e809e6369d2bfe33ee3d6a5 Mon Sep 17 00:00:00 2001
From: lafirest <blankalupo@163.com>
Date: Tue, 16 Nov 2021 18:18:19 +0800
Subject: [PATCH 15/25] refactor(emqx_exhook): refactore exhook and add api
 module

---
 apps/emqx_exhook/etc/emqx_exhook.conf         |  56 +-
 apps/emqx_exhook/src/emqx_exhook.erl          |  68 +-
 apps/emqx_exhook/src/emqx_exhook_api.erl      | 281 +++++++++
 apps/emqx_exhook/src/emqx_exhook_cli.erl      |  84 ---
 apps/emqx_exhook/src/emqx_exhook_mgr.erl      | 596 ++++++++++++++++++
 apps/emqx_exhook/src/emqx_exhook_mngr.erl     | 329 ----------
 apps/emqx_exhook/src/emqx_exhook_schema.erl   |  75 ++-
 apps/emqx_exhook/src/emqx_exhook_server.erl   |  89 +--
 apps/emqx_exhook/src/emqx_exhook_sup.erl      |  19 +-
 apps/emqx_exhook/test/emqx_exhook_SUITE.erl   |  83 +--
 .../test/emqx_exhook_api_SUITE.erl            | 197 ++++++
 .../emqx_exhook/test/emqx_exhook_demo_svr.erl |  40 +-
 .../test/props/prop_exhook_hooks.erl          |  11 +-
 13 files changed, 1283 insertions(+), 645 deletions(-)
 create mode 100644 apps/emqx_exhook/src/emqx_exhook_api.erl
 delete mode 100644 apps/emqx_exhook/src/emqx_exhook_cli.erl
 create mode 100644 apps/emqx_exhook/src/emqx_exhook_mgr.erl
 delete mode 100644 apps/emqx_exhook/src/emqx_exhook_mngr.erl
 create mode 100644 apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl

diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_exhook/etc/emqx_exhook.conf
index 42bd04f19..8769e9a2d 100644
--- a/apps/emqx_exhook/etc/emqx_exhook.conf
+++ b/apps/emqx_exhook/etc/emqx_exhook.conf
@@ -2,43 +2,45 @@
 ## EMQ X Hooks
 ##====================================================================
 
-exhook {
-    ## The default value or action will be returned, while the request to
-    ## the gRPC server failed or no available grpc server running.
-    ##
-    ## Default: deny
-    ## Value: ignore | deny
-    request_failed_action = deny
+emqx_exhook {
 
-    ## The timeout to request grpc server
+    servers = [
+    ##{
+    ## name = default
     ##
-    ## Default: 5s
-    ## Value: Duration
-    request_timeout = 5s
-
     ## Whether to automatically reconnect (initialize) the gRPC server
-    ##
     ## When gRPC is not available, exhook tries to request the gRPC service at
     ## that interval and reinitialize the list of mounted hooks.
     ##
     ## Default: false
     ## Value: false | Duration
-    auto_reconnect = 60s
+    ## auto_reconnect = 60s
 
-    ## The process pool size for gRPC client
+    ## The default value or action will be returned, while the request to
+    ## the gRPC server failed or no available grpc server running.
     ##
-    ## Default: Equals cpu cores
-    ## Value: Integer
-    #pool_size = 16
+    ## Default: deny
+    ## Value: ignore | deny
+    ## failed_action = deny
 
-    servers = [
-    #    { name: "default"
-    #      url: "http://127.0.0.1:9000"
-    #      #ssl: {
-    #      #  cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem"
-    #      #  certfile: "{{ platform_etc_dir }}/certs/cert.pem"
-    #      #  keyfile: "{{ platform_etc_dir }}/certs/key.pem"
-    #      #}
-    #    }
+    ## The timeout to request grpc server
+    ##
+    ## Default: 5s
+    ## Value: Duration
+    ## request_timeout = 5s
+
+    ## url = "http://127.0.0.1:9000"
+    ## ssl {
+    ##    cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem"
+    ##    certfile: "{{ platform_etc_dir }}/certs/cert.pem"
+    ##    keyfile: "{{ platform_etc_dir }}/certs/key.pem"
+    ##    }
+    ##
+    ##  The process pool size for gRPC client
+    ##
+    ##  Default: Equals cpu cores
+    ##  Value: Integer
+    ##  pool_size = 16
+    ##}
     ]
 }
diff --git a/apps/emqx_exhook/src/emqx_exhook.erl b/apps/emqx_exhook/src/emqx_exhook.erl
index c6b02e716..60a16ffc0 100644
--- a/apps/emqx_exhook/src/emqx_exhook.erl
+++ b/apps/emqx_exhook/src/emqx_exhook.erl
@@ -19,90 +19,56 @@
 -include("emqx_exhook.hrl").
 -include_lib("emqx/include/logger.hrl").
 
-
--export([ enable/1
-        , disable/1
-        , list/0
-        ]).
-
 -export([ cast/2
         , call_fold/3
         ]).
 
-%%--------------------------------------------------------------------
-%% Mgmt APIs
-%%--------------------------------------------------------------------
-
--spec enable(binary()) -> ok | {error, term()}.
-enable(Name) ->
-    with_mngr(fun(Pid) -> emqx_exhook_mngr:enable(Pid, Name) end).
-
--spec disable(binary()) -> ok | {error, term()}.
-disable(Name) ->
-    with_mngr(fun(Pid) -> emqx_exhook_mngr:disable(Pid, Name) end).
-
--spec list() -> [atom() | string()].
-list() ->
-    with_mngr(fun(Pid) -> emqx_exhook_mngr:list(Pid) end).
-
-with_mngr(Fun) ->
-    case lists:keyfind(emqx_exhook_mngr, 1,
-                       supervisor:which_children(emqx_exhook_sup)) of
-        {_, Pid, _, _} ->
-            Fun(Pid);
-        _ ->
-            {error, no_manager_svr}
-    end.
-
 %%--------------------------------------------------------------------
 %% Dispatch APIs
 %%--------------------------------------------------------------------
 
 -spec cast(atom(), map()) -> ok.
 cast(Hookpoint, Req) ->
-    cast(Hookpoint, Req, emqx_exhook_mngr:running()).
+    cast(Hookpoint, Req, emqx_exhook_mgr:running()).
 
 cast(_, _, []) ->
     ok;
 cast(Hookpoint, Req, [ServerName|More]) ->
     %% XXX: Need a real asynchronous running
     _ = emqx_exhook_server:call(Hookpoint, Req,
-                                emqx_exhook_mngr:server(ServerName)),
+                                emqx_exhook_mgr:server(ServerName)),
     cast(Hookpoint, Req, More).
 
--spec call_fold(atom(), term(), function())
-  -> {ok, term()}
-   | {stop, term()}.
+-spec call_fold(atom(), term(), function()) -> {ok, term()}
+              | {stop, term()}.
 call_fold(Hookpoint, Req, AccFun) ->
-    FailedAction = emqx_exhook_mngr:get_request_failed_action(),
-    ServerNames = emqx_exhook_mngr:running(),
-    case ServerNames == [] andalso FailedAction == deny of
-        true ->
+    case emqx_exhook_mgr:running() of
+        [] ->
             {stop, deny_action_result(Hookpoint, Req)};
-        _ ->
-            call_fold(Hookpoint, Req, FailedAction, AccFun, ServerNames)
+        ServerNames ->
+            call_fold(Hookpoint, Req, AccFun, ServerNames)
     end.
 
-call_fold(_, Req, _, _, []) ->
+call_fold(_, Req, _, []) ->
     {ok, Req};
-call_fold(Hookpoint, Req, FailedAction, AccFun, [ServerName|More]) ->
-    Server = emqx_exhook_mngr:server(ServerName),
+call_fold(Hookpoint, Req, AccFun, [ServerName|More]) ->
+    Server = emqx_exhook_mgr:server(ServerName),
     case emqx_exhook_server:call(Hookpoint, Req, Server) of
         {ok, Resp} ->
             case AccFun(Req, Resp) of
                 {stop, NReq} ->
                     {stop, NReq};
                 {ok, NReq} ->
-                    call_fold(Hookpoint, NReq, FailedAction, AccFun, More);
+                    call_fold(Hookpoint, NReq, AccFun, More);
                 _ ->
-                    call_fold(Hookpoint, Req, FailedAction, AccFun, More)
+                    call_fold(Hookpoint, Req, AccFun, More)
             end;
         _ ->
-            case FailedAction of
+            case emqx_exhook_server:failed_action(Server) of
+                ignore ->
+                    call_fold(Hookpoint, Req, AccFun, More);
                 deny ->
-                    {stop, deny_action_result(Hookpoint, Req)};
-                _ ->
-                    call_fold(Hookpoint, Req, FailedAction, AccFun, More)
+                    {stop, deny_action_result(Hookpoint, Req)}
             end
     end.
 
diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl
new file mode 100644
index 000000000..4684c6796
--- /dev/null
+++ b/apps/emqx_exhook/src/emqx_exhook_api.erl
@@ -0,0 +1,281 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_exhook_api).
+
+-behaviour(minirest_api).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("emqx/include/logger.hrl").
+
+-export([api_spec/0, paths/0, schema/1, fields/1, namespace/0]).
+
+-export([exhooks/2, action_with_name/2, move/2]).
+
+-import(hoconsc, [mk/2, ref/1, enum/1, array/1]).
+-import(emqx_dashboard_swagger, [schema_with_example/2, error_codes/2]).
+
+-define(TAGS, [<<"exhooks">>]).
+-define(BAD_REQUEST, 'BAD_REQUEST').
+-define(BAD_RPC, 'BAD_RPC').
+
+namespace() -> "exhook".
+
+api_spec() ->
+    emqx_dashboard_swagger:spec(?MODULE).
+
+paths() -> ["/exhooks", "/exhooks/:name", "/exhooks/:name/move"].
+
+schema(("/exhooks")) ->
+    #{
+      'operationId' => exhooks,
+      get => #{tags => ?TAGS,
+               description => <<"List all servers">>,
+               responses => #{200 => mk(array(ref(detailed_server_info)), #{})}
+              },
+      post => #{tags => ?TAGS,
+                description => <<"Add a servers">>,
+                'requestBody' => server_conf_schema(),
+                responses => #{201 => mk(ref(detailed_server_info), #{}),
+                               500 => error_codes([?BAD_RPC], <<"Bad RPC">>)
+                              }
+               }
+     };
+
+schema("/exhooks/:name") ->
+    #{'operationId' => action_with_name,
+      get => #{tags => ?TAGS,
+               description => <<"Get the detail information of server">>,
+               parameters => params_server_name_in_path(),
+               responses => #{200 => mk(ref(detailed_server_info), #{}),
+                              400 => error_codes([?BAD_REQUEST], <<"Bad Request">>)
+                             }
+              },
+      put => #{tags => ?TAGS,
+               description => <<"Update the server">>,
+               parameters => params_server_name_in_path(),
+               'requestBody' => server_conf_schema(),
+               responses => #{200 => <<>>,
+                              400 => error_codes([?BAD_REQUEST], <<"Bad Request">>),
+                              500 => error_codes([?BAD_RPC], <<"Bad RPC">>)
+                             }
+              },
+      delete => #{tags => ?TAGS,
+                  description => <<"Delete the server">>,
+                  parameters => params_server_name_in_path(),
+                  responses => #{204 => <<>>,
+                                 500 => error_codes([?BAD_RPC], <<"Bad RPC">>)                                }
+                 }
+     };
+
+schema("/exhooks/:name/move") ->
+    #{'operationId' => move,
+      post => #{tags => ?TAGS,
+                description => <<"Move the server">>,
+                parameters => params_server_name_in_path(),
+                'requestBody' => mk(ref(move_req), #{}),
+                responses => #{200 => <<>>,
+                               400 => error_codes([?BAD_REQUEST], <<"Bad Request">>),
+                               500 => error_codes([?BAD_RPC], <<"Bad RPC">>)
+                              }
+               }
+     }.
+
+fields(move_req) ->
+    [
+     {position, mk(enum([top, bottom, before, 'after']), #{})},
+     {related, mk(string(), #{desc => <<"Relative position of movement">>,
+                              default => <<>>,
+                              example => <<>>
+                             })}
+    ];
+
+fields(detailed_server_info) ->
+    [ {status, mk(enum([running, waiting, stopped]), #{})}
+    , {hooks, mk(array(string()), #{default => []})}
+    , {node_status, mk(ref(node_status), #{})}
+    ] ++ emqx_exhook_schema:server_config();
+
+fields(node_status) ->
+    [ {node, mk(string(), #{})}
+    , {status, mk(enum([running, waiting, stopped, not_found, error]), #{})}
+    ];
+
+fields(server_config) ->
+    emqx_exhook_schema:server_config().
+
+params_server_name_in_path() ->
+    [{name, mk(string(), #{in => path,
+                           required => true,
+                           example => <<"default">>})}
+    ].
+
+server_conf_schema() ->
+    schema_with_example(ref(server_config),
+                        #{ name => "default"
+                         , enable => true
+                         , url => <<"http://127.0.0.1:8081">>
+                         , request_timeout => "5s"
+                         , failed_action => deny
+                         , auto_reconnect => "60s"
+                         , pool_size => 8
+                         , ssl => #{ enable => false
+                                   , cacertfile => <<"{{ platform_etc_dir }}/certs/cacert.pem">>
+                                   , certfile => <<"{{ platform_etc_dir }}/certs/cert.pem">>
+                                   , keyfile => <<"{{ platform_etc_dir }}/certs/key.pem">>
+                                   }
+                         }).
+
+
+exhooks(get, _) ->
+    ServerL = emqx_exhook_mgr:list(),
+    ServerL2 = nodes_all_server_status(ServerL),
+    {200, ServerL2};
+
+exhooks(post, #{body := Body}) ->
+    case emqx_exhook_mgr:update_config([emqx_exhook, servers], {add, Body}) of
+        {ok, Result} ->
+            {201, Result};
+        {error, Error} ->
+            {500, #{code => <<"BAD_RPC">>,
+                    message => Error
+                   }}
+    end.
+
+action_with_name(get, #{bindings := #{name := Name}}) ->
+    Result = emqx_exhook_mgr:lookup(Name),
+    NodeStatus = nodes_server_status(Name),
+    case Result of
+        not_found ->
+            {400, #{code => <<"BAD_REQUEST">>,
+                    message => <<"Server not found">>
+                   }};
+        ServerInfo ->
+            {200, ServerInfo#{node_status => NodeStatus}}
+    end;
+
+action_with_name(put, #{bindings := #{name := Name}, body := Body}) ->
+    case emqx_exhook_mgr:update_config([emqx_exhook, servers],
+                                       {update, Name, Body}) of
+        {ok, not_found} ->
+            {400, #{code => <<"BAD_REQUEST">>,
+                    message => <<"Server not found">>
+                   }};
+        {ok, {error, Reason}} ->
+            {400, #{code => <<"BAD_REQUEST">>,
+                    message => unicode:characters_to_binary(io_lib:format("Error Reason:~p~n", [Reason]))
+                   }};
+        {ok, _} ->
+            {200};
+        {error, Error} ->
+            {500, #{code => <<"BAD_RPC">>,
+                    message => Error
+                   }}
+    end;
+
+action_with_name(delete, #{bindings := #{name := Name}}) ->
+    case emqx_exhook_mgr:update_config([emqx_exhook, servers],
+                                       {delete, Name}) of
+        {ok, _} ->
+            {200};
+        {error, Error} ->
+            {500, #{code => <<"BAD_RPC">>,
+                    message => Error
+                   }}
+    end.
+
+move(post, #{bindings := #{name := Name}, body := Body}) ->
+    #{<<"position">> := PositionT, <<"related">> := Related} = Body,
+    Position = erlang:binary_to_atom(PositionT),
+    case emqx_exhook_mgr:update_config([emqx_exhook, servers],
+                                       {move, Name, Position, Related}) of
+        {ok, ok} ->
+            {200};
+        {ok, not_found} ->
+            {400, #{code => <<"BAD_REQUEST">>,
+                    message => <<"Server not found">>
+                   }};
+        {error, Error} ->
+            {500, #{code => <<"BAD_RPC">>,
+                    message => Error
+                   }}
+    end.
+
+nodes_server_status(Name) ->
+    StatusL = call_cluster(emqx_exhook_mgr, server_status, [Name]),
+
+    Handler = fun({Node, {error, _}}) ->
+                      #{node => Node,
+                        status => error
+                       };
+                 ({Node, Status}) ->
+                      #{node => Node,
+                        status => Status
+                       }
+              end,
+
+    lists:map(Handler, StatusL).
+
+nodes_all_server_status(ServerL) ->
+    AllStatusL = call_cluster(emqx_exhook_mgr, all_servers_status, []),
+
+    AggreMap = lists:foldl(fun(#{name := Name}, Acc) ->
+                                   Acc#{Name => []}
+                           end,
+                           #{},
+                           ServerL),
+
+    AddToMap = fun(Servers, Node, Status, Map) ->
+                       lists:foldl(fun(Name, Acc) ->
+                                           StatusL = maps:get(Name, Acc),
+                                           StatusL2 = [#{node => Node,
+                                                         status => Status
+                                                        } | StatusL],
+                                           Acc#{Name := StatusL2}
+                                   end,
+                                   Map,
+                                   Servers)
+               end,
+
+    AggreMap2 = lists:foldl(fun({Node, #{running := Running,
+                                         waiting := Waiting,
+                                         stopped := Stopped}},
+                                Acc) ->
+                                    AddToMap(Stopped, Node, stopped,
+                                             AddToMap(Waiting, Node, waiting,
+                                                      AddToMap(Running, Node, running, Acc)))
+                            end,
+                            AggreMap,
+                            AllStatusL),
+
+    Handler = fun(#{name := Name} = Server) ->
+                      Server#{node_status => maps:get(Name, AggreMap2)}
+              end,
+
+    lists:map(Handler, ServerL).
+
+call_cluster(Module, Fun, Args) ->
+    Nodes = mria_mnesia:running_nodes(),
+    [{Node, rpc_call(Node, Module, Fun, Args)} || Node <- Nodes].
+
+rpc_call(Node, Module, Fun, Args) when Node =:= node() ->
+    erlang:apply(Module, Fun, Args);
+
+rpc_call(Node, Module, Fun, Args) ->
+    case rpc:call(Node, Module, Fun, Args) of
+        {badrpc, Reason} -> {error, Reason};
+        Res -> Res
+    end.
diff --git a/apps/emqx_exhook/src/emqx_exhook_cli.erl b/apps/emqx_exhook/src/emqx_exhook_cli.erl
deleted file mode 100644
index a96cdb6cc..000000000
--- a/apps/emqx_exhook/src/emqx_exhook_cli.erl
+++ /dev/null
@@ -1,84 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_exhook_cli).
-
--include("emqx_exhook.hrl").
-
--export([cli/1]).
-
-cli(["server", "list"]) ->
-    if_enabled(fun() ->
-        ServerNames = emqx_exhook:list(),
-        [emqx_ctl:print("Server(~ts)~n", [format(Name)]) || Name <- ServerNames]
-    end);
-
-cli(["server", "enable", Name]) ->
-    if_enabled(fun() ->
-        print(emqx_exhook:enable(iolist_to_binary(Name)))
-    end);
-
-cli(["server", "disable", Name]) ->
-    if_enabled(fun() ->
-        print(emqx_exhook:disable(iolist_to_binary(Name)))
-    end);
-
-cli(["server", "stats"]) ->
-    if_enabled(fun() ->
-        [emqx_ctl:print("~-35s:~w~n", [Name, N]) || {Name, N} <- stats()]
-    end);
-
-cli(_) ->
-    emqx_ctl:usage([{"exhook server list", "List all running exhook server"},
-                    {"exhook server enable <Name>", "Enable a exhook server in the configuration"},
-                    {"exhook server disable <Name>", "Disable a exhook server"},
-                    {"exhook server stats", "Print exhook server statistic"}]).
-
-print(ok) ->
-    emqx_ctl:print("ok~n");
-print({error, Reason}) ->
-    emqx_ctl:print("~p~n", [Reason]).
-
-%%--------------------------------------------------------------------
-%% Internal funcs
-%%--------------------------------------------------------------------
-
-if_enabled(Fun) ->
-    case lists:keymember(?APP, 1, application:which_applications()) of
-        true ->
-            Fun();
-        _ -> hint()
-    end.
-
-hint() ->
-    emqx_ctl:print("Please './bin/emqx_ctl plugins load emqx_exhook' first.~n").
-
-stats() ->
-    lists:usort(lists:foldr(fun({K, N}, Acc) ->
-        case atom_to_list(K) of
-            "exhook." ++ Key -> [{Key, N} | Acc];
-            _ -> Acc
-        end
-    end, [], emqx_metrics:all())).
-
-format(Name) ->
-    case emqx_exhook_mngr:server(Name) of
-        undefined ->
-            lists:flatten(
-              io_lib:format("name=~ts, hooks=#{}, active=false", [Name]));
-        Server ->
-            emqx_exhook_server:format(Server)
-    end.
diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl
new file mode 100644
index 000000000..b9d03ae7d
--- /dev/null
+++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl
@@ -0,0 +1,596 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc Manage the server status and reload strategy
+-module(emqx_exhook_mgr).
+
+-behaviour(gen_server).
+
+-include("emqx_exhook.hrl").
+-include_lib("emqx/include/logger.hrl").
+
+%% APIs
+-export([start_link/0]).
+
+%% Mgmt API
+-export([ list/0
+        , lookup/1
+        , enable/1
+        , disable/1
+        , server_status/1
+        , all_servers_status/0
+        ]).
+
+%% Helper funcs
+-export([ running/0
+        , server/1
+        , init_counter_table/0
+        ]).
+
+-export([ update_config/2
+        , pre_config_update/3
+        , post_config_update/5
+        ]).
+
+%% gen_server callbacks
+-export([ init/1
+        , handle_call/3
+        , handle_cast/2
+        , handle_info/2
+        , terminate/2
+        , code_change/3
+        ]).
+
+-export([roots/0]).
+
+-type state() :: #{%% Running servers
+                   running := servers(),
+                   %% Wait to reload servers
+                   waiting := servers(),
+                   %% Marked stopped servers
+                   stopped := servers(),
+                   %% Timer references
+                   trefs := map(),
+                   orders := orders()
+                  }.
+
+-type server_name() :: binary().
+-type servers() :: #{server_name() => server()}.
+-type server() :: server_options().
+-type server_options() :: map().
+
+-type move_direct() :: top
+                     | bottom
+                     | before
+                     | 'after'.
+
+-type orders() :: #{server_name() => integer()}.
+
+-type server_info() :: #{name := server_name(),
+                         status := running | waiting | stopped,
+
+                         atom() => term()
+                        }.
+
+-define(DEFAULT_TIMEOUT, 60000).
+-define(CNTER, emqx_exhook_counter).
+
+-export_type([server_info/0]).
+
+%%--------------------------------------------------------------------
+%% APIs
+%%--------------------------------------------------------------------
+
+-spec start_link() -> ignore
+              | {ok, pid()}
+              | {error, any()}.
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+list() ->
+    call(list).
+
+-spec lookup(server_name()) -> not_found | server_info().
+lookup(Name) ->
+    call({lookup, Name}).
+
+enable(Name) ->
+    update_config([emqx_exhook, servers], {enable, Name, true}).
+
+disable(Name) ->
+    update_config([emqx_exhook, servers], {enable, Name, false}).
+
+server_status(Name) ->
+    call({server_status, Name}).
+
+all_servers_status() ->
+    call(all_servers_status).
+
+call(Req) ->
+    gen_server:call(?MODULE, Req, ?DEFAULT_TIMEOUT).
+
+init_counter_table() ->
+    _ = ets:new(?CNTER, [named_table, public]).
+
+%%=====================================================================
+%% Hocon schema
+roots() ->
+    emqx_exhook_schema:server_config().
+
+update_config(KeyPath, UpdateReq) ->
+    case emqx_conf:update(KeyPath, UpdateReq, #{override_to => cluster}) of
+        {ok, UpdateResult}  ->
+            #{post_config_update := #{?MODULE := Result}} = UpdateResult,
+            {ok, Result};
+        Error ->
+            Error
+    end.
+
+pre_config_update(_, {add, Conf}, OldConf) ->
+    {ok, OldConf ++ [Conf]};
+
+pre_config_update(_, {update, Name, Conf}, OldConf) ->
+    case replace_conf(Name, fun(_) -> Conf end, OldConf) of
+                    not_found -> {error, not_found};
+                    NewConf -> {ok, NewConf}
+    end;
+
+pre_config_update(_, {delete, ToDelete}, OldConf) ->
+    {ok, lists:dropwhile(fun(#{<<"name">> := Name}) -> Name =:= ToDelete end,
+                         OldConf)};
+
+pre_config_update(_, {move, Name, Position, Relate}, OldConf) ->
+    case do_move(Name, Position, Relate, OldConf) of
+        not_found -> {error, not_found};
+        NewConf -> {ok, NewConf}
+    end;
+
+pre_config_update(_, {enable, Name, Enable}, OldConf) ->
+    case replace_conf(Name,
+                      fun(Conf) -> Conf#{<<"enable">> => Enable} end, OldConf) of
+        not_found -> {error, not_found};
+        NewConf ->
+            ct:pal(">>>> enable Name:~p Enable:~p, New:~p~n", [Name, Enable, NewConf]),
+            {ok, NewConf}
+    end.
+
+post_config_update(_KeyPath, UpdateReq, NewConf, _OldConf, _AppEnvs) ->
+    {ok, call({update_config, UpdateReq, NewConf})}.
+
+%%=====================================================================
+
+%%--------------------------------------------------------------------
+%% gen_server callbacks
+%%--------------------------------------------------------------------
+
+init([]) ->
+    process_flag(trap_exit, true),
+    emqx_conf:add_handler([emqx_exhook, servers], ?MODULE),
+    ServerL = emqx:get_config([emqx_exhook, servers]),
+    {Waiting, Running, Stopped} = load_all_servers(ServerL),
+    Orders = reorder(ServerL),
+    {ok, ensure_reload_timer(
+           #{waiting => Waiting,
+             running => Running,
+             stopped => Stopped,
+             trefs => #{},
+             orders => Orders
+            })}.
+
+-spec load_all_servers(list(server_options())) -> {servers(), servers(), servers()}.
+load_all_servers(ServerL) ->
+    load_all_servers(ServerL, #{}, #{}, #{}).
+
+load_all_servers([#{name := Name} = Options | More], Waiting, Running, Stopped) ->
+    case emqx_exhook_server:load(Name, Options) of
+        {ok, ServerState} ->
+            save(Name, ServerState),
+            load_all_servers(More, Waiting, Running#{Name => Options}, Stopped);
+        {error, _} ->
+            load_all_servers(More, Waiting#{Name => Options}, Running, Stopped);
+        disable ->
+            load_all_servers(More, Waiting, Running, Stopped#{Name => Options})
+    end;
+
+load_all_servers([], Waiting, Running, Stopped) ->
+    {Waiting, Running, Stopped}.
+
+handle_call(list, _From, State = #{running := Running,
+                                   waiting := Waiting,
+                                   stopped := Stopped,
+                                   orders := Orders}) ->
+
+    R = get_servers_info(running, Running),
+    W = get_servers_info(waiting, Waiting),
+    S = get_servers_info(stopped, Stopped),
+
+    Servers = R ++ W ++ S,
+    OrderServers = sort_name_by_order(Servers, Orders),
+
+    {reply, OrderServers, State};
+
+handle_call({update_config, {move, _Name, _Direct, _Related}, NewConfL},
+            _From,
+            State) ->
+    Orders = reorder(NewConfL),
+    {reply, ok, State#{orders := Orders}};
+
+handle_call({update_config, {delete, ToDelete}, _}, _From, State) ->
+    {ok, #{orders := Orders,
+           stopped := Stopped
+          } = State2} = do_unload_server(ToDelete, State),
+
+    State3 = State2#{stopped := maps:remove(ToDelete, Stopped),
+                     orders := maps:remove(ToDelete, Orders)
+                    },
+
+    {reply, ok, State3};
+
+handle_call({update_config, {add, RawConf}, NewConfL},
+            _From,
+            #{running := Running, waiting := Waitting, stopped := Stopped} = State) ->
+    {_, #{name := Name} = Conf} = emqx_config:check_config(?MODULE, RawConf),
+
+    case emqx_exhook_server:load(Name, Conf) of
+        {ok, ServerState} ->
+            save(Name, ServerState),
+            Status = running,
+            Hooks = hooks(Name),
+            State2 = State#{running := Running#{Name => Conf}};
+        {error, _} ->
+            Status = running,
+            Hooks = [],
+            StateT = State#{waiting := Waitting#{Name => Conf}},
+            State2 = ensure_reload_timer(StateT);
+        disable ->
+            Status = stopped,
+            Hooks = [],
+            State2 = State#{stopped := Stopped#{Name => Conf}}
+    end,
+    Orders = reorder(NewConfL),
+    Resulte = maps:merge(Conf, #{status => Status, hooks => Hooks}),
+    {reply, Resulte, State2#{orders := Orders}};
+
+handle_call({lookup, Name}, _From, State) ->
+    case where_is_server(Name, State) of
+        not_found ->
+            Result = not_found;
+        {Where, #{Name := Conf}} ->
+            Result = maps:merge(Conf,
+                                #{ status => Where
+                                 , hooks => hooks(Name)
+                                 })
+    end,
+    {reply, Result, State};
+
+handle_call({update_config, {update, Name, _Conf}, NewConfL}, _From, State) ->
+    {Result, State2} = restart_server(Name, NewConfL, State),
+    {reply, Result, State2};
+
+handle_call({update_config, {enable, Name, _Enable}, NewConfL}, _From, State) ->
+    {Result, State2} = restart_server(Name, NewConfL, State),
+    {reply, Result, State2};
+
+handle_call({server_status, Name}, _From, State) ->
+    case where_is_server(Name, State) of
+        not_found ->
+            Result = not_found;
+        {Status, _} ->
+            Result = Status
+    end,
+    {reply, Result, State};
+
+handle_call(all_servers_status, _From, #{running := Running,
+                                         waiting := Waiting,
+                                         stopped := Stopped} = State) ->
+    {reply, #{running => maps:keys(Running),
+              waiting => maps:keys(Waiting),
+              stopped => maps:keys(Stopped)}, State};
+
+handle_call(_Request, _From, State) ->
+    Reply = ok,
+    {reply, Reply, State}.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+handle_info({timeout, _Ref, {reload, Name}}, State) ->
+    {Result, NState} = do_load_server(Name, State),
+    case Result of
+        ok ->
+            {noreply, NState};
+        {error, not_found} ->
+            {noreply, NState};
+        {error, Reason} ->
+            ?LOG(warning, "Failed to reload exhook callback server \"~ts\", "
+                 "Reason: ~0p", [Name, Reason]),
+            {noreply, ensure_reload_timer(NState)}
+    end;
+
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+terminate(_Reason, State = #{running := Running}) ->
+    _ = maps:fold(fun(Name, _, AccIn) ->
+                          {ok, NAccIn} =  do_unload_server(Name, AccIn),
+                          NAccIn
+                  end, State, Running),
+    _ = unload_exhooks(),
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%--------------------------------------------------------------------
+%% Internal funcs
+%%--------------------------------------------------------------------
+
+unload_exhooks() ->
+    [emqx:unhook(Name, {M, F}) ||
+        {Name, {M, F, _A}} <- ?ENABLED_HOOKS].
+
+-spec do_load_server(server_name(), state()) -> {{error, not_found}, state()}
+              | {{error, already_started}, state()}
+              | {ok, state()}.
+do_load_server(Name, State = #{orders := Orders}) ->
+    case where_is_server(Name, State) of
+        not_found ->
+            {{error, not_found}, State};
+        {running, _} ->
+            {ok, State};
+        {Where, Map} ->
+            State2 = clean_reload_timer(Name, State),
+            {Options, Map2} = maps:take(Name, Map),
+            State3 = State2#{Where := Map2},
+            #{running := Running,
+              stopped := Stopped} = State3,
+            case emqx_exhook_server:load(Name, Options) of
+                    {ok, ServerState} ->
+                        save(Name, ServerState),
+                        update_order(Orders),
+                        ?LOG(info, "Load exhook callback server "
+                             "\"~ts\" successfully!", [Name]),
+                    {ok, State3#{running := maps:put(Name, Options, Running)}};
+                {error, Reason} ->
+                    {{error, Reason}, State};
+                disable ->
+                    {ok, State3#{stopped := Stopped#{Name => Options}}}
+            end
+    end.
+
+-spec do_unload_server(server_name(), state()) -> {ok, state()}.
+do_unload_server(Name, #{stopped := Stopped} = State) ->
+    case where_is_server(Name, State) of
+        {stopped, _} -> {ok, State};
+        {waiting, Waiting} ->
+            {Options, Waiting2} = maps:take(Name, Waiting),
+            {ok, clean_reload_timer(Name,
+                                    State#{waiting := Waiting2,
+                                           stopped := maps:put(Name, Options, Stopped)
+                                          }
+                                   )};
+        {running, Running} ->
+            Service = server(Name),
+            ok = unsave(Name),
+            ok = emqx_exhook_server:unload(Service),
+            {Options, Running2} = maps:take(Name, Running),
+            {ok, State#{running := Running2,
+                        stopped := maps:put(Name, Options, Stopped)
+                       }};
+        not_found -> {ok, State}
+    end.
+
+-spec ensure_reload_timer(state()) -> state().
+ensure_reload_timer(State = #{waiting := Waiting,
+                              stopped := Stopped,
+                              trefs := TRefs}) ->
+    Iter = maps:iterator(Waiting),
+
+    {Waitting2, Stopped2, TRefs2} =
+        ensure_reload_timer(maps:next(Iter), Waiting, Stopped, TRefs),
+
+    State#{waiting := Waitting2,
+           stopped := Stopped2,
+           trefs := TRefs2}.
+
+ensure_reload_timer(none, Waiting, Stopped, TimerRef) ->
+    {Waiting, Stopped, TimerRef};
+
+ensure_reload_timer({Name, #{auto_reconnect := Intv}, Iter},
+                    Waiting,
+                    Stopped,
+                    TimerRef) ->
+    Next = maps:next(Iter),
+    case maps:is_key(Name, TimerRef) of
+        true ->
+            ensure_reload_timer(Next, Waiting, Stopped, TimerRef);
+        _ ->
+            Ref = erlang:start_timer(Intv, self(), {reload, Name}),
+            TimerRef2 = maps:put(Name, Ref, TimerRef),
+            ensure_reload_timer(Next, Waiting, Stopped, TimerRef2)
+    end;
+
+ensure_reload_timer({Name, Opts, Iter}, Waiting, Stopped, TimerRef) ->
+    ensure_reload_timer(maps:next(Iter),
+                        maps:remove(Name, Waiting),
+                        maps:put(Name, Opts, Stopped),
+                        TimerRef).
+
+-spec clean_reload_timer(server_name(), state()) -> state().
+clean_reload_timer(Name, State = #{trefs := TRefs}) ->
+    case maps:take(Name, TRefs) of
+        error -> State;
+        {TRef, NTRefs} ->
+            _ = erlang:cancel_timer(TRef),
+            State#{trefs := NTRefs}
+    end.
+
+-spec do_move(binary(), move_direct(), binary(), list(server_options())) ->
+          not_found | list(server_options()).
+do_move(Name, Direct, ToName, ConfL) ->
+    move(ConfL, Name, Direct, ToName, []).
+
+move([#{<<"name">> := Name} = Server | T], Name, Direct, ToName, HeadL) ->
+    move_to(Direct, ToName, Server, lists:reverse(HeadL) ++ T);
+
+move([Server | T], Name, Direct, ToName, HeadL) ->
+    move(T, Name, Direct, ToName, [Server | HeadL]);
+
+move([], _Name, _Direct, _ToName, _HeadL) ->
+    not_found.
+
+move_to(top, _, Server, ServerL) ->
+    [Server | ServerL];
+
+move_to(bottom, _, Server, ServerL) ->
+    ServerL ++ [Server];
+
+move_to(Direct, ToName, Server, ServerL) ->
+    move_to(ServerL, Direct, ToName, Server, []).
+
+move_to([#{<<"name">> := Name} | _] = T, before, Name, Server, HeadL) ->
+    lists:reverse(HeadL) ++ [Server | T];
+
+move_to([#{<<"name">> := Name} = H | T], 'after', Name, Server, HeadL) ->
+    lists:reverse(HeadL) ++ [H, Server | T];
+
+move_to([H | T], Direct, Name, Server, HeadL) ->
+    move_to(T, Direct, Name, Server, [H | HeadL]);
+
+move_to([], _Direct, _Name, _Server, _HeadL) ->
+    not_found.
+
+-spec reorder(list(server_options())) -> orders().
+reorder(ServerL) ->
+    Orders = reorder(ServerL, 1, #{}),
+    update_order(Orders),
+    Orders.
+
+reorder([#{name := Name} | T], Order, Orders) ->
+    reorder(T, Order + 1, Orders#{Name => Order});
+
+reorder([], _Order, Orders) ->
+    Orders.
+
+get_servers_info(Status, Map) ->
+    Fold = fun(Name, Conf, Acc) ->
+                   [maps:merge(Conf, #{status => Status,
+                                       hooks => hooks(Name)}) | Acc]
+           end,
+    maps:fold(Fold, [], Map).
+
+
+where_is_server(Name, #{running := Running}) when is_map_key(Name, Running) ->
+    {running, Running};
+
+where_is_server(Name, #{waiting := Waiting}) when is_map_key(Name, Waiting) ->
+    {waiting, Waiting};
+
+where_is_server(Name, #{stopped := Stopped}) when is_map_key(Name, Stopped) ->
+    {stopped, Stopped};
+
+where_is_server(_, _) ->
+    not_found.
+
+-type replace_fun() :: fun((server_options()) -> server_options()).
+
+-spec replace_conf(binary(), replace_fun(), list(server_options())) -> not_found
+              | list(server_options()).
+replace_conf(Name, ReplaceFun, ConfL) ->
+    replace_conf(ConfL, Name, ReplaceFun, []).
+
+replace_conf([#{<<"name">> := Name} = H | T], Name, ReplaceFun, HeadL) ->
+    New = ReplaceFun(H),
+    lists:reverse(HeadL) ++ [New | T];
+
+replace_conf([H | T], Name, ReplaceFun, HeadL) ->
+    replace_conf(T, Name, ReplaceFun, [H | HeadL]);
+
+replace_conf([], _, _, _) ->
+    not_found.
+
+-spec restart_server(binary(), list(server_options()), state()) -> {ok, state()}
+              | {{error, term()}, state()}.
+restart_server(Name, ConfL, State) ->
+    case lists:search(fun(#{name := CName}) -> CName =:= Name end, ConfL) of
+        false ->
+            {{error, not_found}, State};
+        {value, Conf} ->
+            case where_is_server(Name, State) of
+                not_found ->
+                    {{error, not_found}, State};
+                {Where, Map} ->
+                    State2 = State#{Where := Map#{Name := Conf}},
+                    {ok, State3} = do_unload_server(Name, State2),
+                    case do_load_server(Name, State3) of
+                        {ok, State4} ->
+                            {ok, State4};
+                        {Error, State4} ->
+                            {Error, State4}
+                    end
+            end
+    end.
+
+sort_name_by_order(Names, Orders) ->
+    lists:sort(fun(A, B) when is_binary(A) ->
+                       maps:get(A, Orders) < maps:get(B, Orders);
+                  (#{name := A}, #{name := B}) ->
+                       maps:get(A, Orders) < maps:get(B, Orders)
+               end,
+               Names).
+%%--------------------------------------------------------------------
+%% Server state persistent
+save(Name, ServerState) ->
+    Saved = persistent_term:get(?APP, []),
+    persistent_term:put(?APP, lists:reverse([Name | Saved])),
+    persistent_term:put({?APP, Name}, ServerState).
+
+unsave(Name) ->
+    case persistent_term:get(?APP, []) of
+        [] ->
+            ok;
+        Saved ->
+            case lists:member(Name, Saved) of
+                false ->
+                    ok;
+                true ->
+                    persistent_term:put(?APP, lists:delete(Name, Saved))
+            end
+    end,
+    persistent_term:erase({?APP, Name}),
+    ok.
+
+running() ->
+    persistent_term:get(?APP, []).
+
+server(Name) ->
+    case persistent_term:get({?APP, Name}, undefined) of
+        undefined -> undefined;
+        Service -> Service
+    end.
+
+update_order(Orders) ->
+    Running = running(),
+    Running2 = sort_name_by_order(Running, Orders),
+    persistent_term:put(?APP, Running2).
+
+hooks(Name) ->
+    case server(Name) of
+        undefined ->
+            [];
+        Service ->
+            emqx_exhook_server:hookpoints(Service)
+    end.
diff --git a/apps/emqx_exhook/src/emqx_exhook_mngr.erl b/apps/emqx_exhook/src/emqx_exhook_mngr.erl
deleted file mode 100644
index cd2658f93..000000000
--- a/apps/emqx_exhook/src/emqx_exhook_mngr.erl
+++ /dev/null
@@ -1,329 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
-%% @doc Manage the server status and reload strategy
--module(emqx_exhook_mngr).
-
--behaviour(gen_server).
-
--include("emqx_exhook.hrl").
--include_lib("emqx/include/logger.hrl").
-
-%% APIs
--export([start_link/3]).
-
-%% Mgmt API
--export([ enable/2
-        , disable/2
-        , list/1
-        ]).
-
-%% Helper funcs
--export([ running/0
-        , server/1
-        , put_request_failed_action/1
-        , get_request_failed_action/0
-        , put_pool_size/1
-        , get_pool_size/0
-        ]).
-
-%% gen_server callbacks
--export([ init/1
-        , handle_call/3
-        , handle_cast/2
-        , handle_info/2
-        , terminate/2
-        , code_change/3
-        ]).
-
--record(state, {
-          %% Running servers
-          running :: map(),         %% XXX: server order?
-          %% Wait to reload servers
-          waiting :: map(),
-          %% Marked stopped servers
-          stopped :: map(),
-          %% Auto reconnect timer interval
-          auto_reconnect :: false | non_neg_integer(),
-          %% Request options
-          request_options :: grpc_client:options(),
-          %% Timer references
-          trefs :: map()
-         }).
-
--type servers() :: [{Name :: atom(), server_options()}].
-
--type server_options() :: [ {scheme, http | https}
-                          | {host, string()}
-                          | {port, inet:port_number()}
-                          ].
-
--define(DEFAULT_TIMEOUT, 60000).
-
--define(CNTER, emqx_exhook_counter).
-
-%%--------------------------------------------------------------------
-%% APIs
-%%--------------------------------------------------------------------
-
--spec start_link(servers(), false | non_neg_integer(), grpc_client:options())
-    ->ignore
-     | {ok, pid()}
-     | {error, any()}.
-start_link(Servers, AutoReconnect, ReqOpts) ->
-    gen_server:start_link(?MODULE, [Servers, AutoReconnect, ReqOpts], []).
-
--spec enable(pid(), binary()) -> ok | {error, term()}.
-enable(Pid, Name) ->
-    call(Pid, {load, Name}).
-
--spec disable(pid(), binary()) -> ok | {error, term()}.
-disable(Pid, Name) ->
-    call(Pid, {unload, Name}).
-
-list(Pid) ->
-    call(Pid, list).
-
-call(Pid, Req) ->
-    gen_server:call(Pid, Req, ?DEFAULT_TIMEOUT).
-
-%%--------------------------------------------------------------------
-%% gen_server callbacks
-%%--------------------------------------------------------------------
-
-init([Servers, AutoReconnect, ReqOpts0]) ->
-    process_flag(trap_exit, true),
-    %% XXX: Due to the ExHook Module in the enterprise,
-    %% this process may start multiple times and they will share this table
-    try
-        _ = ets:new(?CNTER, [named_table, public]), ok
-    catch
-        error:badarg:_ ->
-            ok
-    end,
-
-    %% put the global option
-    put_request_failed_action(
-      maps:get(request_failed_action, ReqOpts0, deny)
-     ),
-    put_pool_size(
-      maps:get(pool_size, ReqOpts0, erlang:system_info(schedulers))
-     ),
-
-    %% Load the hook servers
-    ReqOpts = maps:without([request_failed_action], ReqOpts0),
-    {Waiting, Running} = load_all_servers(Servers, ReqOpts),
-    {ok, ensure_reload_timer(
-           #state{waiting = Waiting,
-                  running = Running,
-                  stopped = #{},
-                  request_options = ReqOpts,
-                  auto_reconnect = AutoReconnect,
-                  trefs = #{}
-                 }
-          )}.
-
-%% @private
-load_all_servers(Servers, ReqOpts) ->
-    load_all_servers(Servers, ReqOpts, #{}, #{}).
-load_all_servers([], _Request, Waiting, Running) ->
-    {Waiting, Running};
-load_all_servers([#{name := Name0} = Options0 | More], ReqOpts, Waiting, Running) ->
-    Name = iolist_to_binary(Name0),
-    Options = Options0#{name => Name},
-    {NWaiting, NRunning} =
-        case emqx_exhook_server:load(Name, Options, ReqOpts) of
-            {ok, ServerState} ->
-                save(Name, ServerState),
-                {Waiting, Running#{Name => Options}};
-            {error, _} ->
-                {Waiting#{Name => Options}, Running}
-        end,
-    load_all_servers(More, ReqOpts, NWaiting, NRunning).
-
-handle_call({load, Name}, _From, State) ->
-    {Result, NState} = do_load_server(Name, State),
-    {reply, Result, NState};
-
-handle_call({unload, Name}, _From, State) ->
-    case do_unload_server(Name, State) of
-        {error, Reason} ->
-            {reply, {error, Reason}, State};
-        {ok, NState} ->
-            {reply, ok, NState}
-    end;
-
-handle_call(list, _From, State = #state{
-                                    running = Running,
-                                    waiting = Waiting,
-                                    stopped = Stopped}) ->
-    ServerNames = maps:keys(Running)
-                    ++ maps:keys(Waiting)
-                    ++ maps:keys(Stopped),
-    {reply, ServerNames, State};
-
-handle_call(_Request, _From, State) ->
-    Reply = ok,
-    {reply, Reply, State}.
-
-handle_cast(_Msg, State) ->
-    {noreply, State}.
-
-handle_info({timeout, _Ref, {reload, Name}}, State) ->
-    {Result, NState} = do_load_server(Name, State),
-    case Result of
-        ok ->
-            {noreply, NState};
-        {error, not_found} ->
-            {noreply, NState};
-        {error, Reason} ->
-            ?SLOG(warning, #{msg => "failed_to_reload_exhook_callback_server",
-                             server_name => Name,
-                             reason => Reason}),
-            {noreply, ensure_reload_timer(NState)}
-    end;
-
-handle_info(_Info, State) ->
-    {noreply, State}.
-
-terminate(_Reason, State = #state{running = Running}) ->
-    _ = maps:fold(fun(Name, _, AccIn) ->
-             case do_unload_server(Name, AccIn) of
-                {ok, NAccIn} -> NAccIn;
-                 _ -> AccIn
-             end
-        end, State, Running),
-    _ = unload_exhooks(),
-    ok.
-
-code_change(_OldVsn, State, _Extra) ->
-    {ok, State}.
-
-%%--------------------------------------------------------------------
-%% Internal funcs
-%%--------------------------------------------------------------------
-
-unload_exhooks() ->
-    [emqx:unhook(Name, {M, F}) ||
-     {Name, {M, F, _A}} <- ?ENABLED_HOOKS].
-
-do_load_server(Name, State0 = #state{
-                                 waiting = Waiting,
-                                 running = Running,
-                                 stopped = Stopped,
-                                 request_options = ReqOpts}) ->
-    State = clean_reload_timer(Name, State0),
-    case maps:get(Name, Running, undefined) of
-        undefined ->
-            case maps:get(Name, Stopped,
-                          maps:get(Name, Waiting, undefined)) of
-                undefined ->
-                    {{error, not_found}, State};
-                Options ->
-                    case emqx_exhook_server:load(Name, Options, ReqOpts) of
-                        {ok, ServerState} ->
-                            save(Name, ServerState),
-                            ?SLOG(info, #{msg => "load_exhook_callback_server_successfully",
-                                          server_name => Name}),
-                            {ok, State#state{
-                                   running = maps:put(Name, Options, Running),
-                                   waiting = maps:remove(Name, Waiting),
-                                   stopped = maps:remove(Name, Stopped)
-                                  }
-                            };
-                        {error, Reason} ->
-                            {{error, Reason}, State}
-                    end
-            end;
-        _ ->
-            {{error, already_started}, State}
-    end.
-
-do_unload_server(Name, State = #state{running = Running, stopped = Stopped}) ->
-    case maps:take(Name, Running) of
-        error -> {error, not_running};
-        {Options, NRunning} ->
-            ok = emqx_exhook_server:unload(server(Name)),
-            ok = unsave(Name),
-            {ok, State#state{running = NRunning,
-                             stopped = maps:put(Name, Options, Stopped)
-                            }}
-    end.
-
-ensure_reload_timer(State = #state{auto_reconnect = false}) ->
-    State;
-ensure_reload_timer(State = #state{waiting = Waiting,
-                                   trefs = TRefs,
-                                   auto_reconnect = Intv}) ->
-    NRefs = maps:fold(fun(Name, _, AccIn) ->
-        case maps:get(Name, AccIn, undefined) of
-            undefined ->
-                Ref = erlang:start_timer(Intv, self(), {reload, Name}),
-                AccIn#{Name => Ref};
-            _HasRef ->
-                AccIn
-        end
-    end, TRefs, Waiting),
-    State#state{trefs = NRefs}.
-
-clean_reload_timer(Name, State = #state{trefs = TRefs}) ->
-    case maps:take(Name, TRefs) of
-        error -> State;
-        {TRef, NTRefs} ->
-            _ = erlang:cancel_timer(TRef),
-            State#state{trefs = NTRefs}
-    end.
-
-%%--------------------------------------------------------------------
-%% Server state persistent
-
-put_request_failed_action(Val) ->
-    persistent_term:put({?APP, request_failed_action}, Val).
-
-get_request_failed_action() ->
-    persistent_term:get({?APP, request_failed_action}).
-
-put_pool_size(Val) ->
-    persistent_term:put({?APP, pool_size}, Val).
-
-get_pool_size() ->
-    %% Avoid the scenario that the parameter is not set after
-    %% the hot upgrade completed.
-    persistent_term:get({?APP, pool_size}, erlang:system_info(schedulers)).
-
-save(Name, ServerState) ->
-    Saved = persistent_term:get(?APP, []),
-    persistent_term:put(?APP, lists:reverse([Name | Saved])),
-    persistent_term:put({?APP, Name}, ServerState).
-
-unsave(Name) ->
-    case persistent_term:get(?APP, []) of
-        [] ->
-            persistent_term:erase(?APP);
-        Saved ->
-            persistent_term:put(?APP, lists:delete(Name, Saved))
-    end,
-    persistent_term:erase({?APP, Name}),
-    ok.
-
-running() ->
-    persistent_term:get(?APP, []).
-
-server(Name) ->
-    case catch persistent_term:get({?APP, Name}) of
-        {'EXIT', {badarg,_}} -> undefined;
-        Service -> Service
-    end.
diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl
index 21ca5c3f0..eb75a10ff 100644
--- a/apps/emqx_exhook/src/emqx_exhook_schema.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl
@@ -32,61 +32,58 @@
 
 -reflect_type([duration/0]).
 
--export([namespace/0, roots/0, fields/1]).
+-export([namespace/0, roots/0, fields/1, server_config/0]).
 
-namespace() -> exhook.
+namespace() -> emqx_exhook.
 
-roots() -> [exhook].
+roots() -> [emqx_exhook].
 
-fields(exhook) ->
-    [ {request_failed_action,
-       sc(hoconsc:enum([deny, ignore]),
-          #{default => deny})}
-    , {request_timeout,
-       sc(duration(),
-          #{default => "5s"})}
-    , {auto_reconnect,
-       sc(hoconsc:union([false, duration()]),
-          #{ default => "60s"
-           })}
-    , {pool_size,
-       sc(integer(),
-          #{ nullable => true
-           })}
-    , {servers,
-       sc(hoconsc:array(ref(servers)),
+fields(emqx_exhook) ->
+    [{servers,
+      sc(hoconsc:array(ref(server)),
           #{default => []})}
     ];
 
-fields(servers) ->
-    [ {name,
-       sc(string(),
-          #{})}
-    , {url,
-       sc(string(),
-          #{})}
+fields(server) ->
+    [ {name, sc(binary(), #{})}
+    , {enable, sc(boolean(), #{default => true})}
+    , {url, sc(binary(), #{})}
+    , {request_timeout,
+       sc(duration(), #{default => "5s"})}
+    , {failed_action, failed_action()}
     , {ssl,
-       sc(ref(ssl_conf),
-          #{})}
+       sc(ref(ssl_conf), #{})}
+    , {auto_reconnect,
+       sc(hoconsc:union([false, duration()]),
+          #{default => "60s"})}
+    , {pool_size,
+       sc(integer(), #{default => 8, example => 8})}
     ];
 
 fields(ssl_conf) ->
-    [ {cacertfile,
-       sc(string(),
-          #{})
-       }
+    [ {enable, sc(boolean(), #{default => true})}
+    , {cacertfile,
+       sc(binary(),
+          #{example => <<"{{ platform_etc_dir }}/certs/cacert.pem">>})
+      }
     , {certfile,
-       sc(string(),
-          #{})
-       }
+       sc(binary(),
+          #{example => <<"{{ platform_etc_dir }}/certs/cert.pem">>})
+      }
     , {keyfile,
-       sc(string(),
-          #{})}
+       sc(binary(),
+          #{example => <<"{{ platform_etc_dir }}/certs/key.pem">>})}
     ].
 
 %% types
-
 sc(Type, Meta) -> Meta#{type => Type}.
 
 ref(Field) ->
     hoconsc:ref(?MODULE, Field).
+
+failed_action() ->
+    sc(hoconsc:enum([deny, ignore]),
+       #{default => deny}).
+
+server_config() ->
+    fields(server).
diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl
index b66b30a26..fff3172d3 100644
--- a/apps/emqx_exhook/src/emqx_exhook_server.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_server.erl
@@ -24,7 +24,7 @@
 -define(PB_CLIENT_MOD, emqx_exhook_v_1_hook_provider_client).
 
 %% Load/Unload
--export([ load/3
+-export([ load/2
         , unload/1
         ]).
 
@@ -33,23 +33,24 @@
 
 %% Infos
 -export([ name/1
+        , hookpoints/1
         , format/1
+        , failed_action/1
         ]).
 
--record(server, {
-          %% Server name (equal to grpc client channel name)
-          name :: binary(),
-          %% The function options
-          options :: map(),
-          %% gRPC channel pid
-          channel :: pid(),
-          %% Registered hook names and options
-          hookspec :: #{hookpoint() => map()},
-          %% Metrcis name prefix
-          prefix :: list()
-       }).
 
--type server() :: #server{}.
+-type server() :: #{%% Server name (equal to grpc client channel name)
+                    name := binary(),
+                    %% The function options
+                    options := map(),
+                    %% gRPC channel pid
+                    channel := pid(),
+                    %% Registered hook names and options
+                    hookspec := #{hookpoint() => map()},
+                    %% Metrcis name prefix
+                    prefix := list()
+                   }.
+
 
 -type hookpoint() :: 'client.connect'
                    | 'client.connack'
@@ -81,9 +82,13 @@
 %% Load/Unload APIs
 %%--------------------------------------------------------------------
 
--spec load(binary(), map(), map()) -> {ok, server()} | {error, term()} .
-load(Name, Opts0, ReqOpts) ->
-    {SvrAddr, ClientOpts} = channel_opts(Opts0),
+-spec load(binary(), map()) -> {ok, server()} | {error, term()} | disable.
+load(_Name, #{enable := false}) ->
+    disable;
+
+load(Name, #{request_timeout := Timeout, failed_action := FailedAction} = Opts) ->
+    ReqOpts = #{timeout => Timeout, failed_action => FailedAction},
+    {SvrAddr, ClientOpts} = channel_opts(Opts),
     case emqx_exhook_sup:start_grpc_client_channel(
            Name,
            SvrAddr,
@@ -92,16 +97,15 @@ load(Name, Opts0, ReqOpts) ->
             case do_init(Name, ReqOpts) of
                 {ok, HookSpecs} ->
                     %% Reigster metrics
-                    Prefix = lists:flatten(
-                               io_lib:format("exhook.~ts.", [Name])),
+                    Prefix = lists:flatten(io_lib:format("exhook.~ts.", [Name])),
                     ensure_metrics(Prefix, HookSpecs),
                     %% Ensure hooks
                     ensure_hooks(HookSpecs),
-                    {ok, #server{name = Name,
-                                 options = ReqOpts,
-                                 channel = _ChannPoolPid,
-                                 hookspec = HookSpecs,
-                                 prefix = Prefix }};
+                    {ok, #{name => Name,
+                           options => ReqOpts,
+                           channel => _ChannPoolPid,
+                           hookspec => HookSpecs,
+                           prefix => Prefix }};
                 {error, _} = E ->
                     emqx_exhook_sup:stop_grpc_client_channel(Name), E
             end;
@@ -110,14 +114,16 @@ load(Name, Opts0, ReqOpts) ->
 
 %% @private
 channel_opts(Opts = #{url := URL}) ->
-    ClientOpts = #{pool_size => emqx_exhook_mngr:get_pool_size()},
+    ClientOpts = maps:merge(#{pool_size => erlang:system_info(schedulers)},
+                            Opts),
     case uri_string:parse(URL) of
-        #{scheme := "http", host := Host, port := Port} ->
+        #{scheme := <<"http">>, host := Host, port := Port} ->
             {format_http_uri("http", Host, Port), ClientOpts};
-        #{scheme := "https", host := Host, port := Port} ->
+        #{scheme := <<"https">>, host := Host, port := Port} ->
             SslOpts =
                 case maps:get(ssl, Opts, undefined) of
                     undefined -> [];
+                    #{enable := false} -> [];
                     MapOpts ->
                         filter(
                           [{cacertfile, maps:get(cacertfile, MapOpts, undefined)},
@@ -131,8 +137,8 @@ channel_opts(Opts = #{url := URL}) ->
                                 transport_opts => SslOpts}
                            },
             {format_http_uri("https", Host, Port), NClientOpts};
-        _ ->
-            error(bad_server_url)
+        Error ->
+            error({bad_server_url, URL, Error})
     end.
 
 format_http_uri(Scheme, Host, Port) ->
@@ -142,7 +148,7 @@ filter(Ls) ->
     [ E || E <- Ls, E /= undefined].
 
 -spec unload(server()) -> ok.
-unload(#server{name = Name, options = ReqOpts, hookspec = HookSpecs}) ->
+unload(#{name := Name, options := ReqOpts, hookspec := HookSpecs}) ->
     _ = do_deinit(Name, ReqOpts),
     _ = may_unload_hooks(HookSpecs),
     _ = emqx_exhook_sup:stop_grpc_client_channel(Name),
@@ -155,7 +161,7 @@ do_deinit(Name, ReqOpts) ->
 do_init(ChannName, ReqOpts) ->
     %% BrokerInfo defined at: exhook.protos
     BrokerInfo = maps:with([version, sysdescr, uptime, datetime],
-                        maps:from_list(emqx_sys:info())),
+                           maps:from_list(emqx_sys:info())),
     Req = #{broker => BrokerInfo},
     case do_call(ChannName, 'on_provider_loaded', Req, ReqOpts) of
         {ok, InitialResp} ->
@@ -227,7 +233,7 @@ may_unload_hooks(HookSpecs) ->
         end
     end, maps:keys(HookSpecs)).
 
-format(#server{name = Name, hookspec = Hooks}) ->
+format(#{name := Name, hookspec := Hooks}) ->
     lists:flatten(
       io_lib:format("name=~ts, hooks=~0p, active=true", [Name, Hooks])).
 
@@ -235,15 +241,17 @@ format(#server{name = Name, hookspec = Hooks}) ->
 %% APIs
 %%--------------------------------------------------------------------
 
-name(#server{name = Name}) ->
+name(#{name := Name}) ->
     Name.
 
--spec call(hookpoint(), map(), server())
-  -> ignore
-   | {ok, Resp :: term()}
-   | {error, term()}.
-call(Hookpoint, Req, #server{name = ChannName, options = ReqOpts,
-                             hookspec = Hooks, prefix = Prefix}) ->
+hookpoints(#{hookspec := Hooks}) ->
+    maps:keys(Hooks).
+
+-spec call(hookpoint(), map(), server()) -> ignore
+              | {ok, Resp :: term()}
+              | {error, term()}.
+call(Hookpoint, Req, #{name := ChannName, options := ReqOpts,
+                       hookspec := Hooks, prefix := Prefix}) ->
     GrpcFunc = hk2func(Hookpoint),
     case maps:get(Hookpoint, Hooks, undefined) of
         undefined -> ignore;
@@ -299,6 +307,9 @@ do_call(ChannName, Fun, Req, ReqOpts) ->
             {error, Reason}
     end.
 
+failed_action(#{options := Opts}) ->
+    maps:get(failed_action, Opts).
+
 %%--------------------------------------------------------------------
 %% Internal funcs
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl
index ca8d7c856..74b97ca1a 100644
--- a/apps/emqx_exhook/src/emqx_exhook_sup.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_sup.erl
@@ -42,25 +42,10 @@ start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init([]) ->
-    Mngr = ?CHILD(emqx_exhook_mngr, worker,
-                  [servers(), auto_reconnect(), request_options()]),
+    _ = emqx_exhook_mgr:init_counter_table(),
+    Mngr = ?CHILD(emqx_exhook_mgr, worker, []),
     {ok, {{one_for_one, 10, 100}, [Mngr]}}.
 
-servers() ->
-    env(servers, []).
-
-auto_reconnect() ->
-    env(auto_reconnect, 60000).
-
-request_options() ->
-    #{timeout => env(request_timeout, 5000),
-      request_failed_action => env(request_failed_action, deny),
-      pool_size => env(pool_size, erlang:system_info(schedulers))
-     }.
-
-env(Key, Def) ->
-    emqx_conf:get([exhook, Key], Def).
-
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
index dd020ce85..a58ccd4bd 100644
--- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
+++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
@@ -21,14 +21,14 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard).
 
 -define(CONF_DEFAULT, <<"
-exhook: {
-    servers: [
-        { name: \"default\"
-          url: \"http://127.0.0.1:9000\"
-        }
-    ]
+emqx_exhook
+{servers = [
+            {name = default,
+             url = \"http://127.0.0.1:9000\"
+            }]
 }
 ">>).
 
@@ -39,27 +39,53 @@ exhook: {
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Cfg) ->
+    application:load(emqx_conf),
+    ok = ekka:start(),
+    ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity),
+    meck:new(emqx_alarm, [non_strict, passthrough, no_link]),
+    meck:expect(emqx_alarm, activate, 3, ok),
+    meck:expect(emqx_alarm, deactivate, 3, ok),
+
     _ = emqx_exhook_demo_svr:start(),
     ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
     emqx_common_test_helpers:start_apps([emqx_exhook]),
     Cfg.
 
 end_per_suite(_Cfg) ->
+    ekka:stop(),
+    mria:stop(),
+    mria_mnesia:delete_schema(),
+    meck:unload(emqx_alarm),
+
     emqx_common_test_helpers:stop_apps([emqx_exhook]),
     emqx_exhook_demo_svr:stop().
 
+init_per_testcase(_, Config) ->
+    {ok, _} = emqx_cluster_rpc:start_link(),
+    timer:sleep(200),
+    Config.
+
+end_per_testcase(_, Config) ->
+    case erlang:whereis(node()) of
+        undefined -> ok;
+        P ->
+            erlang:unlink(P),
+            erlang:exit(P, kill)
+    end,
+    Config.
+
 %%--------------------------------------------------------------------
 %% Test cases
 %%--------------------------------------------------------------------
 
 t_noserver_nohook(_) ->
-    emqx_exhook:disable(<<"default">>),
+    emqx_exhook_mgr:disable(<<"default">>),
     ?assertEqual([], ets:tab2list(emqx_hooks)),
-    ok = emqx_exhook:enable(<<"default">>),
+    {ok, _} = emqx_exhook_mgr:enable(<<"default">>),
     ?assertNotEqual([], ets:tab2list(emqx_hooks)).
 
 t_access_failed_if_no_server_running(_) ->
-    emqx_exhook:disable(<<"default">>),
+    emqx_exhook_mgr:disable(<<"default">>),
     ClientInfo = #{clientid => <<"user-id-1">>,
                    username => <<"usera">>,
                    peerhost => {127,0,0,1},
@@ -76,30 +102,7 @@ t_access_failed_if_no_server_running(_) ->
     Message = emqx_message:make(<<"t/1">>, <<"abc">>),
     ?assertMatch({stop, Message},
                  emqx_exhook_handler:on_message_publish(Message)),
-    emqx_exhook:enable(<<"default">>).
-
-t_cli_list(_) ->
-    meck_print(),
-    ?assertEqual( [[emqx_exhook_server:format(emqx_exhook_mngr:server(Name)) || Name  <- emqx_exhook:list()]]
-                , emqx_exhook_cli:cli(["server", "list"])
-                ),
-    unmeck_print().
-
-t_cli_enable_disable(_) ->
-    meck_print(),
-    ?assertEqual([already_started], emqx_exhook_cli:cli(["server", "enable", "default"])),
-    ?assertEqual(ok, emqx_exhook_cli:cli(["server", "disable", "default"])),
-    ?assertEqual([["name=default, hooks=#{}, active=false"]], emqx_exhook_cli:cli(["server", "list"])),
-
-    ?assertEqual([not_running], emqx_exhook_cli:cli(["server", "disable", "default"])),
-    ?assertEqual(ok, emqx_exhook_cli:cli(["server", "enable", "default"])),
-    unmeck_print().
-
-t_cli_stats(_) ->
-    meck_print(),
-    _ = emqx_exhook_cli:cli(["server", "stats"]),
-    _ = emqx_exhook_cli:cli(x),
-    unmeck_print().
+    emqx_exhook_mgr:enable(<<"default">>).
 
 %%--------------------------------------------------------------------
 %% Utils
@@ -115,13 +118,13 @@ unmeck_print() ->
 
 loaded_exhook_hookpoints() ->
     lists:filtermap(fun(E) ->
-        Name = element(2, E),
-        Callbacks = element(3, E),
-        case lists:any(fun is_exhook_callback/1, Callbacks) of
-            true -> {true, Name};
-            _ -> false
-        end
-    end, ets:tab2list(emqx_hooks)).
+                            Name = element(2, E),
+                            Callbacks = element(3, E),
+                            case lists:any(fun is_exhook_callback/1, Callbacks) of
+                                true -> {true, Name};
+                                _ -> false
+                            end
+                    end, ets:tab2list(emqx_hooks)).
 
 is_exhook_callback(Cb) ->
     Action = element(2, Cb),
diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl
new file mode 100644
index 000000000..6f4ead040
--- /dev/null
+++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl
@@ -0,0 +1,197 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_exhook_api_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-define(HOST, "http://127.0.0.1:18083/").
+-define(API_VERSION, "v5").
+-define(BASE_PATH, "api").
+-define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard).
+
+-define(CONF_DEFAULT, <<"
+emqx_exhook {servers = [
+                        {name = default,
+                         url = \"http://127.0.0.1:9000\"
+                         }
+                       ]
+            }
+">>).
+
+all() ->
+    [t_list, t_get, t_add, t_move_1, t_move_2, t_delete, t_update].
+
+init_per_suite(Config) ->
+    application:load(emqx_conf),
+    ok = ekka:start(),
+    ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity),
+    meck:new(emqx_alarm, [non_strict, passthrough, no_link]),
+    meck:expect(emqx_alarm, activate, 3, ok),
+    meck:expect(emqx_alarm, deactivate, 3, ok),
+
+    _ = emqx_exhook_demo_svr:start(),
+    ok = emqx_config:init_load(emqx_exhook_schema, ?CONF_DEFAULT),
+    emqx_mgmt_api_test_util:init_suite([emqx_exhook]),
+    [Conf] = emqx:get_config([emqx_exhook, servers]),
+    [{template, Conf} | Config].
+
+end_per_suite(Config) ->
+    ekka:stop(),
+    mria:stop(),
+    mria_mnesia:delete_schema(),
+    meck:unload(emqx_alarm),
+
+    emqx_mgmt_api_test_util:end_suite([emqx_exhook]),
+    emqx_exhook_demo_svr:stop(),
+    emqx_exhook_demo_svr:stop(<<"test1">>),
+    Config.
+
+init_per_testcase(t_add, Config) ->
+    {ok, _} = emqx_cluster_rpc:start_link(),
+    _ = emqx_exhook_demo_svr:start(<<"test1">>, 9001),
+    timer:sleep(200),
+    Config;
+
+init_per_testcase(_, Config) ->
+    {ok, _} = emqx_cluster_rpc:start_link(),
+    timer:sleep(200),
+    Config.
+
+end_per_testcase(_, Config) ->
+    case erlang:whereis(node()) of
+        undefined -> ok;
+        P ->
+            erlang:unlink(P),
+            erlang:exit(P, kill)
+    end,
+    Config.
+
+t_list(_) ->
+    {ok, Data} = request_api(get, api_path(["exhooks"]), "",
+                             auth_header_()),
+
+    List = decode_json(Data),
+    ?assertEqual(1, length(List)),
+
+    [Svr] = List,
+
+    ?assertMatch(#{name := <<"default">>,
+                   status := <<"running">>}, Svr).
+
+t_get(_) ->
+    {ok, Data} = request_api(get, api_path(["exhooks", "default"]), "",
+                             auth_header_()),
+
+    Svr = decode_json(Data),
+
+    ?assertMatch(#{name := <<"default">>,
+                   status := <<"running">>}, Svr).
+
+t_add(Cfg) ->
+    Template = proplists:get_value(template, Cfg),
+    Instance = Template#{name => <<"test1">>,
+                         url => "http://127.0.0.1:9001"
+                        },
+    {ok, Data} = request_api(post, api_path(["exhooks"]), "",
+                             auth_header_(), Instance),
+
+    Svr = decode_json(Data),
+
+    ?assertMatch(#{name := <<"test1">>,
+                   status := <<"running">>}, Svr),
+
+    ?assertMatch([<<"default">>, <<"test1">>], emqx_exhook_mgr:running()).
+
+t_move_1(_) ->
+    Result = request_api(post, api_path(["exhooks", "default", "move"]), "",
+                         auth_header_(),
+                         #{position => bottom, related => <<>>}),
+
+    ?assertMatch({ok, <<>>}, Result),
+    ?assertMatch([<<"test1">>, <<"default">>], emqx_exhook_mgr:running()).
+
+t_move_2(_) ->
+    Result = request_api(post, api_path(["exhooks", "default", "move"]), "",
+                         auth_header_(),
+                         #{position => before, related => <<"test1">>}),
+
+    ?assertMatch({ok, <<>>}, Result),
+    ?assertMatch([<<"default">>, <<"test1">>], emqx_exhook_mgr:running()).
+
+t_delete(_) ->
+    Result = request_api(delete, api_path(["exhooks", "test1"]), "",
+                         auth_header_()),
+
+    ?assertMatch({ok, <<>>}, Result),
+    ?assertMatch([<<"default">>], emqx_exhook_mgr:running()).
+
+t_update(Cfg) ->
+    Template = proplists:get_value(template, Cfg),
+    Instance = Template#{enable => false},
+    {ok, <<>>} = request_api(put, api_path(["exhooks", "default"]), "",
+                             auth_header_(), Instance),
+
+    ?assertMatch([], emqx_exhook_mgr:running()).
+
+decode_json(Data) ->
+    BinJosn = emqx_json:decode(Data, [return_maps]),
+    emqx_map_lib:unsafe_atom_key_map(BinJosn).
+
+request_api(Method, Url, Auth) ->
+    request_api(Method, Url, [], Auth, []).
+
+request_api(Method, Url, QueryParams, Auth) ->
+    request_api(Method, Url, QueryParams, Auth, []).
+
+request_api(Method, Url, QueryParams, Auth, []) ->
+    NewUrl = case QueryParams of
+                 "" -> Url;
+                 _ -> Url ++ "?" ++ QueryParams
+             end,
+    do_request_api(Method, {NewUrl, [Auth]});
+request_api(Method, Url, QueryParams, Auth, Body) ->
+    NewUrl = case QueryParams of
+                 "" -> Url;
+                 _ -> Url ++ "?" ++ QueryParams
+             end,
+    do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}).
+
+do_request_api(Method, Request)->
+    case httpc:request(Method, Request, [], [{body_format, binary}]) of
+        {error, socket_closed_remotely} ->
+            {error, socket_closed_remotely};
+        {ok, {{"HTTP/1.1", Code, _}, _, Return} }
+          when Code =:= 200 orelse Code =:= 204 orelse Code =:= 201 ->
+            {ok, Return};
+        {ok, {Reason, _, _}} ->
+            {error, Reason}
+    end.
+
+auth_header_() ->
+    AppId = <<"admin">>,
+    AppSecret = <<"public">>,
+    auth_header_(binary_to_list(AppId), binary_to_list(AppSecret)).
+
+auth_header_(User, Pass) ->
+    Encoded = base64:encode_to_string(lists:append([User,":",Pass])),
+    {"Authorization","Basic " ++ Encoded}.
+
+api_path(Parts)->
+    ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION] ++ Parts).
diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
index b1e3801b2..9d1e084c2 100644
--- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
+++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
@@ -20,7 +20,9 @@
 
 %%
 -export([ start/0
+        , start/2
         , stop/0
+        , stop/1
         , take/0
         , in/1
         ]).
@@ -57,39 +59,45 @@
 %%--------------------------------------------------------------------
 
 start() ->
-    Pid = spawn(fun mngr_main/0),
-    register(?MODULE, Pid),
+    start(?NAME, ?PORT).
+
+start(Name, Port) ->
+    Pid = spawn(fun() ->  mgr_main(Name, Port) end),
+    register(to_atom_name(Name), Pid),
     {ok, Pid}.
 
 stop() ->
-    grpc:stop_server(?NAME),
-    ?MODULE ! stop.
+    stop(?NAME).
+
+stop(Name) ->
+    grpc:stop_server(Name),
+    to_atom_name(Name) ! stop.
 
 take() ->
-    ?MODULE ! {take, self()},
+    to_atom_name(?NAME) ! {take, self()},
     receive {value, V} -> V
     after 5000 -> error(timeout) end.
 
 in({FunName, Req}) ->
-    ?MODULE ! {in, FunName, Req}.
+    to_atom_name(?NAME) ! {in, FunName, Req}.
 
-mngr_main() ->
+mgr_main(Name, Port) ->
     application:ensure_all_started(grpc),
     Services = #{protos => [emqx_exhook_pb],
                  services => #{'emqx.exhook.v1.HookProvider' => emqx_exhook_demo_svr}
                 },
     Options = [],
-    Svr = grpc:start_server(?NAME, ?PORT, Services, Options),
-    mngr_loop([Svr, queue:new(), queue:new()]).
+    Svr = grpc:start_server(Name, Port, Services, Options),
+    mgr_loop([Svr, queue:new(), queue:new()]).
 
-mngr_loop([Svr, Q, Takes]) ->
+mgr_loop([Svr, Q, Takes]) ->
     receive
         {in, FunName, Req} ->
             {NQ1, NQ2} = reply(queue:in({FunName, Req}, Q), Takes),
-            mngr_loop([Svr, NQ1, NQ2]);
+            mgr_loop([Svr, NQ1, NQ2]);
         {take, From} ->
             {NQ1, NQ2} = reply(Q, queue:in(From, Takes)),
-            mngr_loop([Svr, NQ1, NQ2]);
+            mgr_loop([Svr, NQ1, NQ2]);
         stop ->
             exit(normal)
     end.
@@ -105,12 +113,18 @@ reply(Q1, Q2) ->
             {NQ1, NQ2}
     end.
 
+to_atom_name(Name) when is_atom(Name) ->
+    Name;
+
+to_atom_name(Name) ->
+    erlang:binary_to_atom(Name).
+
 %%--------------------------------------------------------------------
 %% callbacks
 %%--------------------------------------------------------------------
 
 -spec on_provider_loaded(emqx_exhook_pb:provider_loaded_request(), grpc:metadata())
-    -> {ok, emqx_exhook_pb:loaded_response(), grpc:metadata()}
+                        -> {ok, emqx_exhook_pb:loaded_response(), grpc:metadata()}
      | {error, grpc_cowboy_h:error_response()}.
 
 on_provider_loaded(Req, Md) ->
diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl
index cbd7a2a2a..076fe7134 100644
--- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl
+++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl
@@ -31,12 +31,11 @@
         ]).
 
 -define(CONF_DEFAULT, <<"
-exhook: {
-    servers: [
-        { name: \"default\"
-          url: \"http://127.0.0.1:9000\"
-        }
-    ]
+emqx_exhook
+{servers = [
+            {name = default,
+             url = \"http://127.0.0.1:9000\"
+            }]
 }
 ">>).
 

From bd31b52e3546e2e83e499108991e99786405849c Mon Sep 17 00:00:00 2001
From: k32 <10274441+k32@users.noreply.github.com>
Date: Thu, 23 Dec 2021 15:29:35 +0100
Subject: [PATCH 16/25] chore(ekka): Bump version to 0.11.2

---
 apps/emqx/rebar.config | 2 +-
 rebar.config           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 330291def..18e4f0e1f 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -15,7 +15,7 @@
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.2"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
     , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
diff --git a/rebar.config b/rebar.config
index 8bcab9922..9cb84e00b 100644
--- a/rebar.config
+++ b/rebar.config
@@ -53,7 +53,7 @@
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}
     , {mria, {git, "https://github.com/emqx/mria", {tag, "0.1.5"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.2"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}}

From b6755d5953dcf512a07f2ad323b00e24850b6bd7 Mon Sep 17 00:00:00 2001
From: lafirest <blankalupo@163.com>
Date: Thu, 23 Dec 2021 17:43:31 +0800
Subject: [PATCH 17/25] fix(emqx_retainer): use base64 to encode payload in
 api's result

---
 apps/emqx_retainer/src/emqx_retainer_api.erl | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl
index 61085d9a9..26d341b53 100644
--- a/apps/emqx_retainer/src/emqx_retainer_api.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_api.erl
@@ -34,6 +34,8 @@
                         , page_params/0
                         , properties/1]).
 
+-define(MAX_BASE64_PAYLOAD_SIZE, 1048576). %% 1MB = 1024 x 1024
+
 api_spec() ->
     {[lookup_retained_api(), with_topic_api(), config_api()], []}.
 
@@ -179,7 +181,13 @@ format_message(#message{ id = ID, qos = Qos, topic = Topic, from = From
 
 format_detail_message(#message{payload = Payload} = Msg) ->
     Base = format_message(Msg),
-    Base#{payload => Payload}.
+    EncodePayload = base64:encode(Payload),
+    case erlang:byte_size(EncodePayload) =< ?MAX_BASE64_PAYLOAD_SIZE of
+        true ->
+            Base#{payload => EncodePayload};
+        _ ->
+            Base#{payload => base64:encode(<<"PAYLOAD_TOO_LARGE">>)}
+    end.
 
 to_bin_string(Data) when is_binary(Data) ->
     Data;

From d2d50443ce0ad407a23224f93ff0bcc738a49880 Mon Sep 17 00:00:00 2001
From: lafirest <blankalupo@163.com>
Date: Thu, 23 Dec 2021 17:43:31 +0800
Subject: [PATCH 18/25] fix(emqx_retainer): use base64 to encode payload in
 api's result

---
 apps/emqx_retainer/src/emqx_retainer_api.erl | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl
index 61085d9a9..26d341b53 100644
--- a/apps/emqx_retainer/src/emqx_retainer_api.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_api.erl
@@ -34,6 +34,8 @@
                         , page_params/0
                         , properties/1]).
 
+-define(MAX_BASE64_PAYLOAD_SIZE, 1048576). %% 1MB = 1024 x 1024
+
 api_spec() ->
     {[lookup_retained_api(), with_topic_api(), config_api()], []}.
 
@@ -179,7 +181,13 @@ format_message(#message{ id = ID, qos = Qos, topic = Topic, from = From
 
 format_detail_message(#message{payload = Payload} = Msg) ->
     Base = format_message(Msg),
-    Base#{payload => Payload}.
+    EncodePayload = base64:encode(Payload),
+    case erlang:byte_size(EncodePayload) =< ?MAX_BASE64_PAYLOAD_SIZE of
+        true ->
+            Base#{payload => EncodePayload};
+        _ ->
+            Base#{payload => base64:encode(<<"PAYLOAD_TOO_LARGE">>)}
+    end.
 
 to_bin_string(Data) when is_binary(Data) ->
     Data;

From e0f860d7d93f1a59fe23ceb465a0b79929072224 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.ru>
Date: Fri, 17 Dec 2021 23:03:25 +0300
Subject: [PATCH 19/25] chore(authz): fix HTTP authz, cover with tests

---
 apps/emqx_authz/src/emqx_authz.erl            |   6 +-
 apps/emqx_authz/src/emqx_authz_http.erl       | 139 ++++--
 apps/emqx_authz/src/emqx_authz_schema.erl     |  11 +-
 apps/emqx_authz/test/emqx_authz_SUITE.erl     |   6 +-
 .../emqx_authz/test/emqx_authz_http_SUITE.erl | 415 +++++++++++++++---
 .../test/emqx_authz_http_test_server.erl      |  89 ++++
 6 files changed, 564 insertions(+), 102 deletions(-)
 create mode 100644 apps/emqx_authz/test/emqx_authz_http_test_server.erl

diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl
index 54438793b..510306efe 100644
--- a/apps/emqx_authz/src/emqx_authz.erl
+++ b/apps/emqx_authz/src/emqx_authz.erl
@@ -59,6 +59,8 @@
 
 -define(METRICS, [?METRIC_ALLOW, ?METRIC_DENY, ?METRIC_NOMATCH]).
 
+-define(IS_ENABLED(Enable), ((Enable =:= true) or (Enable =:= <<"true">>))).
+
 %% Initialize authz backend.
 %% Populate the passed configuration map with necessary data,
 %% like `ResourceID`s
@@ -155,8 +157,8 @@ do_update({?CMD_APPEND, Sources}, Conf) when is_list(Sources), is_list(Conf) ->
     NConf = Conf ++ Sources,
     ok = check_dup_types(NConf),
     NConf;
-do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := true} = Source}, Conf) when is_map(Source),
-                                                                               is_list(Conf) ->
+do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := Enable} = Source}, Conf)
+  when is_map(Source), is_list(Conf), ?IS_ENABLED(Enable) ->
     case create_dry_run(Type, Source)  of
         ok ->
             {_Old, Front, Rear} = take(Type, Conf),
diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl
index c2ee96594..d677704c4 100644
--- a/apps/emqx_authz/src/emqx_authz_http.erl
+++ b/apps/emqx_authz/src/emqx_authz_http.erl
@@ -40,9 +40,8 @@
 description() ->
     "AuthZ with http".
 
-init(#{url := Url} = Source) ->
-    NSource = maps:put(base_url, maps:remove(query, Url), Source),
-    case emqx_authz_utils:create_resource(emqx_connector_http, NSource) of
+init(Source) ->
+    case emqx_authz_utils:create_resource(emqx_connector_http, Source) of
         {error, Reason} -> error({load_config_error, Reason});
         {ok, Id} -> Source#{annotations => #{id => Id}}
     end.
@@ -51,39 +50,60 @@ destroy(#{annotations := #{id := Id}}) ->
     ok = emqx_resource:remove(Id).
 
 dry_run(Source) ->
-    URIMap = maps:get(url, Source),
-    NSource = maps:put(base_url, maps:remove(query, URIMap), Source),
-    emqx_resource:create_dry_run(emqx_connector_http, NSource).
+    emqx_resource:create_dry_run(emqx_connector_http, Source).
 
 authorize(Client, PubSub, Topic,
             #{type := http,
-              url := #{path := Path} = URL,
+              query := Query,
+              path := Path,
               headers := Headers,
               method := Method,
               request_timeout := RequestTimeout,
               annotations := #{id := ResourceID}
              } = Source) ->
     Request = case Method of
-                  get  ->
-                      Query = maps:get(query, URL, ""),
-                      Path1 = replvar(Path ++ "?" ++ Query, PubSub, Topic, Client),
+                  get ->
+                      Path1 = replvar(
+                                Path ++ "?" ++ Query,
+                                PubSub,
+                                Topic,
+                                maps:to_list(Client),
+                                fun var_uri_encode/1),
+
                       {Path1, maps:to_list(Headers)};
+
                   _ ->
-                      Body0 = serialize_body(
-                                maps:get('Accept', Headers, <<"application/json">>),
-                                maps:get(body, Source, #{})
-                              ),
-                      Body1 = replvar(Body0, PubSub, Topic, Client),
-                      Path1 = replvar(Path, PubSub, Topic, Client),
-                      {Path1, maps:to_list(Headers), Body1}
+                      Body0 = maps:get(body, Source, #{}),
+                      Body1 = replvar_deep(
+                                Body0,
+                                PubSub,
+                                Topic,
+                                maps:to_list(Client),
+                                fun var_bin_encode/1),
+
+                      Body2 = serialize_body(
+                                maps:get(<<"content-type">>, Headers, <<"application/json">>),
+                                Body1),
+
+                      Path1 = replvar(
+                                Path,
+                                PubSub,
+                                Topic,
+                                maps:to_list(Client),
+                                fun var_uri_encode/1),
+
+                      {Path1, maps:to_list(Headers), Body2}
               end,
-    case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of
+    HttpResult = emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}),
+    case HttpResult of
         {ok, 200, _Headers} ->
             {matched, allow};
         {ok, 204, _Headers} ->
             {matched, allow};
         {ok, 200, _Headers, _Body} ->
             {matched, allow};
+        {ok, _Status, _Headers} ->
+            nomatch;
         {ok, _Status, _Headers, _Body} ->
             nomatch;
         {error, Reason} ->
@@ -121,30 +141,67 @@ serialize_body(<<"application/json">>, Body) ->
 serialize_body(<<"application/x-www-form-urlencoded">>, Body) ->
     query_string(Body).
 
-replvar(Str0, PubSub, Topic,
-        #{username := Username,
-          clientid := Clientid,
-          peerhost := IpAddress,
-          protocol := Protocol,
-          mountpoint := Mountpoint
-         }) when is_list(Str0);
-                 is_binary(Str0) ->
+
+replvar_deep(Map, PubSub, Topic, Vars, VarEncode) when is_map(Map) ->
+    maps:from_list(
+      lists:map(
+        fun({Key, Value}) ->
+                {replvar(Key, PubSub, Topic, Vars, VarEncode),
+                 replvar(Value, PubSub, Topic, Vars, VarEncode)}
+        end,
+        maps:to_list(Map)));
+replvar_deep(List, PubSub, Topic, Vars, VarEncode) when is_list(List) ->
+    lists:map(
+      fun(Value) ->
+              replvar(Value, PubSub, Topic, Vars, VarEncode)
+      end,
+      List);
+replvar_deep(Number, _PubSub, _Topic, _Vars, _VarEncode) when is_number(Number) ->
+    Number;
+replvar_deep(Binary, PubSub, Topic, Vars, VarEncode) when is_binary(Binary) ->
+    replvar(Binary, PubSub, Topic, Vars, VarEncode).
+
+replvar(Str0, PubSub, Topic, [], VarEncode) ->
     NTopic = emqx_http_lib:uri_encode(Topic),
-    Str1 = re:replace( Str0, emqx_authz:ph_to_re(?PH_S_CLIENTID)
-                     , bin(Clientid), [global, {return, binary}]),
-    Str2 = re:replace( Str1, emqx_authz:ph_to_re(?PH_S_USERNAME)
-                     , bin(Username), [global, {return, binary}]),
-    Str3 = re:replace( Str2, emqx_authz:ph_to_re(?PH_S_HOST)
-                     , inet_parse:ntoa(IpAddress), [global, {return, binary}]),
-    Str4 = re:replace( Str3, emqx_authz:ph_to_re(?PH_S_PROTONAME)
-                     , bin(Protocol), [global, {return, binary}]),
-    Str5 = re:replace( Str4, emqx_authz:ph_to_re(?PH_S_MOUNTPOINT)
-                     , bin(Mountpoint), [global, {return, binary}]),
-    Str6 = re:replace( Str5, emqx_authz:ph_to_re(?PH_S_TOPIC)
-                     , bin(NTopic), [global, {return, binary}]),
-    Str7 = re:replace( Str6, emqx_authz:ph_to_re(?PH_S_ACTION)
-                     , bin(PubSub), [global, {return, binary}]),
-    Str7.
+    Str1 = re:replace(Str0, emqx_authz:ph_to_re(?PH_S_TOPIC),
+                      VarEncode(NTopic), [global, {return, binary}]),
+    re:replace(Str1, emqx_authz:ph_to_re(?PH_S_ACTION),
+               VarEncode(PubSub), [global, {return, binary}]);
+
+
+replvar(Str, PubSub, Topic, [{username, Username} | Rest], VarEncode) ->
+    Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_USERNAME),
+                      VarEncode(Username), [global, {return, binary}]),
+    replvar(Str1, PubSub, Topic, Rest, VarEncode);
+
+replvar(Str, PubSub, Topic, [{clientid, Clientid} | Rest], VarEncode) ->
+    Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_CLIENTID),
+                      VarEncode(Clientid), [global, {return, binary}]),
+    replvar(Str1, PubSub, Topic, Rest, VarEncode);
+
+replvar(Str, PubSub, Topic, [{peerhost, IpAddress}  | Rest], VarEncode) ->
+    Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_PEERHOST),
+                      VarEncode(inet_parse:ntoa(IpAddress)), [global, {return, binary}]),
+    replvar(Str1, PubSub, Topic, Rest, VarEncode);
+
+replvar(Str, PubSub, Topic, [{protocol, Protocol} | Rest], VarEncode) ->
+    Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_PROTONAME),
+                      VarEncode(Protocol), [global, {return, binary}]),
+    replvar(Str1, PubSub, Topic, Rest, VarEncode);
+
+replvar(Str, PubSub, Topic, [{mountpoint, Mountpoint} | Rest], VarEncode) ->
+    Str1 = re:replace(Str, emqx_authz:ph_to_re(?PH_S_MOUNTPOINT),
+                      VarEncode(Mountpoint), [global, {return, binary}]),
+    replvar(Str1, PubSub, Topic, Rest, VarEncode);
+
+replvar(Str, PubSub, Topic, [_Unknown | Rest], VarEncode) ->
+    replvar(Str, PubSub, Topic, Rest, VarEncode).
+
+var_uri_encode(S) ->
+    emqx_http_lib:uri_encode(bin(S)).
+
+var_bin_encode(S) ->
+    bin(S).
 
 bin(A) when is_atom(A) -> atom_to_binary(A, utf8);
 bin(B) when is_binary(B) -> B;
diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl
index 4f7788849..8415e3710 100644
--- a/apps/emqx_authz/src/emqx_authz_schema.erl
+++ b/apps/emqx_authz/src/emqx_authz_schema.erl
@@ -20,14 +20,10 @@
 
 -reflect_type([ permission/0
               , action/0
-              , url/0
               ]).
 
--typerefl_from_string({url/0, emqx_http_lib, uri_parse}).
-
 -type action() :: publish | subscribe | all.
 -type permission() :: allow | deny.
--type url() :: emqx_http_lib:uri_map().
 
 -export([ namespace/0
         , roots/0
@@ -143,10 +139,11 @@ fields(redis_cluster) ->
 http_common_fields() ->
     [ {type,            #{type => http}}
     , {enable,          #{type => boolean(), default => true}}
-    , {url,             #{type => url()}}
     , {request_timeout, mk_duration("request timeout", #{default => "30s"})}
     , {body,            #{type => map(), nullable => true}}
-    ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)).
+    , {path,            #{type => string(), default => ""}}
+    , {query,           #{type => string(), default => ""}}
+    ] ++ emqx_connector_http:fields(config).
 
 mongo_common_fields() ->
     [ {collection, #{type => atom()}}
@@ -203,7 +200,7 @@ check_ssl_opts(Conf)
   when Conf =:= #{} ->
     true;
 check_ssl_opts(Conf) ->
-    case emqx_authz_http:parse_url(hocon_schema:get_value("config.url", Conf)) of
+    case emqx_authz_http:parse_url(hocon_schema:get_value("config.base_url", Conf)) of
         #{scheme := https} ->
             case hocon_schema:get_value("config.ssl.enable", Conf) of
                 true -> ok;
diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl
index e18901fc5..942f74abc 100644
--- a/apps/emqx_authz/test/emqx_authz_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl
@@ -65,7 +65,9 @@ set_special_configs(_App) ->
 
 -define(SOURCE1, #{<<"type">> => <<"http">>,
                    <<"enable">> => true,
-                   <<"url">> => <<"https://fake.com:443/">>,
+                   <<"base_url">> => <<"https://example.com:443/">>,
+                   <<"path">> => <<"a/b">>,
+                   <<"query">> => <<"c=d">>,
                    <<"headers">> => #{},
                    <<"method">> => <<"get">>,
                    <<"request_timeout">> => 5000
@@ -77,7 +79,7 @@ set_special_configs(_App) ->
                    <<"pool_size">> => 1,
                    <<"database">> => <<"mqtt">>,
                    <<"ssl">> => #{<<"enable">> => false},
-                   <<"collection">> => <<"fake">>,
+                   <<"collection">> => <<"authz">>,
                    <<"selector">> => #{<<"a">> => <<"b">>}
                   }).
 -define(SOURCE3, #{<<"type">> => <<"mysql">>,
diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
index c438b3f4b..7196d75ff 100644
--- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
@@ -4,7 +4,8 @@
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
 %% You may obtain a copy of the License at
-%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing, software
 %% distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,75 +23,389 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
+-define(HTTP_PORT, 33333).
+-define(HTTP_PATH, "/authz/[...]").
+
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
-groups() ->
-    [].
-
 init_per_suite(Config) ->
-    meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
-    meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end),
-    meck:expect(emqx_resource, remove, fun(_) -> ok end ),
-
     ok = emqx_common_test_helpers:start_apps(
            [emqx_conf, emqx_authz],
-           fun set_special_configs/1),
-
-    Rules = [#{<<"type">> => <<"http">>,
-               <<"url">> => <<"https://fake.com:443/">>,
-               <<"headers">> => #{},
-               <<"method">> => <<"get">>,
-               <<"request_timeout">> => 5000
-              }
-            ],
-    {ok, _} = emqx_authz:update(replace, Rules),
+           fun set_special_configs/1
+          ),
+    ok = start_apps([emqx_resource, emqx_connector, cowboy]),
     Config.
 
 end_per_suite(_Config) ->
-    {ok, _} = emqx:update_config(
-                [authorization],
-                #{<<"no_match">> => <<"allow">>,
-                  <<"cache">> => #{<<"enable">> => <<"true">>},
-                  <<"sources">> => []}),
-    emqx_common_test_helpers:stop_apps([emqx_authz, emqx_conf]),
-    meck:unload(emqx_resource),
-    ok.
+    ok = emqx_authz_test_lib:restore_authorizers(),
+    ok = stop_apps([emqx_resource, emqx_connector, cowboy]),
+    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
 set_special_configs(emqx_authz) ->
-    {ok, _} = emqx:update_config([authorization, cache, enable], false),
-    {ok, _} = emqx:update_config([authorization, no_match], deny),
-    {ok, _} = emqx:update_config([authorization, sources], []),
-    ok;
-set_special_configs(_App) ->
+    ok = emqx_authz_test_lib:reset_authorizers();
+
+set_special_configs(_) ->
     ok.
 
+init_per_testcase(_Case, Config) ->
+    ok = emqx_authz_test_lib:reset_authorizers(),
+    ok = emqx_authz_http_test_server:start(?HTTP_PORT, ?HTTP_PATH),
+    Config.
+
+end_per_testcase(_Case, _Config) ->
+    ok = emqx_authz_http_test_server:stop().
+
 %%------------------------------------------------------------------------------
-%% Testcases
+%% Tests
 %%------------------------------------------------------------------------------
 
-t_authz(_) ->
-    ClientInfo = #{clientid => <<"my-clientid">>,
-                   username => <<"my-username">>,
+t_response_handling(_Config) ->
+    ClientInfo = #{clientid => <<"clientid">>,
+                   username => <<"username">>,
                    peerhost => {127,0,0,1},
-                   protocol => mqtt,
-                   mountpoint => <<"fake">>,
                    zone => default,
                    listener => {tcp, default}
-                   },
+                  },
 
-    meck:expect(emqx_resource, query, fun(_, _) -> {ok, 204, fake_headers} end),
-    ?assertEqual(allow,
-                 emqx_access_control:authorize(ClientInfo, subscribe, <<"#">>)),
+    %% OK, get, no body
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                   Req = cowboy_req:reply(200, Req0),
+                   {ok, Req, State}
+           end,
+           #{}),
 
-    meck:expect(emqx_resource, query, fun(_, _) -> {ok, 200, fake_headers, fake_body} end),
-    ?assertEqual(allow,
-                 emqx_access_control:authorize(ClientInfo, publish, <<"#">>)),
+    allow = emqx_access_control:authorize(ClientInfo, publish, <<"t">>),
+
+    %% OK, get, body & headers
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                   Req = cowboy_req:reply(
+                           200,
+                           #{<<"content-type">> => <<"text/plain">>},
+                           "Response body",
+                           Req0),
+                   {ok, Req, State}
+           end,
+           #{}),
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
+
+    %% OK, get, 204
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                   Req = cowboy_req:reply(204, Req0),
+                   {ok, Req, State}
+           end,
+           #{}),
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
+
+    %% Not OK, get, 400
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                   Req = cowboy_req:reply(400, Req0),
+                   {ok, Req, State}
+           end,
+           #{}),
+
+    ?assertEqual(
+        deny,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
+
+    %% Not OK, get, 400 + body & headers
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                   Req = cowboy_req:reply(
+                           400,
+                           #{<<"content-type">> => <<"text/plain">>},
+                           "Response body",
+                           Req0),
+                   {ok, Req, State}
+           end,
+           #{}),
+
+    ?assertEqual(
+        deny,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
+
+t_query_params(_Config) ->
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                  #{username := <<"user name">>,
+                    clientid := <<"client id">>,
+                    peerhost := <<"127.0.0.1">>,
+                    proto_name := <<"MQTT">>,
+                    mountpoint := <<"MOUNTPOINT">>,
+                    topic := <<"t">>,
+                    action := <<"publish">>
+                   } = cowboy_req:match_qs(
+                         [username,
+                          clientid,
+                          peerhost,
+                          proto_name,
+                          mountpoint,
+                          topic,
+                          action],
+                         Req0),
+                   Req = cowboy_req:reply(200, Req0),
+                   {ok, Req, State}
+           end,
+           #{<<"query">> => <<"username=${username}&"
+                             "clientid=${clientid}&"
+                             "peerhost=${peerhost}&"
+                             "proto_name=${proto_name}&"
+                             "mountpoint=${mountpoint}&"
+                             "topic=${topic}&"
+                             "action=${action}">>
+            }),
+
+    ClientInfo = #{clientid => <<"client id">>,
+                   username => <<"user name">>,
+                   peerhost => {127,0,0,1},
+                   protocol => <<"MQTT">>,
+                   mountpoint => <<"MOUNTPOINT">>,
+                   zone => default,
+                   listener => {tcp, default}
+                  },
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
+
+t_path_params(_Config) ->
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                   <<"/authz/"
+                     "username/user%20name/"
+                     "clientid/client%20id/"
+                     "peerhost/127.0.0.1/"
+                     "proto_name/MQTT/"
+                     "mountpoint/MOUNTPOINT/"
+                     "topic/t/"
+                     "action/publish">> = cowboy_req:path(Req0),
+                   Req = cowboy_req:reply(200, Req0),
+                   {ok, Req, State}
+           end,
+           #{<<"path">> => <<"username/${username}/"
+                             "clientid/${clientid}/"
+                             "peerhost/${peerhost}/"
+                             "proto_name/${proto_name}/"
+                             "mountpoint/${mountpoint}/"
+                             "topic/${topic}/"
+                             "action/${action}">>
+            }),
+
+    ClientInfo = #{clientid => <<"client id">>,
+                   username => <<"user name">>,
+                   peerhost => {127,0,0,1},
+                   protocol => <<"MQTT">>,
+                   mountpoint => <<"MOUNTPOINT">>,
+                   zone => default,
+                   listener => {tcp, default}
+                  },
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
+
+t_json_body(_Config) ->
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                   ?assertEqual(
+                      <<"/authz/"
+                        "username/user%20name/"
+                        "clientid/client%20id/"
+                        "peerhost/127.0.0.1/"
+                        "proto_name/MQTT/"
+                        "mountpoint/MOUNTPOINT/"
+                        "topic/t/"
+                        "action/publish">>,
+                      cowboy_req:path(Req0)),
+
+                   {ok, RawBody, Req1} = cowboy_req:read_body(Req0),
+
+                   ?assertMatch(
+                      #{<<"username">> := <<"user name">>,
+                        <<"CLIENT_client id">> := <<"client id">>,
+                        <<"peerhost">> := <<"127.0.0.1">>,
+                        <<"proto_name">> := <<"MQTT">>,
+                        <<"mountpoint">> := <<"MOUNTPOINT">>,
+                        <<"topic">> := <<"t">>,
+                        <<"action">> := <<"publish">>},
+                      jiffy:decode(RawBody, [return_maps])),
+
+                   Req = cowboy_req:reply(200, Req1),
+                   {ok, Req, State}
+           end,
+           #{<<"method">> => <<"post">>,
+             <<"path">> => <<"username/${username}/"
+                             "clientid/${clientid}/"
+                             "peerhost/${peerhost}/"
+                             "proto_name/${proto_name}/"
+                             "mountpoint/${mountpoint}/"
+                             "topic/${topic}/"
+                             "action/${action}">>,
+             <<"body">> => #{<<"username">> => <<"${username}">>,
+                             <<"CLIENT_${clientid}">> => <<"${clientid}">>,
+                             <<"peerhost">> => <<"${peerhost}">>,
+                             <<"proto_name">> => <<"${proto_name}">>,
+                             <<"mountpoint">> => <<"${mountpoint}">>,
+                             <<"topic">> => <<"${topic}">>,
+                             <<"action">> => <<"${action}">>}
+            }),
+
+    ClientInfo = #{clientid => <<"client id">>,
+                   username => <<"user name">>,
+                   peerhost => {127,0,0,1},
+                   protocol => <<"MQTT">>,
+                   mountpoint => <<"MOUNTPOINT">>,
+                   zone => default,
+                   listener => {tcp, default}
+                  },
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
 
 
-    meck:expect(emqx_resource, query, fun(_, _) -> {error, other} end),
-    ?assertEqual(deny,
-        emqx_access_control:authorize(ClientInfo, subscribe, <<"+">>)),
-    ?assertEqual(deny,
-        emqx_access_control:authorize(ClientInfo, publish, <<"+">>)),
-    ok.
+t_form_body(_Config) ->
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                   ?assertEqual(
+                      <<"/authz/"
+                        "username/user%20name/"
+                        "clientid/client%20id/"
+                        "peerhost/127.0.0.1/"
+                        "proto_name/MQTT/"
+                        "mountpoint/MOUNTPOINT/"
+                        "topic/t/"
+                        "action/publish">>,
+                      cowboy_req:path(Req0)),
+                    
+                   {ok, PostVars, Req1} = cowboy_req:read_urlencoded_body(Req0),
+
+                   ?assertMatch(
+                      #{<<"username">> := <<"user name">>,
+                        <<"clientid">> := <<"client id">>,
+                        <<"peerhost">> := <<"127.0.0.1">>,
+                        <<"proto_name">> := <<"MQTT">>,
+                        <<"mountpoint">> := <<"MOUNTPOINT">>,
+                        <<"topic">> := <<"t">>,
+                        <<"action">> := <<"publish">>},
+                      maps:from_list(PostVars)),
+
+                   Req = cowboy_req:reply(200, Req1),
+                   {ok, Req, State}
+           end,
+           #{<<"method">> => <<"post">>,
+             <<"path">> => <<"username/${username}/"
+                             "clientid/${clientid}/"
+                             "peerhost/${peerhost}/"
+                             "proto_name/${proto_name}/"
+                             "mountpoint/${mountpoint}/"
+                             "topic/${topic}/"
+                             "action/${action}">>,
+             <<"body">> => #{<<"username">> => <<"${username}">>,
+                             <<"clientid">> => <<"${clientid}">>,
+                             <<"peerhost">> => <<"${peerhost}">>,
+                             <<"proto_name">> => <<"${proto_name}">>,
+                             <<"mountpoint">> => <<"${mountpoint}">>,
+                             <<"topic">> => <<"${topic}">>,
+                             <<"action">> => <<"${action}">>},
+             <<"headers">> => #{<<"content-type">> => <<"application/x-www-form-urlencoded">>}
+            }),
+
+    ClientInfo = #{clientid => <<"client id">>,
+                   username => <<"user name">>,
+                   peerhost => {127,0,0,1},
+                   protocol => <<"MQTT">>,
+                   mountpoint => <<"MOUNTPOINT">>,
+                   zone => default,
+                   listener => {tcp, default}
+                  },
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
+
+
+t_create_replace(_Config) ->
+    ClientInfo = #{clientid => <<"clientid">>,
+                   username => <<"username">>,
+                   peerhost => {127,0,0,1},
+                   zone => default,
+                   listener => {tcp, default}
+                  },
+
+    %% Bad URL
+    ok = setup_handler_and_config(
+           fun(Req0, State) ->
+                   Req = cowboy_req:reply(200, Req0),
+                   {ok, Req, State}
+           end,
+           #{<<"base_url">> => <<"http://127.0.0.1:33331/authz">>}),
+
+
+    ?assertEqual(
+        deny,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
+
+    %% Changing to other bad config does not work
+    BadConfig = maps:merge(
+                  raw_http_authz_config(),
+                  #{<<"base_url">> => <<"http://127.0.0.1:33332/authz">>}),
+
+    ?assertMatch(
+        {error, _},
+        emqx_authz:update({?CMD_REPLACE, http}, BadConfig)),
+
+    ?assertEqual(
+        deny,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
+
+    %% Changing to valid config
+    OkConfig = maps:merge(
+                  raw_http_authz_config(),
+                  #{<<"base_url">> => <<"http://127.0.0.1:33333/authz">>}),
+    
+    ?assertMatch(
+        {ok, _},
+        emqx_authz:update({?CMD_REPLACE, http}, OkConfig)),
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)).
+
+%%------------------------------------------------------------------------------
+%% Helpers
+%%------------------------------------------------------------------------------
+
+raw_http_authz_config() ->
+    #{
+        <<"enable">> => <<"true">>,
+
+        <<"type">> => <<"http">>,
+        <<"method">> => <<"get">>,
+        <<"base_url">> => <<"http://127.0.0.1:33333/authz">>,
+        <<"path">> => <<"users/${username}/">>,
+        <<"query">> => <<"topic=${topic}&action=${action}">>,
+        <<"headers">> => #{<<"X-Test-Header">> => <<"Test Value">>}
+    }.
+
+setup_handler_and_config(Handler, Config) ->
+    ok = emqx_authz_http_test_server:set_handler(Handler),
+    ok = emqx_authz_test_lib:setup_config(
+           raw_http_authz_config(),
+           Config).
+
+start_apps(Apps) ->
+    lists:foreach(fun application:ensure_all_started/1, Apps).
+
+stop_apps(Apps) ->
+    lists:foreach(fun application:stop/1, Apps).
diff --git a/apps/emqx_authz/test/emqx_authz_http_test_server.erl b/apps/emqx_authz/test/emqx_authz_http_test_server.erl
new file mode 100644
index 000000000..19ead4627
--- /dev/null
+++ b/apps/emqx_authz/test/emqx_authz_http_test_server.erl
@@ -0,0 +1,89 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_http_test_server).
+
+-behaviour(gen_server).
+-behaviour(cowboy_handler).
+
+% cowboy_server callbacks
+-export([init/2]).
+
+% gen_server callbacks
+-export([init/1,
+         handle_call/3,
+         handle_cast/2
+        ]).
+
+% API
+-export([start/2,
+         stop/0,
+         set_handler/1
+        ]).
+
+%%------------------------------------------------------------------------------
+%% API
+%%------------------------------------------------------------------------------
+
+start(Port, Path) ->
+    Dispatch = cowboy_router:compile([
+        {'_', [{Path, ?MODULE, []}]}
+    ]),
+    {ok, _} = cowboy:start_clear(?MODULE,
+        [{port, Port}],
+        #{env => #{dispatch => Dispatch}}
+    ),
+    {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []),
+    ok.
+
+stop() ->
+    gen_server:stop(?MODULE),
+    cowboy:stop_listener(?MODULE).
+
+set_handler(F) when is_function(F, 2) ->
+    gen_server:call(?MODULE, {set_handler, F}).
+
+%%------------------------------------------------------------------------------
+%% gen_server API
+%%------------------------------------------------------------------------------
+
+init([]) ->
+    F = fun(Req0, State) ->
+                Req = cowboy_req:reply(
+                        400,
+                        #{<<"content-type">> => <<"text/plain">>},
+                        <<"">>,
+                        Req0),
+                {ok, Req, State}
+        end,
+    {ok, F}.
+
+handle_cast(_, F) ->
+    {noreply, F}.
+
+handle_call({set_handler, F}, _From, _F) ->
+    {reply, ok, F};
+
+handle_call(get_handler, _From, F) ->
+    {reply, F, F}.
+
+%%------------------------------------------------------------------------------
+%% cowboy_server API
+%%------------------------------------------------------------------------------
+
+init(Req, State) ->
+    Handler = gen_server:call(?MODULE, get_handler),
+    Handler(Req, State).

From d75e0104cc78f2f1ab21c83ae43158de38f51934 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.ru>
Date: Mon, 20 Dec 2021 21:12:39 +0300
Subject: [PATCH 20/25] chore(authz): test file authz with real files

---
 apps/emqx_authz/src/emqx_authz_file.erl       |   6 +-
 .../emqx_authz/test/emqx_authz_file_SUITE.erl | 130 ++++++++++++++++++
 2 files changed, 135 insertions(+), 1 deletion(-)
 create mode 100644 apps/emqx_authz/test/emqx_authz_file_SUITE.erl

diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl
index ba4f9c2b7..ad6f39573 100644
--- a/apps/emqx_authz/src/emqx_authz_file.erl
+++ b/apps/emqx_authz/src/emqx_authz_file.erl
@@ -55,7 +55,11 @@ init(#{path := Path} = Source) ->
 
 destroy(_Source) -> ok.
 
-dry_run(_Source) -> ok.
+dry_run(#{path := Path}) ->
+    case file:consult(Path) of
+        {ok, _} -> ok;
+        {error, _} = Error -> Error
+    end.
 
 authorize(Client, PubSub, Topic, #{annotations := #{rules := Rules}}) ->
     emqx_authz_rule:matches(Client, PubSub, Topic, Rules).
diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
new file mode 100644
index 000000000..09c49545c
--- /dev/null
+++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
@@ -0,0 +1,130 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_file_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include("emqx_authz.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+groups() ->
+    [].
+
+init_per_suite(Config) ->
+    ok = emqx_common_test_helpers:start_apps(
+           [emqx_conf, emqx_authz],
+           fun set_special_configs/1),
+    Config.
+
+end_per_suite(_Config) ->
+    ok = emqx_authz_test_lib:restore_authorizers(),
+    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
+
+init_per_testcase(Config) ->
+    ok = emqx_authz_test_lib:reset_authorizers(),
+    Config.
+
+set_special_configs(emqx_authz) ->
+    ok = emqx_authz_test_lib:reset_authorizers();
+
+set_special_configs(_) ->
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Testcases
+%%------------------------------------------------------------------------------
+
+t_ok(_Config) ->
+    ClientInfo = #{clientid => <<"clientid">>,
+                   username => <<"username">>,
+                   peerhost => {127,0,0,1},
+                   zone => default,
+                   listener => {tcp, default}
+                  },
+
+    ok = setup_rules([{allow, {user, "username"}, publish, ["t"]}]),
+    ok = setup_config(#{}),
+
+    ?assertEqual(
+       allow,
+       emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
+
+    ?assertEqual(
+       deny,
+       emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>)).
+
+t_invalid_file(_Config) ->
+    ok = file:write_file(<<"acl.conf">>, <<"{{invalid term">>),
+
+    ?assertMatch(
+       {error, {1, erl_parse, _}},
+       emqx_authz:update(?CMD_REPLACE, [raw_file_authz_config()])).
+
+t_nonexistent_file(_Config) ->
+    ?assertEqual(
+       {error, enoent},
+       emqx_authz:update(?CMD_REPLACE,
+                         [maps:merge(raw_file_authz_config(),
+                                     #{<<"path">> => <<"nonexistent.conf">>})
+                         ])).
+
+t_update(_Config) ->
+    ok = setup_rules([{allow, {user, "username"}, publish, ["t"]}]),
+    ok = setup_config(#{}),
+
+    ?assertMatch(
+       {error, _},
+       emqx_authz:update(
+         {?CMD_REPLACE, file},
+         maps:merge(raw_file_authz_config(),
+                    #{<<"path">> => <<"nonexistent.conf">>}))),
+
+    ?assertMatch(
+       {ok, _},
+       emqx_authz:update(
+         {?CMD_REPLACE, file},
+         raw_file_authz_config())).
+
+%%------------------------------------------------------------------------------
+%% Helpers
+%%------------------------------------------------------------------------------
+
+raw_file_authz_config() ->
+    #{
+        <<"enable">> => <<"true">>,
+
+        <<"type">> => <<"file">>,
+        <<"path">> => <<"acl.conf">>
+    }.
+
+setup_rules(Rules) ->
+    {ok, F} = file:open(<<"acl.conf">>, [write]),
+    lists:foreach(
+      fun(Rule) ->
+              io:format(F, "~p.~n", [Rule])
+      end,
+      Rules),
+    ok = file:close(F).
+
+setup_config(SpecialParams) ->
+    emqx_authz_test_lib:setup_config(
+      raw_file_authz_config(),
+      SpecialParams).

From 2bada0bab8dc27b35effa7bd2d51ddfdb33e4ef1 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.ru>
Date: Tue, 21 Dec 2021 14:08:13 +0300
Subject: [PATCH 21/25] chore(authz): test Mria authz

---
 apps/emqx_authz/src/emqx_authz_mnesia.erl     |  35 +++-
 .../emqx_authz/test/emqx_authz_file_SUITE.erl |   2 +-
 .../test/emqx_authz_mnesia_SUITE.erl          | 175 +++++++++++-------
 .../test/emqx_authz_mysql_SUITE.erl           |   2 +-
 .../test/emqx_authz_postgresql_SUITE.erl      |   2 +-
 .../test/emqx_authz_redis_SUITE.erl           |   2 +-
 apps/emqx_authz/test/emqx_authz_test_lib.erl  |   1 +
 7 files changed, 139 insertions(+), 80 deletions(-)

diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl
index 2ce8215cd..851fe1522 100644
--- a/apps/emqx_authz/src/emqx_authz_mnesia.erl
+++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl
@@ -114,18 +114,19 @@ authorize(#{username := Username,
 %% Management API
 %%--------------------------------------------------------------------
 
+-spec(init_tables() -> ok).
 init_tables() ->
     ok = mria_rlog:wait_for_shards([?ACL_SHARDED], infinity).
 
 -spec(store_rules(who(), rules()) -> ok).
 store_rules({username, Username}, Rules) ->
-    Record = #emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = Rules},
+    Record = #emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = normalize_rules(Rules)},
     mria:dirty_write(Record);
 store_rules({clientid, Clientid}, Rules) ->
-    Record = #emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = Rules},
+    Record = #emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = normalize_rules(Rules)},
     mria:dirty_write(Record);
 store_rules(all, Rules) ->
-    Record = #emqx_acl{who = ?ACL_TABLE_ALL, rules = Rules},
+    Record = #emqx_acl{who = ?ACL_TABLE_ALL, rules = normalize_rules(Rules)},
     mria:dirty_write(Record).
 
 -spec(purge_rules() -> ok).
@@ -176,6 +177,29 @@ record_count() ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
+normalize_rules(Rules) ->
+    lists:map(fun normalize_rule/1, Rules).
+
+normalize_rule({Permission, Action, Topic}) ->
+    {normalize_permission(Permission),
+     normalize_action(Action),
+     normalize_topic(Topic)};
+normalize_rule(Rule) ->
+    error({invalid_rule, Rule}).
+
+normalize_topic(Topic) when is_list(Topic) -> list_to_binary(Topic);
+normalize_topic(Topic) when is_binary(Topic) -> Topic;
+normalize_topic(Topic) -> error({invalid_rule_topic, Topic}).
+
+normalize_action(publish) -> publish;
+normalize_action(subscribe) -> subscribe;
+normalize_action(all) -> all;
+normalize_action(Action) -> error({invalid_rule_action, Action}).
+
+normalize_permission(allow) -> allow;
+normalize_permission(deny) -> deny;
+normalize_permission(Permission) -> error({invalid_rule_permission, Permission}).
+
 do_get_rules(Key) ->
     case mnesia:dirty_read(?ACL_TABLE, Key) of
         [#emqx_acl{rules = Rules}] -> {ok, Rules};
@@ -184,9 +208,8 @@ do_get_rules(Key) ->
 
 do_authorize(_Client, _PubSub, _Topic, []) -> nomatch;
 do_authorize(Client, PubSub, Topic, [ {Permission, Action, TopicFilter} | Tail]) ->
-    case emqx_authz_rule:match(Client, PubSub, Topic,
-                               emqx_authz_rule:compile({Permission, all, Action, [TopicFilter]})
-                              ) of
+    Rule = emqx_authz_rule:compile({Permission, all, Action, [TopicFilter]}),
+    case emqx_authz_rule:match(Client, PubSub, Topic, Rule) of
         {matched, Permission} -> {matched, Permission};
         nomatch -> do_authorize(Client, PubSub, Topic, Tail)
     end.
diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
index 09c49545c..8424c0939 100644
--- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl
@@ -38,7 +38,7 @@ end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
-init_per_testcase(Config) ->
+init_per_testcase(_TestCase, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),
     Config.
 
diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl
index dd98f77d3..4d1dce248 100644
--- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl
@@ -18,10 +18,8 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include("emqx_authz.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
--include_lib("emqx/include/emqx_placeholder.hrl").
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -31,86 +29,123 @@ groups() ->
 
 init_per_suite(Config) ->
     ok = emqx_common_test_helpers:start_apps(
-           [emqx_connector, emqx_conf, emqx_authz],
-           fun set_special_configs/1
-          ),
+           [emqx_conf, emqx_authz],
+           fun set_special_configs/1),
     Config.
 
 end_per_suite(_Config) ->
-    {ok, _} = emqx:update_config(
-                [authorization],
-                #{<<"no_match">> => <<"allow">>,
-                  <<"cache">> => #{<<"enable">> => <<"true">>},
-                  <<"sources">> => []}),
-    emqx_common_test_helpers:stop_apps([emqx_authz, emqx_conf]),
-    ok.
+    ok = emqx_authz_test_lib:restore_authorizers(),
+    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
+
+init_per_testcase(_TestCase, Config) ->
+    ok = emqx_authz_test_lib:reset_authorizers(),
+    ok = setup_config(),
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    ok = emqx_authz_mnesia:purge_rules().
 
 set_special_configs(emqx_authz) ->
-    {ok, _} = emqx:update_config([authorization, cache, enable], false),
-    {ok, _} = emqx:update_config([authorization, no_match], deny),
-    {ok, _} = emqx:update_config([authorization, sources],
-                                 [#{<<"type">> => <<"built-in-database">>}]),
-    ok;
-set_special_configs(_App) ->
+    ok = emqx_authz_test_lib:reset_authorizers();
+
+set_special_configs(_) ->
     ok.
 
-init_per_testcase(t_authz, Config) ->
-     emqx_authz_mnesia:store_rules(
-       {username, <<"test_username">>},
-       [{allow, publish, <<"test/", ?PH_S_USERNAME>>},
-        {allow, subscribe, <<"eq #">>}]),
-
-     emqx_authz_mnesia:store_rules(
-       {clientid, <<"test_clientid">>},
-       [{allow, publish, <<"test/", ?PH_S_CLIENTID>>},
-        {deny, subscribe, <<"eq #">>}]),
-
-     emqx_authz_mnesia:store_rules(
-       all,
-       [{deny, all, <<"#">>}]),
-
-    Config;
-init_per_testcase(_, Config) -> Config.
-
-end_per_testcase(t_authz, Config) ->
-    ok = emqx_authz_mnesia:purge_rules(),
-    Config;
-end_per_testcase(_, Config) -> Config.
-
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
+t_username_topic_rules(_Config) ->
+    ok = test_topic_rules(username).
 
-t_authz(_) ->
-    ClientInfo1 = #{clientid => <<"test">>,
-                    username => <<"test">>,
-                    peerhost => {127,0,0,1},
-                    listener => {tcp, default}
-                   },
-    ClientInfo2 = #{clientid => <<"fake_clientid">>,
-                    username => <<"test_username">>,
-                    peerhost => {127,0,0,1},
-                    listener => {tcp, default}
-                   },
-    ClientInfo3 = #{clientid => <<"test_clientid">>,
-                    username => <<"fake_username">>,
-                    peerhost => {127,0,0,1},
-                    listener => {tcp, default}
-                   },
+t_clientid_topic_rules(_Config) ->
+    ok = test_topic_rules(clientid).
 
-    ?assertEqual(deny, emqx_access_control:authorize(
-                         ClientInfo1, subscribe, <<"#">>)),
-    ?assertEqual(deny, emqx_access_control:authorize(
-                         ClientInfo1, publish, <<"#">>)),
+t_all_topic_rules(_Config) ->
+    ok = test_topic_rules(all).
 
-    ?assertEqual(allow, emqx_access_control:authorize(
-                          ClientInfo2, publish, <<"test/test_username">>)),
-    ?assertEqual(allow, emqx_access_control:authorize(
-                          ClientInfo2, subscribe, <<"#">>)),
+test_topic_rules(Key) ->
+    ClientInfo = #{clientid => <<"clientid">>,
+                   username => <<"username">>,
+                   peerhost => {127,0,0,1},
+                   zone => default,
+                   listener => {tcp, default}
+                  },
 
-    ?assertEqual(allow, emqx_access_control:authorize(
-                          ClientInfo3, publish, <<"test/test_clientid">>)),
-    ?assertEqual(deny,  emqx_access_control:authorize(
-                          ClientInfo3, subscribe, <<"#">>)),
+    SetupSamples = fun(CInfo, Samples) ->
+                           setup_client_samples(CInfo, Samples, Key)
+                   end,
 
-    ok.
+    ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, SetupSamples),
+
+    ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, SetupSamples),
+
+    ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, SetupSamples).
+
+t_normalize_rules(_Config) ->
+    ClientInfo = #{clientid => <<"clientid">>,
+                   username => <<"username">>,
+                   peerhost => {127,0,0,1},
+                   zone => default,
+                   listener => {tcp, default}
+                  },
+
+    ok = emqx_authz_mnesia:store_rules(
+           {username, <<"username">>},
+           [{allow, publish, "t"}]),
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
+
+    ?assertException(
+       error,
+       {invalid_rule, _},
+       emqx_authz_mnesia:store_rules(
+         {username, <<"username">>},
+         [[allow, publish, <<"t">>]])),
+
+    ?assertException(
+       error,
+       {invalid_rule_action, _},
+       emqx_authz_mnesia:store_rules(
+         {username, <<"username">>},
+         [{allow, pub, <<"t">>}])),
+
+    ?assertException(
+       error,
+       {invalid_rule_permission, _},
+       emqx_authz_mnesia:store_rules(
+         {username, <<"username">>},
+         [{accept, publish, <<"t">>}])).
+
+%%------------------------------------------------------------------------------
+%% Helpers
+%%------------------------------------------------------------------------------
+
+raw_mnesia_authz_config() ->
+    #{
+        <<"enable">> => <<"true">>,
+        <<"type">> => <<"built-in-database">>
+    }.
+
+setup_client_samples(ClientInfo, Samples, Key) ->
+    ok = emqx_authz_mnesia:purge_rules(),
+    Rules = lists:flatmap(
+           fun(#{topics := Topics, permission := Permission, action := Action}) ->
+                   lists:map(
+                     fun(Topic) ->
+                             {binary_to_atom(Permission), binary_to_atom(Action), Topic}
+                     end,
+                     Topics)
+           end,
+           Samples),
+    #{username := Username, clientid := ClientId} = ClientInfo,
+    Who = case Key of
+              username -> {username, Username};
+              clientid -> {clientid, ClientId};
+              all -> all
+          end,
+    ok = emqx_authz_mnesia:store_rules(Who, Rules).
+
+setup_config() ->
+    emqx_authz_test_lib:setup_config(raw_mnesia_authz_config(), #{}).
diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
index 7db042dd8..853d78fdf 100644
--- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
@@ -56,7 +56,7 @@ end_per_suite(_Config) ->
     ok = stop_apps([emqx_resource, emqx_connector]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
-init_per_testcase(Config) ->
+init_per_testcase(_TestCase, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),
     Config.
 
diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
index 92c479f92..cda4bb447 100644
--- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
@@ -56,7 +56,7 @@ end_per_suite(_Config) ->
     ok = stop_apps([emqx_resource, emqx_connector]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
-init_per_testcase(Config) ->
+init_per_testcase(_TestCase, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),
     Config.
 
diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
index 93044e044..59d6e653d 100644
--- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl
@@ -57,7 +57,7 @@ end_per_suite(_Config) ->
     ok = stop_apps([emqx_resource, emqx_connector]),
     ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
 
-init_per_testcase(Config) ->
+init_per_testcase(_TestCase, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),
     Config.
 
diff --git a/apps/emqx_authz/test/emqx_authz_test_lib.erl b/apps/emqx_authz/test/emqx_authz_test_lib.erl
index 68686837d..9c186ec4d 100644
--- a/apps/emqx_authz/test/emqx_authz_test_lib.erl
+++ b/apps/emqx_authz/test/emqx_authz_test_lib.erl
@@ -70,6 +70,7 @@ test_samples(ClientInfo, Samples) ->
 test_no_topic_rules(ClientInfo, SetupSamples) ->
     %% No rules
 
+    ok = reset_authorizers(deny, false),
     ok = SetupSamples(ClientInfo, []),
 
     ok = test_samples(

From 68eb13d478e94f74962d8c0fd2d9c82c415bf006 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 27 Dec 2021 15:40:25 +0100
Subject: [PATCH 22/25] fix: portable shebang

---
 .../test/emqx_plugins_SUITE_data/build-demo-plugin.sh           | 2 +-
 bin/emqx                                                        | 2 +-
 bin/emqx_ctl                                                    | 2 +-
 build                                                           | 2 +-
 deploy/docker/docker-entrypoint.sh                              | 2 +-
 deploy/packages/rpm/init.script                                 | 2 +-
 scripts/apps-version-check.sh                                   | 2 +-
 scripts/check-nl-at-eof.sh                                      | 2 +-
 scripts/get-distro.sh                                           | 2 +-
 scripts/get-otp-vsn.sh                                          | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE_data/build-demo-plugin.sh b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/build-demo-plugin.sh
index fe757396b..0f79e9d8e 100755
--- a/apps/emqx_plugins/test/emqx_plugins_SUITE_data/build-demo-plugin.sh
+++ b/apps/emqx_plugins/test/emqx_plugins_SUITE_data/build-demo-plugin.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -euo pipefail
 
diff --git a/bin/emqx b/bin/emqx
index 7862a98b8..9e7f710a1 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # -*- tab-width:4;indent-tabs-mode:nil -*-
 # ex: ts=4 sw=4 et
 
diff --git a/bin/emqx_ctl b/bin/emqx_ctl
index f94946840..ef3946ee9 100755
--- a/bin/emqx_ctl
+++ b/bin/emqx_ctl
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # -*- tab-width:4;indent-tabs-mode:nil -*-
 # ex: ts=4 sw=4 et
 
diff --git a/build b/build
index 10c11d570..15fcee226 100755
--- a/build
+++ b/build
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script helps to build release artifacts.
 # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh
index d558c15ee..c2744aec4 100755
--- a/deploy/docker/docker-entrypoint.sh
+++ b/deploy/docker/docker-entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 ## EMQ docker image start script
 # Huang Rui <vowstar@gmail.com>
 # EMQ X Team <support@emqx.io>
diff --git a/deploy/packages/rpm/init.script b/deploy/packages/rpm/init.script
index bb30ee685..a2656be82 100755
--- a/deploy/packages/rpm/init.script
+++ b/deploy/packages/rpm/init.script
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # emqx
 #
diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh
index 7c2ea5eb2..76b2ca198 100755
--- a/scripts/apps-version-check.sh
+++ b/scripts/apps-version-check.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euo pipefail
 
 latest_release=$(git describe --abbrev=0 --tags)
diff --git a/scripts/check-nl-at-eof.sh b/scripts/check-nl-at-eof.sh
index f4f1ef04f..98b760390 100755
--- a/scripts/check-nl-at-eof.sh
+++ b/scripts/check-nl-at-eof.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -euo pipefail
 
diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh
index 00e95e1d8..89eafc4ee 100755
--- a/scripts/get-distro.sh
+++ b/scripts/get-distro.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 ## This script prints Linux distro name and its version number
 ## e.g. macos, centos8, ubuntu20.04
diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh
index a791318dc..699a16f3f 100755
--- a/scripts/get-otp-vsn.sh
+++ b/scripts/get-otp-vsn.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -euo pipefail
 

From 9363b6110ea16e704b167549721c0436a95b2dd7 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.ru>
Date: Fri, 24 Dec 2021 16:17:49 +0300
Subject: [PATCH 23/25] chore(authz): make test http server more robust

---
 .../emqx_authz/test/emqx_authz_http_SUITE.erl |  2 +-
 .../test/emqx_authz_http_test_server.erl      | 75 +++++++++----------
 2 files changed, 37 insertions(+), 40 deletions(-)

diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
index 7196d75ff..4e303bed7 100644
--- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
@@ -50,7 +50,7 @@ set_special_configs(_) ->
 
 init_per_testcase(_Case, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),
-    ok = emqx_authz_http_test_server:start(?HTTP_PORT, ?HTTP_PATH),
+    {ok, _} = emqx_authz_http_test_server:start_link(?HTTP_PORT, ?HTTP_PATH),
     Config.
 
 end_per_testcase(_Case, _Config) ->
diff --git a/apps/emqx_authz/test/emqx_authz_http_test_server.erl b/apps/emqx_authz/test/emqx_authz_http_test_server.erl
index 19ead4627..5cab1397e 100644
--- a/apps/emqx_authz/test/emqx_authz_http_test_server.erl
+++ b/apps/emqx_authz/test/emqx_authz_http_test_server.erl
@@ -16,20 +16,17 @@
 
 -module(emqx_authz_http_test_server).
 
--behaviour(gen_server).
+-behaviour(supervisor).
 -behaviour(cowboy_handler).
 
 % cowboy_server callbacks
 -export([init/2]).
 
-% gen_server callbacks
--export([init/1,
-         handle_call/3,
-         handle_cast/2
-        ]).
+% supervisor callbacks
+-export([init/1]).
 
 % API
--export([start/2,
+-export([start_link/2,
          stop/0,
          set_handler/1
         ]).
@@ -38,52 +35,52 @@
 %% API
 %%------------------------------------------------------------------------------
 
-start(Port, Path) ->
-    Dispatch = cowboy_router:compile([
-        {'_', [{Path, ?MODULE, []}]}
-    ]),
-    {ok, _} = cowboy:start_clear(?MODULE,
-        [{port, Port}],
-        #{env => #{dispatch => Dispatch}}
-    ),
-    {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []),
-    ok.
+start_link(Port, Path) ->
+    supervisor:start_link({local, ?MODULE}, ?MODULE, [Port, Path]).
 
 stop() ->
-    gen_server:stop(?MODULE),
-    cowboy:stop_listener(?MODULE).
+    gen_server:stop(?MODULE).
 
 set_handler(F) when is_function(F, 2) ->
-    gen_server:call(?MODULE, {set_handler, F}).
+    true = ets:insert(?MODULE, {handler, F}),
+    ok.
 
 %%------------------------------------------------------------------------------
-%% gen_server API
+%% supervisor API
 %%------------------------------------------------------------------------------
 
-init([]) ->
-    F = fun(Req0, State) ->
-                Req = cowboy_req:reply(
-                        400,
-                        #{<<"content-type">> => <<"text/plain">>},
-                        <<"">>,
-                        Req0),
-                {ok, Req, State}
-        end,
-    {ok, F}.
+init([Port, Path]) ->
+    Dispatch = cowboy_router:compile(
+                 [
+                  {'_', [{Path, ?MODULE, []}]}
+                 ]),
+    TransOpts = #{socket_opts => [{port, Port}],
+                  connection_type => supervisor},
+    ProtoOpts = #{env => #{dispatch => Dispatch}},
 
-handle_cast(_, F) ->
-    {noreply, F}.
+    Tab = ets:new(?MODULE, [set, named_table, public]),
+    ets:insert(Tab, {handler, fun default_handler/2}),
 
-handle_call({set_handler, F}, _From, _F) ->
-    {reply, ok, F};
-
-handle_call(get_handler, _From, F) ->
-    {reply, F, F}.
+    ChildSpec = ranch:child_spec(?MODULE, ranch_tcp, TransOpts, cowboy_clear, ProtoOpts),
+    {ok, {{one_for_one, 10, 10}, [ChildSpec]}}.
 
 %%------------------------------------------------------------------------------
 %% cowboy_server API
 %%------------------------------------------------------------------------------
 
 init(Req, State) ->
-    Handler = gen_server:call(?MODULE, get_handler),
+    [{handler, Handler}] = ets:lookup(?MODULE, handler),
     Handler(Req, State).
+
+%%------------------------------------------------------------------------------
+%% Internal functions
+%%------------------------------------------------------------------------------
+
+default_handler(Req0, State) ->
+    Req = cowboy_req:reply(
+            400,
+            #{<<"content-type">> => <<"text/plain">>},
+            <<"">>,
+            Req0),
+    {ok, Req, State}.
+

From dd36bef3bb09c6e60c80d7f41cb11a46b9b5b64f Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Fri, 24 Dec 2021 10:56:50 -0300
Subject: [PATCH 24/25] fix(install_upgrade): remove `vm_args` backup in
 `install`

The `install_upgrade.escript` depended on being able to retrieve the
`vm_args` argument from `init`, but that options has been removed
recently.  Since this backup was unused, we also remove it here.
---
 bin/install_upgrade.escript | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript
index 97548cba8..30430d77a 100755
--- a/bin/install_upgrade.escript
+++ b/bin/install_upgrade.escript
@@ -248,10 +248,6 @@ parse_version(V) when is_list(V) ->
     hd(string:tokens(V,"/")).
 
 check_and_install(TargetNode, Vsn) ->
-    %% Backup the vm.args. VM args should be unchanged during hot upgrade
-    %% but we still backup it here
-    {ok, [[CurrVmArgs]]} = rpc:call(TargetNode, init, get_argument, [vm_args], ?TIMEOUT),
-    {ok, _} = file:copy(CurrVmArgs, filename:join(["releases", Vsn, "vm.args"])),
     %% Backup the sys.config, this will be used when we check and install release
     %% NOTE: We cannot backup the old sys.config directly, because the
     %% configs for plugins are only in app-envs, not in the old sys.config

From 5c615e583f9143326e511596f76c158feea7e8cc Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 28 Dec 2021 09:09:21 -0300
Subject: [PATCH 25/25] fix(install_upgrade): remove `cuttlefish` reference

This file is not present in the current release anymore.
---
 bin/install_upgrade.escript | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript
index 30430d77a..501ba5487 100755
--- a/bin/install_upgrade.escript
+++ b/bin/install_upgrade.escript
@@ -304,7 +304,7 @@ permafy(TargetNode, RelName, Vsn) ->
                   make_permanent, [Vsn], ?TIMEOUT),
     ?INFO("Made release permanent: ~p", [Vsn]),
     %% upgrade/downgrade the scripts by replacing them
-    Scripts = [RelNameStr, RelNameStr++"_ctl", "cuttlefish", "nodetool",
+    Scripts = [RelNameStr, RelNameStr ++ "_ctl", "nodetool",
                "install_upgrade.escript"],
     [{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]),
                          filename:join(["bin", File]))