From 668180388c08c6655358cb92a0f9a7216e6f2907 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Mon, 20 Dec 2021 09:23:00 +0800
Subject: [PATCH 1/7] feat(trace): replace logger_formatter by
 emqx_trace_formatter

---
 apps/emqx/include/logger.hrl                  |   2 +
 apps/emqx/src/emqx_broker.erl                 |   3 +-
 apps/emqx/src/emqx_channel.erl                |   2 +-
 apps/emqx/src/emqx_connection.erl             |  15 ++-
 apps/emqx/src/emqx_packet.erl                 |   3 +-
 apps/emqx/src/emqx_trace/emqx_trace.erl       | 101 ++++++++++++------
 .../src/emqx_trace/emqx_trace_formatter.erl   |  60 +++++++++++
 .../src/emqx_trace/emqx_trace_handler.erl     |  65 +++++------
 apps/emqx/src/emqx_ws_connection.erl          |  11 +-
 apps/emqx/test/emqx_trace_handler_SUITE.erl   |  18 ++--
 .../src/emqx_connector_http.erl               |   5 +-
 .../src/emqx_connector_ldap.erl               |   5 +-
 .../src/emqx_connector_mongo.erl              |   5 +-
 .../src/emqx_connector_mqtt.erl               |   4 +-
 .../src/emqx_connector_mysql.erl              |   4 +-
 .../src/emqx_connector_pgsql.erl              |   4 +-
 .../src/emqx_connector_redis.erl              |   4 +-
 .../src/emqx_mgmt_api_trace.erl               |   4 +-
 apps/emqx_management/src/emqx_mgmt_cli.erl    |  27 +++--
 .../src/emqx_rule_outputs.erl                 |   4 +-
 .../src/emqx_rule_runtime.erl                 |   2 +-
 .../src/emqx_rule_sqltester.erl               |   2 +-
 22 files changed, 214 insertions(+), 136 deletions(-)
 create mode 100644 apps/emqx/src/emqx_trace/emqx_trace_formatter.erl

diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl
index c2ee5ab95..ecedfafe7 100644
--- a/apps/emqx/include/logger.hrl
+++ b/apps/emqx/include/logger.hrl
@@ -69,6 +69,8 @@
                 ok
         end).
 
+-define(TRACE(Action, Meta, Msg), emqx_trace:log(Action, Meta, Msg)).
+
 %% print to 'user' group leader
 -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
 -define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)).
diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl
index a82ab9b45..6dce69136 100644
--- a/apps/emqx/src/emqx_broker.erl
+++ b/apps/emqx/src/emqx_broker.erl
@@ -205,8 +205,7 @@ publish(Msg) when is_record(Msg, message) ->
     emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'),
     case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of
         #message{headers = #{allow_publish := false}} ->
-            ?SLOG(debug, #{msg => "message_not_published",
-                           payload => emqx_message:to_log_map(Msg)}),
+            ?TRACE("NotAllow", #{payload => emqx_message:to_log_map(Msg)}, "message_not_published"),
             [];
         Msg1 = #message{topic = Topic} ->
             emqx_persistent_session:persist_message(Msg1),
diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index eb71aca58..7fe366ad1 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -292,7 +292,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
                    fun check_banned/2
                   ], ConnPkt, Channel#channel{conn_state = connecting}) of
         {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
-            ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]),
+            ?TRACE("RECV", #{}, Packet),
             NChannel1 = NChannel#channel{
                                         will_msg = emqx_packet:will_msg(NConnPkt),
                                         alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 6919c6ff8..b8a6b4b7b 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -449,14 +449,12 @@ handle_msg({'$gen_cast', Req}, State) ->
     {ok, NewState};
 
 handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl ->
-    ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => Inet}),
     Oct = iolist_size(Data),
     inc_counter(incoming_bytes, Oct),
     ok = emqx_metrics:inc('bytes.received', Oct),
     when_bytes_in(Oct, Data, State);
 
 handle_msg({quic, Data, _Sock, _, _, _}, State) ->
-    ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => quic}),
     Oct = iolist_size(Data),
     inc_counter(incoming_bytes, Oct),
     ok = emqx_metrics:inc('bytes.received', Oct),
@@ -528,7 +526,7 @@ handle_msg({connack, ConnAck}, State) ->
     handle_outgoing(ConnAck, State);
 
 handle_msg({close, Reason}, State) ->
-    ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}),
+    ?TRACE("CLOSE", #{reason => Reason}, "force_socket_close"),
     handle_info({sock_closed, Reason}, close_socket(State));
 
 handle_msg({event, connected}, State = #state{channel = Channel}) ->
@@ -566,7 +564,8 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport,
         Channel1 = emqx_channel:set_conn_state(disconnected, Channel),
         emqx_congestion:cancel_alarms(Socket, Transport, Channel1),
         emqx_channel:terminate(Reason, Channel1),
-        close_socket_ok(State)
+        close_socket_ok(State),
+        ?TRACE("TERMINATE", #{reason => Reason}, "terminated")
     catch
         E : C : S ->
             ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S})
@@ -716,7 +715,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
 
 handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) ->
     ok = inc_incoming_stats(Packet),
-    ?SLOG(debug, #{msg => "RECV_packet", packet => emqx_packet:format(Packet)}),
+    ?TRACE("RECV", #{}, Packet),
     with_channel(handle_in, [Packet], State);
 
 handle_incoming(FrameError, State) ->
@@ -760,10 +759,8 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
                     ok = emqx_metrics:inc('delivery.dropped.too_large'),
                     ok = emqx_metrics:inc('delivery.dropped'),
                     <<>>;
-            Data -> ?SLOG(debug, #{
-                        msg => "SEND_packet",
-                        packet => emqx_packet:format(Packet)
-                    }),
+            Data ->
+                    ?TRACE("SEND", #{}, Packet),
                     ok = inc_outgoing_stats(Packet),
                     Data
         catch
diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl
index 60835d4ab..02165a6b5 100644
--- a/apps/emqx/src/emqx_packet.erl
+++ b/apps/emqx/src/emqx_packet.erl
@@ -453,7 +453,7 @@ format_variable(undefined, _) ->
 format_variable(Variable, undefined) ->
     format_variable(Variable);
 format_variable(Variable, Payload) ->
-    io_lib:format("~ts, Payload=~0p", [format_variable(Variable), Payload]).
+    io_lib:format("~ts, Payload=~ts", [format_variable(Variable), Payload]).
 
 format_variable(#mqtt_packet_connect{
                  proto_ver    = ProtoVer,
@@ -520,4 +520,3 @@ format_password(_Password) -> '******'.
 i(true)  -> 1;
 i(false) -> 0;
 i(I) when is_integer(I) -> I.
-
diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl
index 42e4d0baf..f4679e073 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace.erl
@@ -26,6 +26,7 @@
 -export([ publish/1
         , subscribe/3
         , unsubscribe/2
+        , log/3
         ]).
 
 -export([ start_link/0
@@ -36,6 +37,7 @@
         , delete/1
         , clear/0
         , update/2
+        , check/0
         ]).
 
 -export([ format/1
@@ -50,6 +52,8 @@
 
 -define(TRACE, ?MODULE).
 -define(MAX_SIZE, 30).
+-define(TRACE_FILTER, emqx_trace_filter).
+-define(OWN_KEYS,[level,filters,filter_default,handlers]).
 
 -ifdef(TEST).
 -export([ log_file/2
@@ -80,27 +84,53 @@ mnesia(boot) ->
 publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore;
 publish(#message{from = From, topic = Topic, payload = Payload}) when
     is_binary(From); is_atom(From) ->
-    emqx_logger:info(
-        #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
-        "PUBLISH to ~s: ~0p",
-        [Topic, Payload]
-    ).
+    ?TRACE("PUBLISH", #{topic => Topic}, {publish, Payload}).
 
 subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore;
 subscribe(Topic, SubId, SubOpts) ->
-    emqx_logger:info(
-        #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
-        "~ts SUBSCRIBE ~ts: Options: ~0p",
-        [SubId, Topic, SubOpts]
-    ).
+    ?TRACE("SUBSCRIBE", #{topic => Topic}, {subscribe, SubId, SubOpts}).
 
 unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore;
 unsubscribe(Topic, SubOpts) ->
-    emqx_logger:info(
-        #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
-        "~ts UNSUBSCRIBE ~ts: Options: ~0p",
-        [maps:get(subid, SubOpts, ""), Topic, SubOpts]
-    ).
+    ?TRACE("UNSUBSCRIBE", #{topic => Topic}, {unsubscribe, SubOpts}).
+
+log(Action, Meta0, Msg) ->
+    case persistent_term:get(?TRACE_FILTER, undefined) of
+        undefined -> ok;
+        List ->
+            Meta = maps:merge(logger:get_process_metadata(), Meta0),
+            Log = #{level => trace, action => Action, meta => Meta, msg => Msg},
+            log_filter(List, Log)
+    end.
+
+log_filter([], _Log) -> ok;
+log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
+    case FilterFun(Log0, {Filter, Name}) of
+        stop -> stop;
+        ignore -> ignore;
+        Log ->
+            case logger_config:get(ets:whereis(logger), Id) of
+                {ok, #{module := Module} = HandlerConfig0} ->
+                    HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
+                    try Module:log(Log, HandlerConfig)
+                    catch C:R:S ->
+                        case logger:remove_handler(Id) of
+                            ok ->
+                                logger:internal_log(error, {removed_failing_handler, Id});
+                            {error,{not_found,_}} ->
+                                %% Probably already removed by other client
+                                %% Don't report again
+                                ok;
+                            {error,Reason} ->
+                                logger:internal_log(error,
+                                    {removed_handler_failed, Id, Reason, C, R, S})
+                        end
+                    end;
+                {error, {not_found, Id}} -> ok;
+                {error, Reason} -> logger:internal_log(error, {find_handle_id_failed, Id, Reason})
+            end
+    end,
+    log_filter(Rest, Log0).
 
 -spec(start_link() -> emqx_types:startlink_ret()).
 start_link() ->
@@ -161,6 +191,9 @@ update(Name, Enable) ->
            end,
     transaction(Tran).
 
+check() ->
+    erlang:send(?MODULE, {mnesia_table_event, check}).
+
 -spec get_trace_filename(Name :: binary()) ->
     {ok, FileName :: string()} | {error, not_found}.
 get_trace_filename(Name) ->
@@ -196,14 +229,13 @@ format(Traces) ->
 init([]) ->
     ok = mria:wait_for_tables([?TRACE]),
     erlang:process_flag(trap_exit, true),
-    OriginLogLevel = emqx_logger:get_primary_log_level(),
     ok = filelib:ensure_dir(trace_dir()),
     ok = filelib:ensure_dir(zip_dir()),
     {ok, _} = mnesia:subscribe({table, ?TRACE, simple}),
     Traces = get_enable_trace(),
-    ok = update_log_primary_level(Traces, OriginLogLevel),
     TRef = update_trace(Traces),
-    {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}.
+    update_trace_handler(),
+    {ok, #{timer => TRef, monitors => #{}}}.
 
 handle_call(Req, _From, State) ->
     ?SLOG(error, #{unexpected_call => Req}),
@@ -224,10 +256,10 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor
             {noreply, State#{monitors => NewMonitors}}
     end;
 handle_info({timeout, TRef, update_trace},
-    #{timer := TRef, primary_log_level := OriginLogLevel} = State) ->
+    #{timer := TRef} = State) ->
     Traces = get_enable_trace(),
-    ok = update_log_primary_level(Traces, OriginLogLevel),
     NextTRef = update_trace(Traces),
+    update_trace_handler(),
     {noreply, State#{timer => NextTRef}};
 
 handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) ->
@@ -238,11 +270,11 @@ handle_info(Info, State) ->
     ?SLOG(error, #{unexpected_info => Info}),
     {noreply, State}.
 
-terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) ->
-    ok = set_log_primary_level(OriginLogLevel),
+terminate(_Reason, #{timer := TRef}) ->
     _ = mnesia:unsubscribe({table, ?TRACE, simple}),
     emqx_misc:cancel_timer(TRef),
     stop_all_trace_handler(),
+    update_trace_handler(),
     _ = file:del_dir_r(zip_dir()),
     ok.
 
@@ -270,7 +302,7 @@ update_trace(Traces) ->
     disable_finished(Finished),
     Started = emqx_trace_handler:running(),
     {NeedRunning, AllStarted} = start_trace(Running, Started),
-    NeedStop = AllStarted -- NeedRunning,
+    NeedStop = filter_cli_handler(AllStarted) -- NeedRunning,
     ok = stop_trace(NeedStop, Started),
     clean_stale_trace_files(),
     NextTime = find_closest_time(Traces, Now),
@@ -481,11 +513,20 @@ transaction(Tran) ->
         {aborted, Reason} -> {error, Reason}
     end.
 
-update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel);
-update_log_primary_level(_, _) -> set_log_primary_level(debug).
-
-set_log_primary_level(NewLevel) ->
-    case NewLevel =/= emqx_logger:get_primary_log_level() of
-        true -> emqx_logger:set_primary_log_level(NewLevel);
-        false -> ok
+update_trace_handler() ->
+    case emqx_trace_handler:running() of
+        [] -> persistent_term:erase(?TRACE_FILTER);
+        Running ->
+            List = lists:map(fun(#{id := Id, filter_fun := FilterFun,
+                filter := Filter, name := Name}) ->
+                {Id, FilterFun, Filter, Name} end, Running),
+            case List =/= persistent_term:get(?TRACE_FILTER, undefined) of
+                true -> persistent_term:put(?TRACE_FILTER, List);
+                false -> ok
+            end
     end.
+
+filter_cli_handler(Names) ->
+    lists:filter(fun(Name) ->
+        notmatch =:= re:run(Name, "^CLI-+.", [])
+                 end, Names).
diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
new file mode 100644
index 000000000..1b37b7130
--- /dev/null
+++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
@@ -0,0 +1,60 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_trace_formatter).
+
+-export([format/2]).
+
+%%%-----------------------------------------------------------------
+%%% API
+-spec format(LogEvent, Config) -> unicode:chardata() when
+    LogEvent :: logger:log_event(),
+    Config :: logger:config().
+format(#{level := trace, msg := Msg, meta := Meta, action := Action}, _Config) ->
+    Time = calendar:system_time_to_rfc3339(erlang:system_time(second)),
+    ClientId = maps:get(clientid, Meta, ""),
+    Peername = maps:get(peername, Meta, ""),
+    MsgBin = format_msg(Msg),
+    MetaBin = format_map(maps:without([clientid, peername], Meta)),
+    [Time, " [", Action, "] ", ClientId, "@", Peername, " ", MsgBin, " ( ",
+        MetaBin, ")\n"];
+
+format(Event, Config) ->
+    emqx_logger_textfmt:format(Event, Config).
+
+format_msg(Bin)when is_binary(Bin) -> Bin;
+format_msg(List) when is_list(List) -> List;
+format_msg({publish, Payload}) ->
+    io_lib:format("Publish Payload:(~ts) TO ", [Payload]);
+format_msg({subscribe, SubId, SubOpts}) ->
+    [io_lib:format("SUBSCRIBE ~ts, Opts( ", [SubId]),
+        format_map(SubOpts), ")"];
+format_msg({unsubscribe, SubOpts}) ->
+    [io_lib:format("UNSUBSCRIBE ~ts, Opts( ", [maps:get(subid, SubOpts, "undefined")]),
+        format_map(maps:without([subid], SubOpts)), ")"];
+format_msg(Packet) ->
+    emqx_packet:format(Packet).
+
+format_map(Map) ->
+    maps:fold(fun(K, V, Acc) ->
+        [to_iolist(K), ":", to_iolist(V), " "|Acc]
+              end, [], Map).
+
+to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom);
+to_iolist(Int) when is_integer(Int) -> integer_to_list(Int);
+to_iolist(Float) when is_float(Float) -> float_to_list(Float, [{decimals, 2}]);
+to_iolist(Bin)when is_binary(Bin)  -> unicode:characters_to_binary(Bin);
+to_iolist(List) when is_list(List) -> unicode:characters_to_list(List);
+to_iolist(Term) -> io_lib:format("~0p", [Term]).
diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
index c76bf1aa9..9c991301c 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
@@ -25,6 +25,7 @@
 -export([ running/0
         , install/3
         , install/4
+        , install/5
         , uninstall/1
         , uninstall/2
         ]).
@@ -77,22 +78,18 @@ install(Type, Filter, Level, LogFile) ->
 -spec install(tracer(), logger:level() | all, string()) -> ok | {error, term()}.
 install(Who, all, LogFile) ->
     install(Who, debug, LogFile);
-install(Who, Level, LogFile) ->
-    PrimaryLevel = emqx_logger:get_primary_log_level(),
-    try logger:compare_levels(Level, PrimaryLevel) of
-        lt ->
-            {error,
-                io_lib:format(
-                    "Cannot trace at a log level (~s) "
-                    "lower than the primary log level (~s)",
-                    [Level, PrimaryLevel]
-                )};
-        _GtOrEq ->
-            install_handler(Who, Level, LogFile)
-    catch
-        error:badarg ->
-            {error, {invalid_log_level, Level}}
-    end.
+install(Who = #{name := Name, type := Type}, Level, LogFile) ->
+    HandlerId = handler_id(Name, Type),
+    Config = #{
+        level => Level,
+        formatter => formatter(Who),
+        filter_default => stop,
+        filters => filters(Who),
+        config => ?CONFIG(LogFile)
+    },
+    Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
+    show_prompts(Res, Who, "start_trace"),
+    Res.
 
 -spec uninstall(Type :: clientid | topic | ip_address,
     Name :: binary() | list()) -> ok | {error, term()}.
@@ -121,38 +118,25 @@ uninstall(HandlerId) ->
 running() ->
     lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)).
 
--spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore.
+-spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop.
 filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log;
-filter_clientid(_Log, _ExpectId) -> ignore.
+filter_clientid(_Log, _ExpectId) -> stop.
 
--spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore.
+-spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop.
 filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) ->
     case emqx_topic:match(Topic, TopicFilter) of
         true -> Log;
-        false -> ignore
+        false -> stop
     end;
-filter_topic(_Log, _ExpectId) -> ignore.
+filter_topic(_Log, _ExpectId) -> stop.
 
--spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore.
+-spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop.
 filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) ->
     case lists:prefix(IP, Peername) of
         true -> Log;
-        false -> ignore
+        false -> stop
     end;
-filter_ip_address(_Log, _ExpectId) -> ignore.
-
-install_handler(Who = #{name := Name, type := Type}, Level, LogFile) ->
-    HandlerId = handler_id(Name, Type),
-    Config = #{
-        level => Level,
-        formatter => formatter(Who),
-        filter_default => stop,
-        filters => filters(Who),
-        config => ?CONFIG(LogFile)
-    },
-    Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
-    show_prompts(Res, Who, "Start trace"),
-    Res.
+filter_ip_address(_Log, _ExpectId) -> stop.
 
 filters(#{type := clientid, filter := Filter, name := Name}) ->
     [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}];
@@ -162,7 +146,7 @@ filters(#{type := ip_address, filter := Filter, name := Name}) ->
     [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}].
 
 formatter(#{type := Type}) ->
-    {logger_formatter,
+    {emqx_trace_formatter,
         #{
             template => template(Type),
             single_line => false,
@@ -176,7 +160,6 @@ formatter(#{type := Type}) ->
 %% (actually should use `~ts`), the utf8 characters clientid will become very difficult to read.
 template(clientid) ->
     [time, " [", level, "] ", {peername, [peername, " "], []}, msg, "\n"];
-%% TODO better format when clientid is utf8.
 template(_) ->
     [time, " [", level, "] ",
         {clientid,
@@ -189,11 +172,11 @@ template(_) ->
 filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) ->
     Init = #{id => Id, level => Level, dst => Dst},
     case Filters of
-        [{Type, {_FilterFun, {Filter, Name}}}] when
+        [{Type, {FilterFun, {Filter, Name}}}] when
             Type =:= topic orelse
                 Type =:= clientid orelse
                 Type =:= ip_address ->
-            [Init#{type => Type, filter => Filter, name => Name} | Acc];
+            [Init#{type => Type, filter => Filter, name => Name, filter_fun => FilterFun} | Acc];
         _ ->
             Acc
     end.
diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl
index 375b1ae2f..6e6bc1c90 100644
--- a/apps/emqx/src/emqx_ws_connection.erl
+++ b/apps/emqx/src/emqx_ws_connection.erl
@@ -347,7 +347,6 @@ websocket_handle({binary, Data}, State) when is_list(Data) ->
     websocket_handle({binary, iolist_to_binary(Data)}, State);
 
 websocket_handle({binary, Data}, State) ->
-    ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => websocket}),
     State2 = ensure_stats_timer(State),
     {Packets, State3} = parse_incoming(Data, [], State2),
     LenMsg = erlang:length(Packets),
@@ -432,11 +431,11 @@ websocket_info(Info, State) ->
 websocket_close({_, ReasonCode, _Payload}, State) when is_integer(ReasonCode) ->
     websocket_close(ReasonCode, State);
 websocket_close(Reason, State) ->
-    ?SLOG(debug, #{msg => "websocket_closed", reason => Reason}),
+    ?TRACE("CLOSED", #{transport => websocket, reason => Reason}, "websocket_closed"),
     handle_info({sock_closed, Reason}, State).
 
 terminate(Reason, _Req, #state{channel = Channel}) ->
-    ?SLOG(debug, #{msg => "terminated", reason => Reason}),
+    ?TRACE("TERMINATE", #{transport => websocket, reason => Reason}, "webscoket_terminated"),
     emqx_channel:terminate(Reason, Channel);
 
 terminate(_Reason, _Req, _UnExpectedState) ->
@@ -480,7 +479,7 @@ handle_info({connack, ConnAck}, State) ->
     return(enqueue(ConnAck, State));
 
 handle_info({close, Reason}, State) ->
-    ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}),
+    ?TRACE("CLOSE", #{reason => Reason}, "force_socket_close"),
     return(enqueue({close, Reason}, State));
 
 handle_info({event, connected}, State = #state{channel = Channel}) ->
@@ -663,7 +662,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
 
 handle_incoming(Packet, State = #state{listener = {Type, Listener}})
   when is_record(Packet, mqtt_packet) ->
-    ?SLOG(debug, #{msg => "RECV", packet => emqx_packet:format(Packet)}),
+    ?TRACE("RECV", #{transport => websocket}, Packet),
     ok = inc_incoming_stats(Packet),
     NState = case emqx_pd:get_counter(incoming_pubs) >
                   get_active_n(Type, Listener) of
@@ -727,7 +726,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
                     ok = emqx_metrics:inc('delivery.dropped.too_large'),
                     ok = emqx_metrics:inc('delivery.dropped'),
                     <<>>;
-            Data -> ?SLOG(debug, #{msg => "SEND", packet => Packet}),
+            Data -> ?TRACE("SEND", #{transport => websocket}, Packet),
                     ok = inc_outgoing_stats(Packet),
                     Data
         catch
diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl
index abe233b58..01d587028 100644
--- a/apps/emqx/test/emqx_trace_handler_SUITE.erl
+++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl
@@ -32,36 +32,30 @@ all() -> [t_trace_clientid, t_trace_topic, t_trace_ip_address, t_trace_clientid_
 
 init_per_suite(Config) ->
     emqx_common_test_helpers:boot_modules(all),
-    emqx_common_test_helpers:start_apps([]),
+    emqx_common_test_helpers:start_apps([emqx_modules]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([]).
+    emqx_common_test_helpers:stop_apps([emqx_modules]).
 
 init_per_testcase(t_trace_clientid, Config) ->
     Config;
 init_per_testcase(_Case, Config) ->
-    ok = emqx_logger:set_log_level(debug),
     _ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()],
     Config.
 
 end_per_testcase(_Case, _Config) ->
-    ok = emqx_logger:set_log_level(warning),
     ok.
 
 t_trace_clientid(_Config) ->
     %% Start tracing
-    emqx_logger:set_log_level(error),
-    {error, _} = emqx_trace_handler:install(clientid, <<"client">>, debug, "tmp/client.log"),
-    emqx_logger:set_log_level(debug),
     %% add list clientid
     ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"),
     ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"),
     ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"),
-    {error, {invalid_log_level, bad_level}} =
-        emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"),
     {error, {handler_not_added, {file_error, ".", eisdir}}} =
         emqx_trace_handler:install(clientid, <<"client5">>, debug, "."),
+    emqx_trace:check(),
     ok = filesync(<<"client">>, clientid),
     ok = filesync(<<"client2">>, clientid),
     ok = filesync(<<"client3">>, clientid),
@@ -106,10 +100,9 @@ t_trace_clientid(_Config) ->
     ?assertEqual([], emqx_trace_handler:running()).
 
 t_trace_clientid_utf8(_) ->
-    emqx_logger:set_log_level(debug),
-
     Utf8Id = <<"client 漢字編碼"/utf8>>,
     ok = emqx_trace_handler:install(clientid, Utf8Id, debug, "tmp/client-utf8.log"),
+    emqx_trace:check(),
     {ok, T} = emqtt:start_link([{clientid, Utf8Id}]),
     emqtt:connect(T),
     [begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)],
@@ -126,9 +119,9 @@ t_trace_topic(_Config) ->
     emqtt:connect(T),
 
     %% Start tracing
-    emqx_logger:set_log_level(debug),
     ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"),
     ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"),
+    emqx_trace:check(),
     ok = filesync(<<"x/#">>, topic),
     ok = filesync(<<"y/#">>, topic),
 
@@ -174,6 +167,7 @@ t_trace_ip_address(_Config) ->
     %% Start tracing
     ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"),
     ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"),
+    emqx_trace:check(),
     ok = filesync(<<"127.0.0.1">>, ip_address),
     ok = filesync(<<"192.168.1.1">>, ip_address),
 
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 2b9bd48aa..b54d87f12 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -199,9 +199,8 @@ on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) ->
     on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State);
 on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery,
         #{pool_name := PoolName, base_path := BasePath} = State) ->
-    ?SLOG(debug, #{msg => "http connector received request",
-                   request => Request, connector => InstId,
-                   state => State}),
+    ?TRACE("QUERY", #{request => Request, connector => InstId, state => State},
+        "http connector received request"),
     NRequest = update_path(BasePath, Request),
     case Result = ehttpc:request(case KeyOrNum of
                                      undefined -> PoolName;
diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl
index 8af516b82..8aa1f9319 100644
--- a/apps/emqx_connector/src/emqx_connector_ldap.erl
+++ b/apps/emqx_connector/src/emqx_connector_ldap.erl
@@ -87,9 +87,8 @@ on_stop(InstId, #{poolname := PoolName}) ->
 
 on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) ->
     Request = {Base, Filter, Attributes},
-    ?SLOG(debug, #{msg => "ldap connector received request",
-                   request => Request, connector => InstId,
-                   state => State}),
+    ?TRACE("QUERY", #{request => Request, connector => InstId, state => State},
+        "ldap connector received request"),
     case Result = ecpool:pick_and_do(
                     PoolName,
                     {?MODULE, search, [Base, Filter, Attributes]},
diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index 6a1b15e57..d2594ab93 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -137,9 +137,8 @@ on_query(InstId,
          AfterQuery,
          #{poolname := PoolName} = State) ->
     Request = {Action, Collection, Selector, Docs},
-    ?SLOG(debug, #{msg => "mongodb connector received request",
-        request => Request, connector => InstId,
-        state => State}),
+    ?TRACE("QUERY", #{request => Request, connector => InstId, state => State},
+        "mongodb connector received request"),
     case ecpool:pick_and_do(PoolName,
                             {?MODULE, mongo_query, [Action, Collection, Selector, Docs]},
                             no_handover) of
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index f8d17ce32..6d620cc14 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -150,8 +150,8 @@ on_stop(_InstId, #{name := InstanceId}) ->
     end.
 
 on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) ->
-    ?SLOG(debug, #{msg => "send msg to remote node", message => Msg,
-        connector => InstanceId}),
+    ?TRACE("QUERY", #{message => Msg, connector => InstanceId},
+        "send msg to remote node"),
     emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
     emqx_resource:query_success(AfterQuery).
 
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index c93a1e350..def3904b4 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -85,8 +85,8 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
 on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) ->
     on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State);
 on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) ->
-    ?SLOG(debug, #{msg => "mysql connector received sql query",
-        connector => InstId, sql => SQL, state => State}),
+    ?TRACE("QUERY", #{connector => InstId, sql => SQL, state => State},
+        "mysql connector received sql query"),
     case Result = ecpool:pick_and_do(
                     PoolName,
                     {mysql, query, [SQL, Params, Timeout]},
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index f42bed666..ac864a45d 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -83,8 +83,8 @@ on_stop(InstId, #{poolname := PoolName}) ->
 on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
     on_query(InstId, {sql, SQL, []}, AfterQuery, State);
 on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) ->
-    ?SLOG(debug, #{msg => "postgresql connector received sql query",
-        connector => InstId, sql => SQL, state => State}),
+    ?TRACE("QUERY", #{connector => InstId, sql => SQL, state => State},
+        "postgresql connector received sql query"),
     case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of
         {error, Reason} ->
             ?SLOG(error, #{
diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl
index 075ede0bc..61b716b8b 100644
--- a/apps/emqx_connector/src/emqx_connector_redis.erl
+++ b/apps/emqx_connector/src/emqx_connector_redis.erl
@@ -125,8 +125,8 @@ on_stop(InstId, #{poolname := PoolName}) ->
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
 on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) ->
-    ?SLOG(debug, #{msg => "redis connector received cmd query",
-        connector => InstId, sql => Command, state => State}),
+    ?TRACE("QUERY", #{connector => InstId, sql => Command, state => State},
+        "redis connector received cmd query"),
     Result = case Type of
                  cluster -> eredis_cluster:q(PoolName, Command);
                  _ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover)
diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
index d6902d123..5669e5653 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
@@ -142,11 +142,11 @@ fields(trace) ->
             #{desc => """Filter type""",
                 nullable => false,
                 example => <<"clientid">>})},
-        {topic, hoconsc:mk(binary(),
+        {topic, hoconsc:mk(emqx_schema:unicode_binary(),
             #{desc => """support mqtt wildcard topic.""",
                 nullable => true,
                 example => <<"/dev/#">>})},
-        {clientid, hoconsc:mk(binary(),
+        {clientid, hoconsc:mk(emqx_schema:unicode_binary(),
             #{desc => """mqtt clientid.""",
                 nullable => true,
                 example => <<"dev-001">>})},
diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl
index 5885a2b17..0084855ab 100644
--- a/apps/emqx_management/src/emqx_mgmt_cli.erl
+++ b/apps/emqx_management/src/emqx_mgmt_cli.erl
@@ -395,9 +395,11 @@ trace(["stop", Operation, ClientId]) ->
 trace(["start", Operation, ClientId, LogFile]) ->
     trace(["start", Operation, ClientId, LogFile, "all"]);
 
-trace(["start", Operation, ClientId, LogFile, Level]) ->
+trace(["start", Operation, Filter, LogFile, Level]) ->
     case trace_type(Operation) of
-        {ok, Type} -> trace_on(Type, ClientId, list_to_existing_atom(Level), LogFile);
+        {ok, Type} ->
+            trace_on(name(Filter), Type, Filter,
+                list_to_existing_atom(Level), LogFile);
         error -> trace([])
     end;
 
@@ -417,20 +419,22 @@ trace(_) ->
             "Stop tracing for a client ip on local node"}
     ]).
 
-trace_on(Who, Name, Level, LogFile) ->
-    case emqx_trace_handler:install(Who, Name, Level, LogFile) of
+trace_on(Name, Type, Filter, Level, LogFile) ->
+    case emqx_trace_handler:install(Name, Type, Filter, Level, LogFile) of
         ok ->
-            emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]);
+            emqx_trace:check(),
+            emqx_ctl:print("trace ~s ~s successfully~n", [Filter, Name]);
         {error, Error} ->
-            emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Who, Name, Error])
+            emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Filter, Name, Error])
     end.
 
-trace_off(Who, Name) ->
-    case emqx_trace_handler:uninstall(Who, Name) of
+trace_off(Who, Filter) ->
+    case emqx_trace_handler:uninstall(Who, name(Filter)) of
         ok ->
-            emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]);
+            emqx_trace:check(),
+            emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Filter]);
         {error, Error} ->
-            emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Name, Error])
+            emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Filter, Error])
     end.
 
 %%--------------------------------------------------------------------
@@ -716,3 +720,6 @@ format_listen_on({Addr, Port}) when is_list(Addr) ->
     io_lib:format("~ts:~w", [Addr, Port]);
 format_listen_on({Addr, Port}) when is_tuple(Addr) ->
     io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]).
+
+name(Filter) ->
+    iolist_to_binary(["CLI-", Filter]).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl
index 61a520e81..70aa68cf5 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl
@@ -85,7 +85,7 @@ republish(Selected, #{flags := Flags, metadata := #{rule_id := RuleId}},
     Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected),
     QoS = replace_simple_var(QoSTks, Selected, 0),
     Retain = replace_simple_var(RetainTks, Selected, false),
-    ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}),
+    ?TRACE("REPUBLISH", #{topic => Topic, payload => Payload}, "republish message"),
     safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload);
 
 %% in case this is a "$events/" event
@@ -99,7 +99,7 @@ republish(Selected, #{metadata := #{rule_id := RuleId}},
     Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected),
     QoS = replace_simple_var(QoSTks, Selected, 0),
     Retain = replace_simple_var(RetainTks, Selected, false),
-    ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}),
+    ?TRACE("REPUBLISH", #{topic => Topic, payload => Payload}, "republish"),
     safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload).
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index 4225c6f72..c61296d87 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -248,7 +248,7 @@ handle_output(OutId, Selected, Envs) ->
     end.
 
 do_handle_output(BridgeId, Selected, _Envs) when is_binary(BridgeId) ->
-    ?SLOG(debug, #{msg => "output to bridge", bridge_id => BridgeId}),
+    ?TRACE("SEND", #{bridge_id => BridgeId}, "output to bridge"),
     emqx_bridge:send_message(BridgeId, Selected);
 do_handle_output(#{mod := Mod, func := Func, args := Args}, Selected, Envs) ->
     Mod:Func(Selected, Envs, Args).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
index 74ec1bb1c..7a9da25a2 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
@@ -77,7 +77,7 @@ flatten([D1 | L]) when is_list(D1) ->
     D1 ++ flatten(L).
 
 echo_action(Data, Envs) ->
-    ?SLOG(debug, #{msg => "testing_rule_sql_ok", data => Data, envs => Envs}),
+    ?TRACE("TEST", #{data => Data, envs => Envs}, "testing_rule_sql_ok"),
     Data.
 
 fill_default_values(Event, Context) ->

From 4b6bba11eb3ddec8fbd64745f44b0bc9f42035d5 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Tue, 28 Dec 2021 23:46:22 +0800
Subject: [PATCH 2/7] feat(trace): struct log for trace

---
 apps/emqx/include/logger.hrl                  |  2 +-
 apps/emqx/src/emqx_broker.erl                 |  2 +-
 apps/emqx/src/emqx_channel.erl                |  2 +-
 apps/emqx/src/emqx_connection.erl             | 10 +--
 apps/emqx/src/emqx_packet.erl                 | 85 +++++++++++--------
 apps/emqx/src/emqx_schema.erl                 | 17 ++++
 apps/emqx/src/emqx_trace/emqx_trace.erl       | 15 ++--
 .../src/emqx_trace/emqx_trace_formatter.erl   | 43 +++++-----
 .../src/emqx_trace/emqx_trace_handler.erl     | 11 ++-
 apps/emqx/src/emqx_ws_connection.erl          | 10 +--
 .../emqx_conf/test/emqx_cluster_rpc_SUITE.erl | 14 ++-
 .../src/emqx_connector_http.erl               |  4 +-
 .../src/emqx_connector_ldap.erl               |  9 +-
 .../src/emqx_connector_mongo.erl              |  6 +-
 .../src/emqx_connector_mqtt.erl               |  9 +-
 .../src/emqx_connector_mysql.erl              |  5 +-
 .../src/emqx_connector_pgsql.erl              |  6 +-
 .../src/emqx_connector_redis.erl              |  6 +-
 .../src/emqx_rule_outputs.erl                 |  4 +-
 .../src/emqx_rule_runtime.erl                 |  2 +-
 .../src/emqx_rule_sqltester.erl               |  2 +-
 21 files changed, 154 insertions(+), 110 deletions(-)

diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl
index ecedfafe7..42d598ef9 100644
--- a/apps/emqx/include/logger.hrl
+++ b/apps/emqx/include/logger.hrl
@@ -69,7 +69,7 @@
                 ok
         end).
 
--define(TRACE(Action, Meta, Msg), emqx_trace:log(Action, Meta, Msg)).
+-define(TRACE(Event, Msg, Meta), emqx_trace:log(Event, Msg, Meta)).
 
 %% print to 'user' group leader
 -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl
index 6dce69136..dec753fc2 100644
--- a/apps/emqx/src/emqx_broker.erl
+++ b/apps/emqx/src/emqx_broker.erl
@@ -205,7 +205,7 @@ publish(Msg) when is_record(Msg, message) ->
     emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'),
     case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of
         #message{headers = #{allow_publish := false}} ->
-            ?TRACE("NotAllow", #{payload => emqx_message:to_log_map(Msg)}, "message_not_published"),
+            ?TRACE("MQTT", "msg_publish_not_allowed", #{message => emqx_message:to_log_map(Msg)}),
             [];
         Msg1 = #message{topic = Topic} ->
             emqx_persistent_session:persist_message(Msg1),
diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index 7fe366ad1..0f83b04ff 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -292,7 +292,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
                    fun check_banned/2
                   ], ConnPkt, Channel#channel{conn_state = connecting}) of
         {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
-            ?TRACE("RECV", #{}, Packet),
+            ?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
             NChannel1 = NChannel#channel{
                                         will_msg = emqx_packet:will_msg(NConnPkt),
                                         alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index b8a6b4b7b..37e15f522 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -526,7 +526,7 @@ handle_msg({connack, ConnAck}, State) ->
     handle_outgoing(ConnAck, State);
 
 handle_msg({close, Reason}, State) ->
-    ?TRACE("CLOSE", #{reason => Reason}, "force_socket_close"),
+    ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}),
     handle_info({sock_closed, Reason}, close_socket(State));
 
 handle_msg({event, connected}, State = #state{channel = Channel}) ->
@@ -565,7 +565,7 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport,
         emqx_congestion:cancel_alarms(Socket, Transport, Channel1),
         emqx_channel:terminate(Reason, Channel1),
         close_socket_ok(State),
-        ?TRACE("TERMINATE", #{reason => Reason}, "terminated")
+        ?TRACE("SOCKET", "tcp_socket_terminated", #{reason => Reason})
     catch
         E : C : S ->
             ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S})
@@ -715,7 +715,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
 
 handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) ->
     ok = inc_incoming_stats(Packet),
-    ?TRACE("RECV", #{}, Packet),
+    ?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
     with_channel(handle_in, [Packet], State);
 
 handle_incoming(FrameError, State) ->
@@ -754,13 +754,13 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
             <<>> -> ?SLOG(warning, #{
                         msg => "packet_is_discarded",
                         reason => "frame_is_too_large",
-                        packet => emqx_packet:format(Packet)
+                        packet => emqx_packet:format(Packet, null)
                     }),
                     ok = emqx_metrics:inc('delivery.dropped.too_large'),
                     ok = emqx_metrics:inc('delivery.dropped'),
                     <<>>;
             Data ->
-                    ?TRACE("SEND", #{}, Packet),
+                    ?TRACE("MQTT", "mqtt_packet_sent", #{packet => Packet}),
                     ok = inc_outgoing_stats(Packet),
                     Data
         catch
diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl
index 02165a6b5..53e4eabfe 100644
--- a/apps/emqx/src/emqx_packet.erl
+++ b/apps/emqx/src/emqx_packet.erl
@@ -44,7 +44,9 @@
         , will_msg/1
         ]).
 
--export([format/1]).
+-export([ format/1
+        , format/2
+        ]).
 
 -define(TYPE_NAMES,
         { 'CONNECT'
@@ -435,25 +437,28 @@ will_msg(#mqtt_packet_connect{clientid     = ClientId,
 
 %% @doc Format packet
 -spec(format(emqx_types:packet()) -> iolist()).
-format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}) ->
-    format_header(Header, format_variable(Variable, Payload)).
+format(Packet) -> format(Packet, emqx_trace_handler:payload_encode()).
+
+%% @doc Format packet
+-spec(format(emqx_types:packet(), hex | text | null) -> iolist()).
+format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, PayloadEncode) ->
+    HeaderIO = format_header(Header),
+    case format_variable(Variable, Payload, PayloadEncode) of
+        "" -> HeaderIO;
+        VarIO -> [HeaderIO,",", VarIO]
+    end.
 
 format_header(#mqtt_packet_header{type = Type,
                                   dup = Dup,
                                   qos = QoS,
-                                  retain = Retain}, S) ->
-    S1 = case S == undefined of
-             true -> <<>>;
-             false -> [", ", S]
-         end,
-    io_lib:format("~ts(Q~p, R~p, D~p~ts)", [type_name(Type), QoS, i(Retain), i(Dup), S1]).
+                                  retain = Retain}) ->
+    io_lib:format("~ts(Q~p, R~p, D~p)", [type_name(Type), QoS, i(Retain), i(Dup)]).
 
-format_variable(undefined, _) ->
-    undefined;
-format_variable(Variable, undefined) ->
-    format_variable(Variable);
-format_variable(Variable, Payload) ->
-    io_lib:format("~ts, Payload=~ts", [format_variable(Variable), Payload]).
+format_variable(undefined, _, _) -> "";
+format_variable(Variable, undefined, PayloadEncode) ->
+    format_variable(Variable, PayloadEncode);
+format_variable(Variable, Payload, PayloadEncode) ->
+    [format_variable(Variable, PayloadEncode), format_payload(Payload, PayloadEncode)].
 
 format_variable(#mqtt_packet_connect{
                  proto_ver    = ProtoVer,
@@ -467,55 +472,63 @@ format_variable(#mqtt_packet_connect{
                  will_topic   = WillTopic,
                  will_payload = WillPayload,
                  username     = Username,
-                 password     = Password}) ->
-    Format = "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts",
-    Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)],
-    {Format1, Args1} = if
-                        WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~ts, Payload=~0p)",
-                                     Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]};
-                        true -> {Format, Args}
-                       end,
-    io_lib:format(Format1, Args1);
+                 password     = Password},
+    PayloadEncode) ->
+    Base = io_lib:format(
+        "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts",
+        [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)]),
+    case WillFlag of
+        true ->
+            [Base, io_lib:format(", Will(Q~p, R~p, Topic=~ts ",
+                [WillQoS, i(WillRetain), WillTopic]),
+                format_payload(WillPayload, PayloadEncode), ")"];
+        false ->
+            Base
+    end;
 
 format_variable(#mqtt_packet_disconnect
-                {reason_code = ReasonCode}) ->
+                {reason_code = ReasonCode}, _) ->
     io_lib:format("ReasonCode=~p", [ReasonCode]);
 
 format_variable(#mqtt_packet_connack{ack_flags   = AckFlags,
-                                     reason_code = ReasonCode}) ->
+                                     reason_code = ReasonCode}, _) ->
     io_lib:format("AckFlags=~p, ReasonCode=~p", [AckFlags, ReasonCode]);
 
 format_variable(#mqtt_packet_publish{topic_name = TopicName,
-                                     packet_id  = PacketId}) ->
+                                     packet_id  = PacketId}, _) ->
     io_lib:format("Topic=~ts, PacketId=~p", [TopicName, PacketId]);
 
 format_variable(#mqtt_packet_puback{packet_id = PacketId,
-                                    reason_code = ReasonCode}) ->
+                                    reason_code = ReasonCode}, _) ->
     io_lib:format("PacketId=~p, ReasonCode=~p", [PacketId, ReasonCode]);
 
 format_variable(#mqtt_packet_subscribe{packet_id     = PacketId,
-                                       topic_filters = TopicFilters}) ->
+                                       topic_filters = TopicFilters}, _) ->
     io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, TopicFilters]);
 
 format_variable(#mqtt_packet_unsubscribe{packet_id     = PacketId,
-                                         topic_filters = Topics}) ->
+                                         topic_filters = Topics}, _) ->
     io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, Topics]);
 
 format_variable(#mqtt_packet_suback{packet_id = PacketId,
-                                    reason_codes = ReasonCodes}) ->
+                                    reason_codes = ReasonCodes}, _) ->
     io_lib:format("PacketId=~p, ReasonCodes=~p", [PacketId, ReasonCodes]);
 
-format_variable(#mqtt_packet_unsuback{packet_id = PacketId}) ->
+format_variable(#mqtt_packet_unsuback{packet_id = PacketId}, _) ->
     io_lib:format("PacketId=~p", [PacketId]);
 
-format_variable(#mqtt_packet_auth{reason_code = ReasonCode}) ->
+format_variable(#mqtt_packet_auth{reason_code = ReasonCode}, _) ->
     io_lib:format("ReasonCode=~p", [ReasonCode]);
 
-format_variable(PacketId) when is_integer(PacketId) ->
+format_variable(PacketId, _) when is_integer(PacketId) ->
     io_lib:format("PacketId=~p", [PacketId]).
 
-format_password(undefined) -> undefined;
-format_password(_Password) -> '******'.
+format_password(undefined) -> "undefined";
+format_password(_Password) -> "******".
+
+format_payload(Payload, text) -> ["Payload=", io_lib:format("~ts", [Payload])];
+format_payload(Payload, hex) -> ["Payload(hex)=", binary:encode_hex(Payload)];
+format_payload(_, null) -> "Payload=******".
 
 i(true)  -> 1;
 i(false) -> 0;
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 85b411217..cbf21683d 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -184,6 +184,12 @@ roots(low) ->
     , {"latency_stats",
        sc(ref("latency_stats"),
           #{})}
+    , {"trace",
+       sc(ref("trace"),
+          #{desc => """
+Real-time filtering logs for the ClientID or Topic or IP for debugging.
+"""
+          })}
     ].
 
 fields("persistent_session_store") ->
@@ -981,6 +987,17 @@ when deactivated, but after the retention time.
 fields("latency_stats") ->
     [ {"samples", sc(integer(), #{default => 10,
                                   desc => "the number of smaples for calculate the average latency of delivery"})}
+    ];
+fields("trace") ->
+    [ {"payload_encode", sc(hoconsc:enum([hex, text, null]), #{
+        default => text,
+        desc => """
+Determine the format of the payload format in the trace file.</br>
+- `text`: Text-based protocol or plain text protocol. It is recommended when payload is json encode.</br>
+- `hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.</br>
+- `null`: Don't show payload in trace log file.
+        """
+    })}
     ].
 
 mqtt_listener() ->
diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl
index f4679e073..7bca3ca3c 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace.erl
@@ -84,22 +84,22 @@ mnesia(boot) ->
 publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore;
 publish(#message{from = From, topic = Topic, payload = Payload}) when
     is_binary(From); is_atom(From) ->
-    ?TRACE("PUBLISH", #{topic => Topic}, {publish, Payload}).
+    ?TRACE("PUBLISH", "publish_to", #{topic => Topic, payload => Payload}).
 
 subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore;
 subscribe(Topic, SubId, SubOpts) ->
-    ?TRACE("SUBSCRIBE", #{topic => Topic}, {subscribe, SubId, SubOpts}).
+    ?TRACE("SUBSCRIBE", "subscribe", #{topic => Topic, sub_opts => SubOpts, sub_id => SubId}).
 
 unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore;
 unsubscribe(Topic, SubOpts) ->
-    ?TRACE("UNSUBSCRIBE", #{topic => Topic}, {unsubscribe, SubOpts}).
+    ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}).
 
-log(Action, Meta0, Msg) ->
+log(Event, Msg, Meta0) ->
     case persistent_term:get(?TRACE_FILTER, undefined) of
         undefined -> ok;
         List ->
             Meta = maps:merge(logger:get_process_metadata(), Meta0),
-            Log = #{level => trace, action => Action, meta => Meta, msg => Msg},
+            Log = #{level => trace, event => Event, meta => Meta, msg => Msg},
             log_filter(List, Log)
     end.
 
@@ -112,11 +112,12 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
             case logger_config:get(ets:whereis(logger), Id) of
                 {ok, #{module := Module} = HandlerConfig0} ->
                     HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
+                    io:format("~p~n", [{Module, Log, HandlerConfig}]),
                     try Module:log(Log, HandlerConfig)
                     catch C:R:S ->
                         case logger:remove_handler(Id) of
                             ok ->
-                                logger:internal_log(error, {removed_failing_handler, Id});
+                                logger:internal_log(error, {removed_failing_handler, Id, C, R, S});
                             {error,{not_found,_}} ->
                                 %% Probably already removed by other client
                                 %% Don't report again
@@ -528,5 +529,5 @@ update_trace_handler() ->
 
 filter_cli_handler(Names) ->
     lists:filter(fun(Name) ->
-        notmatch =:= re:run(Name, "^CLI-+.", [])
+        nomatch =:= re:run(Name, "^CLI-+.", [])
                  end, Names).
diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
index 1b37b7130..a9c4ca31d 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
@@ -22,35 +22,36 @@
 -spec format(LogEvent, Config) -> unicode:chardata() when
     LogEvent :: logger:log_event(),
     Config :: logger:config().
-format(#{level := trace, msg := Msg, meta := Meta, action := Action}, _Config) ->
+format(#{level := trace, event := Event, meta := Meta, msg := Msg},
+    #{payload_encode := PEncode}) ->
     Time = calendar:system_time_to_rfc3339(erlang:system_time(second)),
     ClientId = maps:get(clientid, Meta, ""),
     Peername = maps:get(peername, Meta, ""),
-    MsgBin = format_msg(Msg),
-    MetaBin = format_map(maps:without([clientid, peername], Meta)),
-    [Time, " [", Action, "] ", ClientId, "@", Peername, " ", MsgBin, " ( ",
-        MetaBin, ")\n"];
+    MetaBin = format_meta(Meta, PEncode),
+    [Time, " [", Event, "] ", ClientId, "@", Peername, " msg: ", Msg, MetaBin, "\n"];
 
 format(Event, Config) ->
     emqx_logger_textfmt:format(Event, Config).
 
-format_msg(Bin)when is_binary(Bin) -> Bin;
-format_msg(List) when is_list(List) -> List;
-format_msg({publish, Payload}) ->
-    io_lib:format("Publish Payload:(~ts) TO ", [Payload]);
-format_msg({subscribe, SubId, SubOpts}) ->
-    [io_lib:format("SUBSCRIBE ~ts, Opts( ", [SubId]),
-        format_map(SubOpts), ")"];
-format_msg({unsubscribe, SubOpts}) ->
-    [io_lib:format("UNSUBSCRIBE ~ts, Opts( ", [maps:get(subid, SubOpts, "undefined")]),
-        format_map(maps:without([subid], SubOpts)), ")"];
-format_msg(Packet) ->
-    emqx_packet:format(Packet).
+format_meta(Meta0, Encode) ->
+    Packet = format_packet(maps:get(packet, Meta0, undefined), Encode),
+    Payload = format_payload(maps:get(payload, Meta0, undefined), Encode),
+    Meta1 = maps:without([msg, clientid, peername, packet, payload], Meta0),
+    case Meta1 =:= #{} of
+        true -> [Packet, Payload];
+        false ->
+            Meta2 = lists:map(fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end,
+                maps:to_list(Meta1)),
+            [Packet, ", ", lists:join(",", Meta2), Payload]
+    end.
 
-format_map(Map) ->
-    maps:fold(fun(K, V, Acc) ->
-        [to_iolist(K), ":", to_iolist(V), " "|Acc]
-              end, [], Map).
+format_packet(undefined, _) -> "";
+format_packet(Packet, Encode) -> [", packet: ", emqx_packet:format(Packet, Encode)].
+
+format_payload(undefined, _) -> "";
+format_payload(Payload, text) -> [", payload: ", io_lib:format("~ts", [Payload])];
+format_payload(Payload, hex) -> [", payload(hex): ", binary:encode_hex(Payload)];
+format_payload(_, null) -> ", payload=******".
 
 to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom);
 to_iolist(Int) when is_integer(Int) -> integer_to_list(Int);
diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
index 9c991301c..d1fbd20b8 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
@@ -35,8 +35,10 @@
         , filter_topic/2
         , filter_ip_address/2
         ]).
+-export([template/1]).
 
 -export([handler_id/2]).
+-export([payload_encode/0]).
 
 -type tracer() :: #{
          name := binary(),
@@ -145,13 +147,14 @@ filters(#{type := topic, filter := Filter, name := Name}) ->
 filters(#{type := ip_address, filter := Filter, name := Name}) ->
     [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}].
 
-formatter(#{type := Type}) ->
+formatter(#{type := _Type}) ->
     {emqx_trace_formatter,
         #{
-            template => template(Type),
+            template => [],
             single_line => false,
             max_size => unlimited,
-            depth => unlimited
+            depth => unlimited,
+            payload_encode => payload_encode()
         }
     }.
 
@@ -181,6 +184,8 @@ filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc)
             Acc
     end.
 
+payload_encode() -> emqx_config:get([trace, payload_encode], text).
+
 handler_id(Name, Type) ->
     try
         do_handler_id(Name, Type)
diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl
index 6e6bc1c90..f70d7ac6c 100644
--- a/apps/emqx/src/emqx_ws_connection.erl
+++ b/apps/emqx/src/emqx_ws_connection.erl
@@ -431,11 +431,11 @@ websocket_info(Info, State) ->
 websocket_close({_, ReasonCode, _Payload}, State) when is_integer(ReasonCode) ->
     websocket_close(ReasonCode, State);
 websocket_close(Reason, State) ->
-    ?TRACE("CLOSED", #{transport => websocket, reason => Reason}, "websocket_closed"),
+    ?TRACE("SOCKET", "websocket_closed", #{reason => Reason}),
     handle_info({sock_closed, Reason}, State).
 
 terminate(Reason, _Req, #state{channel = Channel}) ->
-    ?TRACE("TERMINATE", #{transport => websocket, reason => Reason}, "webscoket_terminated"),
+    ?TRACE("SOCKET", "websocket_terminated", #{reason => Reason}),
     emqx_channel:terminate(Reason, Channel);
 
 terminate(_Reason, _Req, _UnExpectedState) ->
@@ -479,7 +479,7 @@ handle_info({connack, ConnAck}, State) ->
     return(enqueue(ConnAck, State));
 
 handle_info({close, Reason}, State) ->
-    ?TRACE("CLOSE", #{reason => Reason}, "force_socket_close"),
+    ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}),
     return(enqueue({close, Reason}, State));
 
 handle_info({event, connected}, State = #state{channel = Channel}) ->
@@ -662,7 +662,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
 
 handle_incoming(Packet, State = #state{listener = {Type, Listener}})
   when is_record(Packet, mqtt_packet) ->
-    ?TRACE("RECV", #{transport => websocket}, Packet),
+    ?TRACE("WS-MQTT", "mqtt_packet_received", #{packet => Packet}),
     ok = inc_incoming_stats(Packet),
     NState = case emqx_pd:get_counter(incoming_pubs) >
                   get_active_n(Type, Listener) of
@@ -726,7 +726,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
                     ok = emqx_metrics:inc('delivery.dropped.too_large'),
                     ok = emqx_metrics:inc('delivery.dropped'),
                     <<>>;
-            Data -> ?TRACE("SEND", #{transport => websocket}, Packet),
+            Data -> ?TRACE("WS-MQTT", "mqtt_packet_sent", #{packet => Packet}),
                     ok = inc_outgoing_stats(Packet),
                     Data
         catch
diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl
index ad74faf99..66b05c95a 100644
--- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl
+++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl
@@ -74,9 +74,19 @@ t_base_test(_Config) ->
     ?assertEqual(node(), maps:get(initiator, Query)),
     ?assert(maps:is_key(created_at, Query)),
     ?assertEqual(ok, receive_msg(3, test)),
+    ?assertEqual({ok, 2, ok}, emqx_cluster_rpc:multicall(M, F, A)),
     {atomic, Status} = emqx_cluster_rpc:status(),
-    ?assertEqual(3, length(Status)),
-    ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 1 end, Status)),
+    case length(Status) =:= 3 of
+        true -> ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status));
+        false ->
+            %% wait for mnesia to write in.
+            ct:sleep(42),
+            {atomic, Status1} = emqx_cluster_rpc:status(),
+            ct:pal("status: ~p", Status),
+            ct:pal("status1: ~p", Status1),
+            ?assertEqual(3, length(Status1)),
+            ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status))
+    end,
     ok.
 
 t_commit_fail_test(_Config) ->
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index b54d87f12..7d7771503 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -199,8 +199,8 @@ on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) ->
     on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State);
 on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery,
         #{pool_name := PoolName, base_path := BasePath} = State) ->
-    ?TRACE("QUERY", #{request => Request, connector => InstId, state => State},
-        "http connector received request"),
+    ?TRACE("QUERY", "http_connector_received",
+        #{request => Request, connector => InstId, state => State}),
     NRequest = update_path(BasePath, Request),
     case Result = ehttpc:request(case KeyOrNum of
                                      undefined -> PoolName;
diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl
index 8aa1f9319..97e963f18 100644
--- a/apps/emqx_connector/src/emqx_connector_ldap.erl
+++ b/apps/emqx_connector/src/emqx_connector_ldap.erl
@@ -87,16 +87,15 @@ on_stop(InstId, #{poolname := PoolName}) ->
 
 on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) ->
     Request = {Base, Filter, Attributes},
-    ?TRACE("QUERY", #{request => Request, connector => InstId, state => State},
-        "ldap connector received request"),
+    ?TRACE("QUERY", "ldap_connector_received",
+        #{request => Request, connector => InstId, state => State}),
     case Result = ecpool:pick_and_do(
                     PoolName,
                     {?MODULE, search, [Base, Filter, Attributes]},
                     no_handover) of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "ldap connector do request failed",
-                           request => Request, connector => InstId,
-                           reason => Reason}),
+            ?SLOG(error, #{msg => "ldap_connector_do_request_failed",
+                request => Request, connector => InstId, reason => Reason}),
             emqx_resource:query_failed(AfterQuery);
         _ ->
             emqx_resource:query_success(AfterQuery)
diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index d2594ab93..eacb3ec2d 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -137,13 +137,13 @@ on_query(InstId,
          AfterQuery,
          #{poolname := PoolName} = State) ->
     Request = {Action, Collection, Selector, Docs},
-    ?TRACE("QUERY", #{request => Request, connector => InstId, state => State},
-        "mongodb connector received request"),
+    ?TRACE("QUERY", "mongodb_connector_received",
+        #{request => Request, connector => InstId, state => State}),
     case ecpool:pick_and_do(PoolName,
                             {?MODULE, mongo_query, [Action, Collection, Selector, Docs]},
                             no_handover) of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "mongodb connector do query failed",
+            ?SLOG(error, #{msg => "mongodb_connector_do_query_failed",
                 request => Request, reason => Reason,
                 connector => InstId}),
             emqx_resource:query_failed(AfterQuery),
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index 6d620cc14..27d001806 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -118,7 +118,7 @@ on_message_received(Msg, HookPoint) ->
 %% ===================================================================
 on_start(InstId, Conf) ->
     InstanceId = binary_to_atom(InstId, utf8),
-    ?SLOG(info, #{msg => "starting mqtt connector",
+    ?SLOG(info, #{msg => "starting_mqtt_connector",
                   connector => InstanceId, config => Conf}),
     BasicConf = basic_config(Conf),
     BridgeConf = BasicConf#{
@@ -139,19 +139,18 @@ on_start(InstId, Conf) ->
     end.
 
 on_stop(_InstId, #{name := InstanceId}) ->
-    ?SLOG(info, #{msg => "stopping mqtt connector",
+    ?SLOG(info, #{msg => "stopping_mqtt_connector",
                   connector => InstanceId}),
     case ?MODULE:drop_bridge(InstanceId) of
         ok -> ok;
         {error, not_found} -> ok;
         {error, Reason} ->
-            ?SLOG(error, #{msg => "stop mqtt connector",
+            ?SLOG(error, #{msg => "stop_mqtt_connector",
                 connector => InstanceId, reason => Reason})
     end.
 
 on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) ->
-    ?TRACE("QUERY", #{message => Msg, connector => InstanceId},
-        "send msg to remote node"),
+    ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
     emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
     emqx_resource:query_success(AfterQuery).
 
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index def3904b4..ae8239936 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -85,14 +85,13 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
 on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) ->
     on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State);
 on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) ->
-    ?TRACE("QUERY", #{connector => InstId, sql => SQL, state => State},
-        "mysql connector received sql query"),
+    ?TRACE("QUERY", "mysql_connector_received", #{connector => InstId, sql => SQL, state => State}),
     case Result = ecpool:pick_and_do(
                     PoolName,
                     {mysql, query, [SQL, Params, Timeout]},
                     no_handover) of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "mysql connector do sql query failed",
+            ?SLOG(error, #{msg => "mysql_connector_do_sql_query_failed",
                 connector => InstId, sql => SQL, reason => Reason}),
             emqx_resource:query_failed(AfterQuery);
         _ ->
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index ac864a45d..9b6f559b4 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -83,12 +83,12 @@ on_stop(InstId, #{poolname := PoolName}) ->
 on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
     on_query(InstId, {sql, SQL, []}, AfterQuery, State);
 on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) ->
-    ?TRACE("QUERY", #{connector => InstId, sql => SQL, state => State},
-        "postgresql connector received sql query"),
+    ?TRACE("QUERY", "postgresql_connector_received",
+        #{connector => InstId, sql => SQL, state => State}),
     case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of
         {error, Reason} ->
             ?SLOG(error, #{
-                msg => "postgresql connector do sql query failed",
+                msg => "postgresql_connector_do_sql_query_failed",
                 connector => InstId, sql => SQL, reason => Reason}),
             emqx_resource:query_failed(AfterQuery);
         _ ->
diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl
index 61b716b8b..94f4eca3e 100644
--- a/apps/emqx_connector/src/emqx_connector_redis.erl
+++ b/apps/emqx_connector/src/emqx_connector_redis.erl
@@ -125,15 +125,15 @@ on_stop(InstId, #{poolname := PoolName}) ->
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
 on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) ->
-    ?TRACE("QUERY", #{connector => InstId, sql => Command, state => State},
-        "redis connector received cmd query"),
+    ?TRACE("QUERY", "redis_connector_received",
+        #{connector => InstId, sql => Command, state => State}),
     Result = case Type of
                  cluster -> eredis_cluster:q(PoolName, Command);
                  _ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover)
              end,
     case Result of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "redis connector do cmd query failed",
+            ?SLOG(error, #{msg => "redis_connector_do_cmd_query_failed",
                 connector => InstId, sql => Command, reason => Reason}),
             emqx_resource:query_failed(AfterCommand);
         _ ->
diff --git a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl
index 70aa68cf5..d02f62d70 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl
@@ -85,7 +85,7 @@ republish(Selected, #{flags := Flags, metadata := #{rule_id := RuleId}},
     Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected),
     QoS = replace_simple_var(QoSTks, Selected, 0),
     Retain = replace_simple_var(RetainTks, Selected, false),
-    ?TRACE("REPUBLISH", #{topic => Topic, payload => Payload}, "republish message"),
+    ?TRACE("RULE", "republish_message", #{topic => Topic, payload => Payload}),
     safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload);
 
 %% in case this is a "$events/" event
@@ -99,7 +99,7 @@ republish(Selected, #{metadata := #{rule_id := RuleId}},
     Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected),
     QoS = replace_simple_var(QoSTks, Selected, 0),
     Retain = replace_simple_var(RetainTks, Selected, false),
-    ?TRACE("REPUBLISH", #{topic => Topic, payload => Payload}, "republish"),
+    ?TRACE("RULE", "republish_message_with_flags", #{topic => Topic, payload => Payload}),
     safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload).
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
index c61296d87..60a7cbaad 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl
@@ -248,7 +248,7 @@ handle_output(OutId, Selected, Envs) ->
     end.
 
 do_handle_output(BridgeId, Selected, _Envs) when is_binary(BridgeId) ->
-    ?TRACE("SEND", #{bridge_id => BridgeId}, "output to bridge"),
+    ?TRACE("BRIDGE", "output_to_bridge", #{bridge_id => BridgeId}),
     emqx_bridge:send_message(BridgeId, Selected);
 do_handle_output(#{mod := Mod, func := Func, args := Args}, Selected, Envs) ->
     Mod:Func(Selected, Envs, Args).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
index 7a9da25a2..cd4d0ce6b 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
@@ -77,7 +77,7 @@ flatten([D1 | L]) when is_list(D1) ->
     D1 ++ flatten(L).
 
 echo_action(Data, Envs) ->
-    ?TRACE("TEST", #{data => Data, envs => Envs}, "testing_rule_sql_ok"),
+    ?TRACE("RULE", "testing_rule_sql_ok", #{data => Data, envs => Envs}),
     Data.
 
 fill_default_values(Event, Context) ->

From 8b5b3a448a4c894e2a7d245bcab45c85d24191d4 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 29 Dec 2021 01:15:47 +0800
Subject: [PATCH 3/7] fix(test):  trace_handler ct fail

---
 apps/emqx/etc/emqx.conf.rendered              | 1671 +++++++++++++++++
 apps/emqx/src/emqx_trace/emqx_trace.erl       |    1 -
 apps/emqx/test/emqx_trace_handler_SUITE.erl   |   84 +-
 .../src/emqx_mgmt_api_trace.erl               |    4 +-
 .../etc/emqx_modules.conf.rendered            |   50 +
 5 files changed, 1768 insertions(+), 42 deletions(-)
 create mode 100644 apps/emqx/etc/emqx.conf.rendered
 create mode 100644 apps/emqx_modules/etc/emqx_modules.conf.rendered

diff --git a/apps/emqx/etc/emqx.conf.rendered b/apps/emqx/etc/emqx.conf.rendered
new file mode 100644
index 000000000..afa640621
--- /dev/null
+++ b/apps/emqx/etc/emqx.conf.rendered
@@ -0,0 +1,1671 @@
+##==================================================================
+## Listeners
+##==================================================================
+## MQTT/TCP - TCP Listeners for MQTT Protocol
+## syntax: listeners.tcp.<name>
+## example: listeners.tcp.my_tcp_listener
+listeners.tcp.default {
+  ## The IP address and port that the listener will bind.
+  ##
+  ## @doc listeners.tcp.<name>.bind
+  ## ValueType: IPAddress | Port | IPAddrPort
+  ## Required: true
+  ## Examples: 1883, 127.0.0.1:1883, ::1:1883
+  bind = "0.0.0.0:1883"
+
+  ## The configuration zone this listener is using.
+  ## If not set, the global configs are used for this listener.
+  ##
+  ## See `zones.<name>` for more details.
+  ##
+  ## @doc listeners.tcp.<name>.zone
+  ## ValueType: String
+  ## Required: false
+  #zone = default
+
+  ## The size of the acceptor pool for this listener.
+  ##
+  ## @doc listeners.tcp.<name>.acceptors
+  ## ValueType: Number
+  ## Default: 16
+  acceptors = 16
+
+  ## Maximum number of concurrent connections.
+  ##
+  ## @doc listeners.tcp.<name>.max_connections
+  ## ValueType: Number | infinity
+  ## Default: infinity
+  max_connections = 1024000
+
+  ## The access control rules for this listener.
+  ##
+  ## See: https://github.com/emqtt/esockd#allowdeny
+  ##
+  ## @doc listeners.tcp.<name>.access_rules
+  ## ValueType: Array<AccessRules>
+  ## Default: []
+  ## Examples:
+  ##   access_rules: [
+  ##     "deny 192.168.0.0/24",
+  ##     "all all"
+  ##   ]
+  access_rules = [
+    "allow all"
+  ]
+
+  ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed
+  ## behind HAProxy or Nginx.
+  ##
+  ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/
+  ##
+  ## @doc listeners.tcp.<name>.proxy_protocol
+  ## ValueType: Boolean
+  ## Default: false
+  proxy_protocol = false
+
+  ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection
+  ## if no proxy protocol packet received within the timeout.
+  ##
+  ## @doc listeners.tcp.<name>.proxy_protocol_timeout
+  ## ValueType: Duration
+  ## Default: 3s
+  proxy_protocol_timeout = 3s
+
+  ## When publishing or subscribing, prefix all topics with a mountpoint string.
+  ## The prefixed string will be removed from the topic name when the message
+  ## is delivered to the subscriber. The mountpoint is a way that users can use
+  ## to implement isolation of message routing between different listeners.
+  ##
+  ## For example if a clientA subscribes to "t" with `listeners.tcp.<name>.mountpoint`
+  ## set to "some_tenant", then the client accually subscribes to the topic
+  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
+  ## with the clientA) send a message to topic "t", the message is accually route
+  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
+  ## message, with topic name "t".
+  ##
+  ## Set to "" to disable the feature.
+  ##
+  ## Variables in mountpoint string:
+  ##  - ${clientid}: clientid
+  ##  - ${username}: username
+  ##
+  ## @doc listeners.tcp.<name>.mountpoint
+  ## ValueType: String
+  ## Default: ""
+  mountpoint = ""
+
+  ## TCP options
+  ## See ${example_common_tcp_options} for more information
+  tcp.backlog = 1024
+  tcp.buffer = 4KB
+}
+
+## MQTT/SSL - SSL Listeners for MQTT Protocol
+## syntax: listeners.ssl.<name>
+## example: listeners.ssl.my_ssl_listener
+listeners.ssl.default {
+  ## The IP address and port that the listener will bind.
+  ##
+  ## @doc listeners.ssl.<name>.bind
+  ## ValueType: IPAddress | Port | IPAddrPort
+  ## Required: true
+  ## Examples: 8883, 127.0.0.1:8883, ::1:8883
+  bind = "0.0.0.0:8883"
+
+  ## The configuration zone this listener is using.
+  ## If not set, the global configs are used for this listener.
+  ##
+  ## See `zones.<name>` for more details.
+  ##
+  ## @doc listeners.ssl.<name>.zone
+  ## ValueType: String
+  ## Required: false
+  #zone = default
+
+  ## The size of the acceptor pool for this listener.
+  ##
+  ## @doc listeners.ssl.<name>.acceptors
+  ## ValueType: Number
+  ## Default: 16
+  acceptors = 16
+
+  ## Maximum number of concurrent connections.
+  ##
+  ## @doc listeners.ssl.<name>.max_connections
+  ## ValueType: Number | infinity
+  ## Default: infinity
+  max_connections = 512000
+
+  ## The access control rules for this listener.
+  ##
+  ## See: https://github.com/emqtt/esockd#allowdeny
+  ##
+  ## @doc listeners.ssl.<name>.access_rules
+  ## ValueType: Array<AccessRules>
+  ## Default: []
+  ## Examples:
+  ##   access_rules: [
+  ##     "deny 192.168.0.0/24",
+  ##     "all all"
+  ##   ]
+  access_rules = [
+    "allow all"
+  ]
+
+  ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed
+  ## behind HAProxy or Nginx.
+  ##
+  ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/
+  ##
+  ## @doc listeners.ssl.<name>.proxy_protocol
+  ## ValueType: Boolean
+  ## Default: true
+  proxy_protocol = false
+
+  ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection
+  ## if no proxy protocol packet received within the timeout.
+  ##
+  ## @doc listeners.ssl.<name>.proxy_protocol_timeout
+  ## ValueType: Duration
+  ## Default: 3s
+  proxy_protocol_timeout = 3s
+
+  ## When publishing or subscribing, prefix all topics with a mountpoint string.
+  ## The prefixed string will be removed from the topic name when the message
+  ## is delivered to the subscriber. The mountpoint is a way that users can use
+  ## to implement isolation of message routing between different listeners.
+  ##
+  ## For example if a clientA subscribes to "t" with `listeners.ssl.<name>.mountpoint`
+  ## set to "some_tenant", then the client accually subscribes to the topic
+  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
+  ## with the clientA) send a message to topic "t", the message is accually route
+  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
+  ## message, with topic name "t".
+  ##
+  ## Set to "" to disable the feature.
+  ##
+  ## Variables in mountpoint string:
+  ##  - ${clientid}: clientid
+  ##  - ${username}: username
+  ##
+  ## @doc listeners.ssl.<name>.mountpoint
+  ## ValueType: String
+  ## Default: ""
+  mountpoint = ""
+
+  ## SSL options
+  ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
+  ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
+  ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem"
+
+  # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
+  # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256"
+  # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA"
+  # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA"
+  # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config
+  # ssl.ciphers = ""
+
+  ## TCP options
+  ## See ${example_common_tcp_options} for more information
+  tcp.backlog = 1024
+  tcp.buffer = 4KB
+}
+
+## MQTT/QUIC - QUIC Listeners for MQTT Protocol
+## syntax: listeners.quic.<name>
+## example: listeners.quic.my_quic_listener
+listeners.quic.default {
+  ## The IP address and port that the listener will bind.
+  ##
+  ## @doc listeners.quic.<name>.bind
+  ## ValueType: IPAddress | Port | IPAddrPort
+  ## Required: true
+  ## Examples: 14567, 127.0.0.1:14567, ::1:14567
+  bind = "0.0.0.0:14567"
+
+  ## The configuration zone this listener is using.
+  ## If not set, the global configs are used for this listener.
+  ##
+  ## See `zones.<name>` for more details.
+  ## NOTE: This is a cluster-wide configuration.
+  ## It requires all nodes to be stopped before changing it.
+  ##
+  ## @doc listeners.quic.<name>.zone
+  ## ValueType: String
+  ## Required: false
+  #zone = default
+
+  ## The size of the acceptor pool for this listener.
+  ##
+  ## @doc listeners.quic.<name>.acceptors
+  ## ValueType: Number
+  ## Default: 16
+  acceptors = 16
+
+  ## Maximum number of concurrent connections.
+  ##
+  ## @doc listeners.quic.<name>.max_connections
+  ## ValueType: Number | infinity
+  ## Default: infinity
+  max_connections = 1024000
+
+  ## Path to the file containing the user's private PEM-encoded key.
+  ##
+  ## @doc listeners.quic.<name>.keyfile
+  ## ValueType: String
+  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
+  keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
+
+  ## Path to a file containing the user certificate.
+  ##
+  ## @doc listeners.quic.<name>.certfile
+  ## ValueType: String
+  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
+  certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
+
+  ## When publishing or subscribing, prefix all topics with a mountpoint string.
+  ## The prefixed string will be removed from the topic name when the message
+  ## is delivered to the subscriber. The mountpoint is a way that users can use
+  ## to implement isolation of message routing between different listeners.
+  ##
+  ## For example if a clientA subscribes to "t" with `listeners.quic.<name>.mountpoint`
+  ## set to "some_tenant", then the client accually subscribes to the topic
+  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
+  ## with the clientA) send a message to topic "t", the message is accually route
+  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
+  ## message, with topic name "t".
+  ##
+  ## Set to "" to disable the feature.
+  ##
+  ## Variables in mountpoint string:
+  ##  - ${clientid}: clientid
+  ##  - ${username}: username
+  ##
+  ## @doc listeners.quic.<name>.mountpoint
+  ## ValueType: String
+  ## Default: ""
+  mountpoint = ""
+}
+
+## MQTT/WS - Websocket Listeners for MQTT Protocol
+## syntax: listeners.ws.<name>
+## example: listeners.ws.my_ws_listener
+listeners.ws.default {
+  ## The IP address and port that the listener will bind.
+  ##
+  ## @doc listeners.ws.<name>.bind
+  ## ValueType: IPAddress | Port | IPAddrPort
+  ## Required: true
+  ## Examples: 8083, 127.0.0.1:8083, ::1:8083
+  bind = "0.0.0.0:8083"
+
+  ## The configuration zone this listener is using.
+  ## If not set, the global configs are used for this listener.
+  ##
+  ## See `zones.<name>` for more details.
+  ##
+  ## @doc listeners.ws.<name>.zone
+  ## ValueType: String
+  ## Required: false
+  #zone = default
+
+  ## The size of the acceptor pool for this listener.
+  ##
+  ## @doc listeners.ws.<name>.acceptors
+  ## ValueType: Number
+  ## Default: 16
+  acceptors = 16
+
+  ## Maximum number of concurrent connections.
+  ##
+  ## @doc listeners.ws.<name>.max_connections
+  ## ValueType: Number | infinity
+  ## Default: infinity
+  max_connections = 1024000
+
+  ## The access control rules for this listener.
+  ##
+  ## See: https://github.com/emqtt/esockd#allowdeny
+  ##
+  ## @doc listeners.ws.<name>.access_rules
+  ## ValueType: Array<AccessRules>
+  ## Default: []
+  ## Examples:
+  ##   access_rules: [
+  ##     "deny 192.168.0.0/24",
+  ##     "all all"
+  ##   ]
+  access_rules = [
+    "allow all"
+  ]
+
+  ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed
+  ## behind HAProxy or Nginx.
+  ##
+  ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/
+  ##
+  ## @doc listeners.ws.<name>.proxy_protocol
+  ## ValueType: Boolean
+  ## Default: true
+  proxy_protocol = false
+
+  ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection
+  ## if no proxy protocol packet received within the timeout.
+  ##
+  ## @doc listeners.ws.<name>.proxy_protocol_timeout
+  ## ValueType: Duration
+  ## Default: 3s
+  proxy_protocol_timeout = 3s
+
+  ## When publishing or subscribing, prefix all topics with a mountpoint string.
+  ## The prefixed string will be removed from the topic name when the message
+  ## is delivered to the subscriber. The mountpoint is a way that users can use
+  ## to implement isolation of message routing between different listeners.
+  ##
+  ## For example if a clientA subscribes to "t" with `listeners.ws.<name>.mountpoint`
+  ## set to "some_tenant", then the client accually subscribes to the topic
+  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
+  ## with the clientA) send a message to topic "t", the message is accually route
+  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
+  ## message, with topic name "t".
+  ##
+  ## Set to "" to disable the feature.
+  ##
+  ## Variables in mountpoint string:
+  ##  - ${clientid}: clientid
+  ##  - ${username}: username
+  ##
+  ## @doc listeners.ws.<name>.mountpoint
+  ## ValueType: String
+  ## Default: ""
+  mountpoint = ""
+
+  ## TCP options
+  ## See ${example_common_tcp_options} for more information
+  tcp.backlog = 1024
+  tcp.buffer = 4KB
+
+  ## Websocket options
+  ## See ${example_common_websocket_options} for more information
+  websocket.idle_timeout = 86400s
+}
+
+## MQTT/WSS - WebSocket Secure Listeners for MQTT Protocol
+## syntax: listeners.wss.<name>
+## example: listeners.wss.my_wss_listener
+listeners.wss.default {
+  ## The IP address and port that the listener will bind.
+  ##
+  ## @doc listeners.wss.<name>.bind
+  ## ValueType: IPAddress | Port | IPAddrPort
+  ## Required: true
+  ## Examples: 8084, 127.0.0.1:8084, ::1:8084
+  bind = "0.0.0.0:8084"
+
+  ## The configuration zone this listener is using.
+  ## If not set, the global configs are used for this listener.
+  ##
+  ## See `zones.<name>` for more details.
+  ##
+  ## @doc listeners.wss.<name>.zone
+  ## ValueType: String
+  ## Required: false
+  #zone = default
+
+  ## The size of the acceptor pool for this listener.
+  ##
+  ## @doc listeners.wss.<name>.acceptors
+  ## ValueType: Number
+  ## Default: 16
+  acceptors = 16
+
+  ## Maximum number of concurrent connections.
+  ##
+  ## @doc listeners.wss.<name>.max_connections
+  ## ValueType: Number | infinity
+  ## Default: infinity
+  max_connections = 512000
+
+  ## The access control rules for this listener.
+  ##
+  ## See: https://github.com/emqtt/esockd#allowdeny
+  ##
+  ## @doc listeners.wss.<name>.access_rules
+  ## ValueType: Array<AccessRules>
+  ## Default: []
+  ## Examples:
+  ##   access_rules: [
+  ##     "deny 192.168.0.0/24",
+  ##     "all all"
+  ##   ]
+  access_rules = [
+    "allow all"
+  ]
+
+  ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed
+  ## behind HAProxy or Nginx.
+  ##
+  ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/
+  ##
+  ## @doc listeners.wss.<name>.proxy_protocol
+  ## ValueType: Boolean
+  ## Default: true
+  proxy_protocol = false
+
+  ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection
+  ## if no proxy protocol packet received within the timeout.
+  ##
+  ## @doc listeners.wss.<name>.proxy_protocol_timeout
+  ## ValueType: Duration
+  ## Default: 3s
+  proxy_protocol_timeout = 3s
+
+  ## When publishing or subscribing, prefix all topics with a mountpoint string.
+  ## The prefixed string will be removed from the topic name when the message
+  ## is delivered to the subscriber. The mountpoint is a way that users can use
+  ## to implement isolation of message routing between different listeners.
+  ##
+  ## For example if a clientA subscribes to "t" with `listeners.wss.<name>.mountpoint`
+  ## set to "some_tenant", then the client accually subscribes to the topic
+  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
+  ## with the clientA) send a message to topic "t", the message is accually route
+  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
+  ## message, with topic name "t".
+  ##
+  ## Set to "" to disable the feature.
+  ##
+  ## Variables in mountpoint string:
+  ##  - ${clientid}: clientid
+  ##  - ${username}: username
+  ##
+  ## @doc listeners.wss.<name>.mountpoint
+  ## ValueType: String
+  ## Default: ""
+  mountpoint = ""
+
+  ## SSL options
+  ## See ${example_common_ssl_options} for more information
+  ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
+  ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
+  ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem"
+
+  ## TCP options
+  ## See ${example_common_tcp_options} for more information
+  tcp.backlog = 1024
+  tcp.buffer = 4KB
+
+  ## Websocket options
+  ## See ${example_common_websocket_options} for more information
+  websocket.idle_timeout = 86400s
+
+}
+
+## Enable per connection statistics.
+##
+## @doc stats.enable
+## ValueType: Boolean
+## Default: true
+stats.enable = true
+
+authorization {
+  ## Behaviour after not matching a rule.
+  ##
+  ## @doc authorization.no_match
+  ## ValueType: allow | deny
+  ## Default: allow
+  no_match: allow
+
+  ## The action when authorization check reject current operation
+  ##
+  ## @doc authorization.deny_action
+  ## ValueType: ignore | disconnect
+  ## Default: ignore
+  deny_action: ignore
+
+  ## Whether to enable Authorization cache.
+  ##
+  ## If enabled, Authorization roles for each client will be cached in the memory
+  ##
+  ## @doc authorization.cache.enable
+  ## ValueType: Boolean
+  ## Default: true
+  cache.enable: true
+
+  ## The maximum count of Authorization entries can be cached for a client.
+  ##
+  ## @doc authorization.cache.max_size
+  ## ValueType: Integer
+  ## Range: [0, 1048576]
+  ## Default: 32
+  cache.max_size: 32
+
+  ## The time after which an Authorization cache entry will be deleted
+  ##
+  ## @doc authorization.cache.ttl
+  ## ValueType: Duration
+  ## Default: 1m
+  cache.ttl: 1m
+}
+
+mqtt {
+  ## How long time the MQTT connection will be disconnected if the
+  ## TCP connection is established but MQTT CONNECT has not been
+  ## received.
+  ##
+  ## @doc mqtt.idle_timeout
+  ## ValueType: Duration
+  ## Default: 15s
+  idle_timeout = 15s
+
+  ## Maximum MQTT packet size allowed.
+  ##
+  ## @doc mqtt.max_packet_size
+  ## ValueType: Bytes
+  ## Default: 1MB
+  max_packet_size = 1MB
+
+  ## Maximum length of MQTT clientId allowed.
+  ##
+  ## @doc mqtt.max_clientid_len
+  ## ValueType: Integer
+  ## Range: [23, 65535]
+  ## Default: 65535
+  max_clientid_len = 65535
+
+  ## Maximum topic levels allowed.
+  ##
+  ## @doc mqtt.max_topic_levels
+  ## ValueType: Integer
+  ## Range: [1, 65535]
+  ## Default: 128
+  ## Depth so big may lead to subscribing performance issues
+  max_topic_levels = 128
+
+  ## Maximum QoS allowed.
+  ##
+  ## @doc mqtt.max_qos_allowed
+  ## ValueType: 0 | 1 | 2
+  ## Default: 2
+  max_qos_allowed = 2
+
+  ## Maximum Topic Alias, 0 means no topic alias supported.
+  ##
+  ## @doc mqtt.max_topic_alias
+  ## ValueType: Integer
+  ## Range: [0, 65535]
+  ## Default: 65535
+  max_topic_alias = 65535
+
+  ## Whether the Server supports MQTT retained messages.
+  ##
+  ## @doc mqtt.retain_available
+  ## ValueType: Boolean
+  ## Default: true
+  retain_available = true
+
+  ## Whether the Server supports MQTT Wildcard Subscriptions
+  ##
+  ## @doc mqtt.wildcard_subscription
+  ## ValueType: Boolean
+  ## Default: true
+  wildcard_subscription = true
+
+  ## Whether the Server supports MQTT Shared Subscriptions.
+  ##
+  ## @doc mqtt.shared_subscription
+  ## ValueType: Boolean
+  ## Default: true
+  shared_subscription = true
+
+  ## Whether to ignore loop delivery of messages.(for mqtt v3.1.1)
+  ##
+  ## @doc mqtt.ignore_loop_deliver
+  ## ValueType: Boolean
+  ## Default: false
+  ignore_loop_deliver = false
+
+  ## Whether to parse the MQTT frame in strict mode
+  ##
+  ## @doc mqtt.strict_mode
+  ## ValueType: Boolean
+  ## Default: false
+  strict_mode = false
+
+  ## Specify the response information returned to the client
+  ##
+  ## This feature is disabled if is set to ""
+  ##
+  ## @doc mqtt.response_information
+  ## ValueType: String
+  ## Default: ""
+  response_information = ""
+
+  ## Server Keep Alive of MQTT 5.0
+  ##
+  ## @doc mqtt.server_keepalive
+  ## ValueType: Number | disabled
+  ## Default: disabled
+  server_keepalive = disabled
+
+  ## The backoff for MQTT keepalive timeout. The broker will kick a connection out
+  ## until 'Keepalive * backoff * 2' timeout.
+  ##
+  ## @doc mqtt.keepalive_backoff
+  ## ValueType: Float
+  ## Range: (0.5, 1]
+  ## Default: 0.75
+  keepalive_backoff = 0.75
+
+  ## Maximum number of subscriptions allowed.
+  ##
+  ## @doc mqtt.max_subscriptions
+  ## ValueType: Integer | infinity
+  ## Range: [1, infinity)
+  ## Default: infinity
+  max_subscriptions = infinity
+
+  ## Force to upgrade QoS according to subscription.
+  ##
+  ## @doc mqtt.upgrade_qos
+  ## ValueType: Boolean
+  ## Default: false
+  upgrade_qos = false
+
+  ## Maximum size of the Inflight Window storing QoS1/2 messages delivered but unacked.
+  ##
+  ## @doc mqtt.max_inflight
+  ## ValueType: Integer
+  ## Range: [1, 65535]
+  ## Default: 32
+  max_inflight = 32
+
+  ## Retry interval for QoS1/2 message delivering.
+  ##
+  ## @doc mqtt.retry_interval
+  ## ValueType: Duration
+  ## Default: 30s
+  retry_interval = 30s
+
+  ## Maximum QoS2 packets (Client -> Broker) awaiting PUBREL.
+  ##
+  ## @doc mqtt.max_awaiting_rel
+  ## ValueType: Integer | infinity
+  ## Range: [1, infinity)
+  ## Default: 100
+  max_awaiting_rel = 100
+
+  ## The QoS2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout.
+  ##
+  ## @doc mqtt.await_rel_timeout
+  ## ValueType: Duration
+  ## Default: 300s
+  await_rel_timeout = 300s
+
+  ## Default session expiry interval for MQTT V3.1.1 connections.
+  ##
+  ## @doc mqtt.session_expiry_interval
+  ## ValueType: Duration
+  ## Default: 2h
+  session_expiry_interval = 2h
+
+  ## Maximum queue length. Enqueued messages when persistent client disconnected,
+  ## or inflight window is full.
+  ##
+  ## @doc mqtt.max_mqueue_len
+  ## ValueType: Integer | infinity
+  ## Range: [0, infinity)
+  ## Default: 1000
+  max_mqueue_len = 1000
+
+  ## Topic priorities.
+  ##
+  ## There's no priority table by default, hence all messages
+  ## are treated equal.
+  ##
+  ## Priority number [1-255]
+  ##
+  ## NOTE: comma and equal signs are not allowed for priority topic names
+  ## NOTE: Messages for topics not in the priority table are treated as
+  ## either highest or lowest priority depending on the configured
+  ## value for mqtt.mqueue_default_priority
+  ##
+  ## @doc mqtt.mqueue_priorities
+  ## ValueType: Map | disabled
+  ## Examples:
+  ##  To configure "topic/1" > "topic/2":
+  ##    mqueue_priorities: {"topic/1": 10, "topic/2": 8}
+  ## Default: disabled
+  mqueue_priorities = disabled
+
+  ## Default to highest priority for topics not matching priority table
+  ##
+  ## @doc mqtt.mqueue_default_priority
+  ## ValueType: highest | lowest
+  ## Default: lowest
+  mqueue_default_priority = lowest
+
+  ## Whether to enqueue QoS0 messages.
+  ##
+  ## @doc mqtt.mqueue_store_qos0
+  ## ValueType: Boolean
+  ## Default: true
+  mqueue_store_qos0 = true
+
+  ## Whether use username replace client id
+  ##
+  ## @doc mqtt.use_username_as_clientid
+  ## ValueType: Boolean
+  ## Default: false
+  use_username_as_clientid = false
+
+  ## Use the CN, DN or CRT field from the client certificate as a username.
+  ## Only works for SSL connection.
+  ##
+  ## @doc mqtt.peer_cert_as_username
+  ## ValueType: cn | dn | crt | disabled
+  ## Default: disabled
+  peer_cert_as_username = disabled
+
+  ## Use the CN, DN or CRT field from the client certificate as a clientid.
+  ## Only works for SSL connection.
+  ##
+  ## @doc mqtt.peer_cert_as_clientid
+  ## ValueType: cn | dn | crt | disabled
+  ## Default: disabled
+  peer_cert_as_clientid = disabled
+}
+
+flapping_detect {
+  ## Enable Flapping Detection.
+  ##
+  ## This config controls the allowed maximum number of CONNECT received
+  ## from the same clientid in a time frame defined by `window_time`.
+  ## After the limit is reached, successive CONNECT requests are forbidden
+  ## (banned) until the end of the time period defined by `ban_time`.
+  ##
+  ## @doc flapping_detect.enable
+  ## ValueType: Boolean
+  ## Default: true
+  enable = false
+
+  ## The max disconnect allowed of a MQTT Client in `window_time`
+  ##
+  ## @doc flapping_detect.max_count
+  ## ValueType: Integer
+  ## Default: 15
+  max_count = 15
+
+  ## The time window for flapping detect
+  ##
+  ## @doc flapping_detect.window_time
+  ## ValueType: Duration
+  ## Default: 1m
+  window_time = 1m
+
+  ## How long the clientid will be banned
+  ##
+  ## @doc flapping_detect.ban_time
+  ## ValueType: Duration
+  ## Default: 5m
+  ban_time = 5m
+
+}
+
+force_shutdown {
+  ## Enable force_shutdown
+  ##
+  ## @doc force_shutdown.enable
+  ## ValueType: Boolean
+  ## Default: true
+  enable = true
+
+  ## Max message queue length
+  ## @doc force_shutdown.max_message_queue_len
+  ## ValueType: Integer
+  ## Range: (0, infinity)
+  ## Default: 1000
+  max_message_queue_len = 1000
+
+  ## Total heap size
+  ##
+  ## @doc force_shutdown.max_heap_size
+  ## ValueType: Size
+  ## Default: 32MB
+  max_heap_size = 32MB
+}
+
+overload_protection {
+ ## React on system overload or not
+ ## @doc overload_protection.enable
+ ## ValueType: Boolean
+ ## Default: false
+ enable = false
+
+ ## Backoff delay in ms
+ ## @doc overload_protection.backoff_delay
+ ## ValueType: Integer
+ ## Range: (0, infinity)
+ ## Default: 1
+ backoff_delay = 1
+
+ ## Backoff GC enabled
+ ## @doc overload_protection.backoff_gc
+ ## ValueType: Boolean
+ ## Default: false
+ backoff_gc = false
+
+ ## Backoff hibernation enabled
+ ## @doc overload_protection.backoff_hibernation
+ ## ValueType: Boolean
+ ## Default: true
+ backoff_hibernation = true
+
+ ## Backoff hibernation enabled
+ ## @doc overload_protection.backoff_hibernation
+ ## ValueType: Boolean
+ ## Default: true
+ backoff_new_conn = true
+}
+
+force_gc {
+  ## Force the MQTT connection process GC after this number of
+  ## messages or bytes passed through.
+  ##
+  ## @doc force_gc.enable
+  ## ValueType: Boolean
+  ## Default: true
+  enable = true
+
+  ## GC the process after how many messages received
+  ## @doc force_gc.max_message_queue_len
+  ## ValueType: Integer
+  ## Range: (0, infinity)
+  ## Default: 16000
+  count = 16000
+
+  ## GC the process after how much bytes passed through
+  ##
+  ## @doc force_gc.bytes
+  ## ValueType: Size
+  ## Default: 16MB
+  bytes = 16MB
+}
+
+conn_congestion {
+  ## Whether to alarm the congested connections.
+  ##
+  ## Sometimes the mqtt connection (usually an MQTT subscriber) may
+  ## get "congested" because there're too many packets to sent.
+  ## The socket trys to buffer the packets until the buffer is
+  ## full. If more packets comes after that, the packets will be
+  ## "pending" in a queue and we consider the connection is
+  ## "congested".
+  ##
+  ## Enable this to send an alarm when there's any bytes pending in
+  ## the queue. You could set the `sndbuf` to a larger value if the
+  ## alarm is triggered too often.
+  ##
+  ## The name of the alarm is of format "conn_congestion/<ClientID>/<Username>".
+  ## Where the <ClientID> is the client-id of the congested MQTT connection.
+  ## And the <Username> is the username or "unknown_user" of not provided by the client.
+  ##
+  ## @doc conn_congestion.enable_alarm
+  ## ValueType: Boolean
+  ## Default: true
+  enable_alarm = true
+
+  ## Won't clear the congested alarm in how long time.
+  ## The alarm is cleared only when there're no pending bytes in
+  ## the queue, and also it has been `min_alarm_sustain_duration`
+  ## time since the last time we considered the connection is "congested".
+  ##
+  ## This is to avoid clearing and sending the alarm again too often.
+  ##
+  ## @doc conn_congestion.min_alarm_sustain_duration
+  ## ValueType: Duration
+  ## Default: 1m
+  min_alarm_sustain_duration = 1m
+}
+
+rate_limit {
+  ## Maximum connections per second.
+  ##
+  ## @doc zones.<name>.max_conn_rate
+  ## ValueType: Number | infinity
+  ## Default: 1000
+  ## Examples:
+  ##   max_conn_rate: 1000
+  max_conn_rate = 1000
+
+  ## Message limit for the a external MQTT connection.
+  ##
+  ## @doc rate_limit.conn_messages_in
+  ## ValueType: String | infinity
+  ## Default: infinity
+  ## Examples: 100 messages per 10 seconds.
+  ##   conn_messages_in: "100,10s"
+  conn_messages_in = "100,10s"
+
+  ## Limit the rate of receiving packets for a MQTT connection.
+  ## The rate is counted by bytes of packets per second.
+  ##
+  ## The connection won't accept more messages if the messages come
+  ## faster than the limit.
+  ##
+  ## @doc rate_limit.conn_bytes_in
+  ## ValueType: String | infinity
+  ## Default: infinity
+  ## Examples: 100KB incoming per 10 seconds.
+  ##   conn_bytes_in: "100KB,10s"
+  ##
+  conn_bytes_in = "100KB,10s"
+}
+
+quota {
+  ## Messages quota for the each of external MQTT connection.
+  ## This value consumed by the number of recipient on a message.
+  ##
+  ## @doc quota.conn_messages_routing
+  ## ValueType: String | infinity
+  ## Default: infinity
+  ## Examples: 100 messaegs per 1s:
+  ##   quota.conn_messages_routing: "100,1s"
+  conn_messages_routing = "100,1s"
+
+  ## Messages quota for the all of external MQTT connections.
+  ## This value consumed by the number of recipient on a message.
+  ##
+  ## @doc quota.overall_messages_routing
+  ## ValueType: String | infinity
+  ## Default: infinity
+  ## Examples: 200000 messages per 1s:
+  ##    quota.overall_messages_routing: "200000,1s"
+  ##
+  overall_messages_routing = "200000,1s"
+}
+
+##==================================================================
+## Zones
+##==================================================================
+## A zone contains a set of configurations for listeners.
+##
+## A zone can be used by a listener via `listener.<type>.<name>.zone`.
+##
+## The configs defined in zones will override the global configs with the same key.
+##
+## For example given the following config:
+##
+## ```
+## a {
+##     b: 1, c: 1
+## }
+##
+## zone.my_zone {
+##   a {
+##     b:2
+##   }
+## }
+## ```
+##
+## The global config "a" is overridden by the configs "a" inside the zone "my_zone".
+## If there is a listener uses the zone "my_zone", the value of config "a" will be:
+## `{b:2, c: 1}`.
+## Note that although the default value of `a.c` is `0`, the global value is used.
+## i.e. configs in the zone have no default values. To overridde `a.c` we must configure
+## it explicitly in the zone.
+##
+## All the global configs that can be overridden in zones are:
+##   - `stats.*`
+##   - `mqtt.*`
+##   - `authorization.*`
+##   - `flapping_detect.*`
+##   - `force_shutdown.*`
+##   - `conn_congestion.*`
+##   - `rate_limit.*`
+##   - `quota.*`
+##   - `force_gc.*`
+##
+## syntax: zones.<zone-name>
+## example: zones.my_zone
+zones.default {
+
+}
+
+##==================================================================
+## Broker
+##==================================================================
+broker {
+  ## System interval of publishing $SYS messages.
+  ##
+  ## @doc broker.sys_msg_interval
+  ## ValueType: Duration | disabled
+  ## Default: 1m
+  sys_msg_interval = 1m
+
+  ## System heartbeat interval of publishing following heart beat message:
+  ##  - "$SYS/brokers/<node>/uptime"
+  ##  - "$SYS/brokers/<node>/datetime"
+  ##
+  ## @doc broker.sys_heartbeat_interval
+  ## ValueType: Duration
+  ## Default: 30s | disabled
+  sys_heartbeat_interval = 30s
+
+  ## Session locking strategy in a cluster.
+  ##
+  ## @doc broker.session_locking_strategy
+  ## ValueType: local | one | quorum | all
+  ##   - local: only lock the session locally on the current node
+  ##   - one: select only one remove node to lock the session
+  ##   - quorum: select some nodes to lock the session
+  ##   - all: lock the session on all of the nodes in the cluster
+  ## Default: quorum
+  session_locking_strategy = quorum
+
+  ## Dispatch strategy for shared subscription
+  ##
+  ## @doc broker.shared_subscription_strategy
+  ## ValueType: random | round_robin | sticky | hash
+  ##   - random: dispatch the message to a random selected subscriber
+  ##   - round_robin: select the subscribers in a round-robin manner
+  ##   - sticky: always use the last selected subscriber to dispatch,
+  ##     until the susbcriber disconnected.
+  ##   - hash: select the subscribers by the hash of clientIds
+  ## Default: round_robin
+  shared_subscription_strategy = round_robin
+
+  ## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages
+  ## This should allow messages to be dispatched to a different subscriber in
+  ## the group in case the picked (based on shared_subscription_strategy) one # is offline
+  ##
+  ## @doc broker.shared_dispatch_ack_enabled
+  ## ValueType: Boolean
+  ## Default: false
+  shared_dispatch_ack_enabled = false
+
+  ## Enable batch clean for deleted routes.
+  ##
+  ## @doc broker.route_batch_clean
+  ## ValueType: Boolean
+  ## Default: true
+  route_batch_clean = true
+
+  ## Performance toggle for subscribe/unsubscribe wildcard topic.
+  ## Change this toggle only when there are many wildcard topics.
+  ##
+  ## NOTE: when changing from/to 'global' lock, it requires all
+  ## nodes in the cluster to be stopped before the change.
+  ##
+  ## @doc broker.perf.route_lock_type
+  ## ValueType: key | tab | global
+  ##  - key:   mnesia translational updates with per-key locks. recommended for single node setup.
+  ##  - tab:   mnesia translational updates with table lock. recommended for multi-nodes setup.
+  ##  - global: global lock protected updates. recommended for larger cluster.
+  ## Default: key
+  perf.route_lock_type = key
+
+  ## Enable trie path compaction.
+  ## Enabling it significantly improves wildcard topic subscribe
+  ## rate, if wildcard topics have unique prefixes like:
+  ## 'sensor//+/', where ID is unique per subscriber.
+  ##
+  ## Topic match performance (when publishing) may degrade if messages
+  ## are mostly published to topics with large number of levels.
+  ##
+  ## NOTE: This is a cluster-wide configuration.
+  ## It requires all nodes to be stopped before changing it.
+  ##
+  ## @doc broker.perf.trie_compaction
+  ## ValueType: Boolean
+  ## Default: true
+  perf.trie_compaction = true
+}
+
+##==================================================================
+## System Monitor
+##==================================================================
+sysmon {
+  ## The time interval for the periodic process limit check
+  ##
+  ## @doc sysmon.vm.process_check_interval
+  ## ValueType: Duration
+  ## Default: 30s
+  vm.process_check_interval = 30s
+
+  ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is set.
+  ##
+  ## @doc sysmon.vm.process_high_watermark
+  ## ValueType: Percentage
+  ## Default: 80%
+  vm.process_high_watermark = 80%
+
+  ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is clear.
+  ##
+  ## @doc sysmon.vm.process_low_watermark
+  ## ValueType: Percentage
+  ## Default: 60%
+  vm.process_low_watermark = 60%
+
+  ## Enable Long GC monitoring.
+  ## Notice: don't enable the monitor in production for:
+  ## https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421
+  ##
+  ## @doc sysmon.vm.long_gc
+  ## ValueType: Duration | disabled
+  ## Default: disabled
+  vm.long_gc = disabled
+
+  ## Enable Long Schedule(ms) monitoring.
+  ##
+  ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2
+  ##
+  ## @doc sysmon.vm.long_schedule
+  ## ValueType: Duration | disabled
+  ## Default: disabled
+  vm.long_schedule = 240ms
+
+  ## Enable Large Heap monitoring.
+  ##
+  ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2
+  ##
+  ## @doc sysmon.vm.large_heap
+  ## ValueType: Size | disabled
+  ## Default: 32MB
+  vm.large_heap = 32MB
+
+  ## Enable Busy Port monitoring.
+  ##
+  ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2
+  ##
+  ## @doc sysmon.vm.busy_port
+  ## ValueType: Boolean
+  ## Default: true
+  vm.busy_port = true
+
+  ## Enable Busy Dist Port monitoring.
+  ##
+  ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2
+  ##
+  ## @doc sysmon.vm.busy_dist_port
+  ## ValueType: Boolean
+  ## Default: true
+  vm.busy_dist_port = true
+
+  ## The time interval for the periodic cpu check
+  ##
+  ## @doc sysmon.os.cpu_check_interval
+  ## ValueType: Duration
+  ## Default: 60s
+  os.cpu_check_interval = 60s
+
+  ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is set.
+  ##
+  ## @doc sysmon.os.cpu_high_watermark
+  ## ValueType: Percentage
+  ## Default: 80%
+  os.cpu_high_watermark = 80%
+
+  ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is clear.
+  ##
+  ## @doc sysmon.os.cpu_low_watermark
+  ## ValueType: Percentage
+  ## Default: 60%
+  os.cpu_low_watermark = 60%
+
+  ## The time interval for the periodic memory check
+  ##
+  ## @doc sysmon.os.mem_check_interval
+  ## ValueType: Duration | disabled
+  ## Default: 60s
+  os.mem_check_interval = 60s
+
+  ## The threshold, as percentage of system memory, for how much system memory can be allocated before the corresponding alarm is set.
+  ##
+  ## @doc sysmon.os.sysmem_high_watermark
+  ## ValueType: Percentage
+  ## Default: 70%
+  os.sysmem_high_watermark = 70%
+
+  ## The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before the corresponding alarm is set.
+  ##
+  ## @doc sysmon.os.procmem_high_watermark
+  ## ValueType: Percentage
+  ## Default: 5%
+  os.procmem_high_watermark = 5%
+}
+
+##==================================================================
+## Alarm
+##==================================================================
+alarm {
+  ## Specifies the actions to take when an alarm is activated
+  ##
+  ## @doc alarm.actions
+  ## ValueType: Array<AlarmAction>
+  ## Default: [log, publish]
+  actions = [log, publish]
+
+  ## The maximum number of deactivated alarms
+  ##
+  ## @doc alarm.size_limit
+  ## ValueType: Integer
+  ## Default: 1000
+  size_limit = 1000
+
+  ## Validity Period of deactivated alarms
+  ##
+  ## @doc alarm.validity_period
+  ## ValueType: Duration
+  ## Default: 24h
+  validity_period = 24h
+}
+
+## Config references for listeners
+
+## Socket options for TCP connections
+## See: http://erlang.org/doc/man/inet.html
+example_common_tcp_options {
+  ## Specify the {active, N} option for this Socket.
+  ##
+  ## See: https://erlang.org/doc/man/inet.html#setopts-2
+  ##
+  ## @doc listeners.<name>.tcp.active_n
+  ## ValueType: Number
+  ## Default: 100
+  tcp.active_n = 100
+
+  ## TCP backlog defines the maximum length that the queue of
+  ## pending connections can grow to.
+  ##
+  ## @doc listeners.<name>.tcp.backlog
+  ## ValueType: Number
+  ## Range: [0, 1048576]
+  ## Default: 1024
+  tcp.backlog = 1024
+
+  ## The TCP send timeout for the connections.
+  ##
+  ## @doc listeners.<name>.tcp.send_timeout
+  ## ValueType: Duration
+  ## Default: 15s
+  tcp.send_timeout = 15s
+
+  ## Close the connection if send timeout.
+  ##
+  ## @doc listeners.<name>.tcp.send_timeout_close
+  ## ValueType: Boolean
+  ## Default: true
+  tcp.send_timeout_close = true
+
+  ## The TCP receive buffer(os kernel) for the connections.
+  ##
+  ## @doc listeners.<name>.tcp.recbuf
+  ## ValueType: Size
+  ## Default: notset
+  #tcp.recbuf: 2KB
+
+  ## The TCP send buffer(os kernel) for the connections.
+  ##
+  ## @doc listeners.<name>.tcp.sndbuf
+  ## ValueType: Size
+  ## Default: notset
+  #tcp.sndbuf: 4KB
+
+  ## The size of the user-level software buffer used by the driver.
+  ##
+  ## @doc listeners.<name>.tcp.buffer
+  ## ValueType: Size
+  ## Default: notset
+  #tcp.buffer: 4KB
+
+  ## The socket is set to a busy state when the amount of data queued internally
+  ## by the ERTS socket implementation reaches this limit.
+  ##
+  ## @doc listeners.<name>.tcp.high_watermark
+  ## ValueType: Size
+  ## Default: 1MB
+  tcp.high_watermark = 1MB
+
+  ## The TCP_NODELAY flag for the connections.
+  ##
+  ## @doc listeners.<name>.tcp.nodelay
+  ## ValueType: Boolean
+  ## Default: false
+  tcp.nodelay = false
+
+  ## The SO_REUSEADDR flag for the connections.
+  ##
+  ## @doc listeners.<name>.tcp.reuseaddr
+  ## ValueType: Boolean
+  ## Default: true
+  tcp.reuseaddr = true
+}
+
+## Socket options for SSL connections
+## See: http://erlang.org/doc/man/ssl.html
+example_common_ssl_options {
+
+  ## A performance optimization setting, it allows clients to reuse
+  ## pre-existing sessions, instead of initializing new ones.
+  ## Read more about it here.
+  ##
+  ## @doc listeners.<name>.ssl.reuse_sessions
+  ## ValueType: Boolean
+  ## Default: true
+  ssl.reuse_sessions = true
+
+  ## SSL parameter renegotiation is a feature that allows a client and a server
+  ## to renegotiate the parameters of the SSL connection on the fly.
+  ## RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation,
+  ## you drop support for the insecure renegotiation, prone to MitM attacks.
+  ##
+  ## @doc listeners.<name>.ssl.secure_renegotiate
+  ## ValueType: Boolean
+  ## Default: true
+  ssl.secure_renegotiate = true
+
+  ## In protocols that support client-initiated renegotiation,
+  ## the cost of resources of such an operation is higher for the server than the client.
+  ## This can act as a vector for denial of service attacks.
+  ## The SSL application already takes measures to counter-act such attempts,
+  ## but client-initiated renegotiation can be strictly disabled by setting this option to false.
+  ## The default value is true. Note that disabling renegotiation can result in
+  ## long-lived connections becoming unusable due to limits on
+  ## the number of messages the underlying cipher suite can encipher.
+  ssl.client_renegotiation = true
+
+  ## An important security setting, it forces the cipher to be set based
+  ## on the server-specified order instead of the client-specified order,
+  ## hence enforcing the (usually more properly configured) security
+  ## ordering of the server administrator.
+  ##
+  ## @doc listeners.<name>.ssl.honor_cipher_order
+  ## ValueType: Boolean
+  ## Default: true
+  ssl.honor_cipher_order = true
+
+  # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
+  # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256"
+  # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA"
+  # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA"
+  # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config
+  # NOTE: by default, ALL ciphers are enabled
+  # ssl.ciphers = ""
+
+  ## TLS Handshake timeout.
+  ##
+  ## @doc listeners.<name>.ssl.handshake_timeout
+  ## ValueType: Duration
+  ## Default: 15s
+  ssl.handshake_timeout = 15s
+
+  ## Maximum number of non-self-issued intermediate certificates that
+  ## can follow the peer certificate in a valid certification path.
+  ##
+  ## @doc listeners.<name>.ssl.depth
+  ## ValueType: Integer
+  ## Default: 10
+  ssl.depth = 10
+
+  ## Path to the file containing the user's private PEM-encoded key.
+  ##
+  ## @doc listeners.<name>.ssl.keyfile
+  ## ValueType: File
+  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
+  ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
+
+  ## Path to a file containing the user certificate.
+  ##
+  ## @doc listeners.<name>.ssl.certfile
+  ## ValueType: File
+  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
+  ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
+
+  ## Path to the file containing PEM-encoded CA certificates. The CA certificates
+  ## are used during server authentication and when building the client certificate chain.
+  ##
+  ## @doc listeners.<name>.ssl.cacertfile
+  ## ValueType: File
+  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem"
+  ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem"
+
+  ## Maximum number of non-self-issued intermediate certificates that
+  ## can follow the peer certificate in a valid certification path.
+  ##
+  ## @doc listeners.<name>.ssl.depth
+  ## ValueType: Number
+  ## Default: 10
+  ssl.depth = 10
+
+  ## String containing the user's password. Only used if the private keyfile
+  ## is password-protected.
+  ##
+  ## See: listener.ssl.$name.key_password
+  ##
+  ## @doc listeners.<name>.ssl.depth
+  ## ValueType: String
+  ## Default: ""
+  #ssl.key_password: ""
+
+  ## The Ephemeral Diffie-Helman key exchange is a very effective way of
+  ## ensuring Forward Secrecy by exchanging a set of keys that never hit
+  ## the wire. Since the DH key is effectively signed by the private key,
+  ## it needs to be at least as strong as the private key. In addition,
+  ## the default DH groups that most of the OpenSSL installations have
+  ## are only a handful (since they are distributed with the OpenSSL
+  ## package that has been built for the operating system it’s running on)
+  ## and hence predictable (not to mention, 1024 bits only).
+  ## In order to escape this situation, first we need to generate a fresh,
+  ## strong DH group, store it in a file and then use the option above,
+  ## to force our SSL application to use the new DH group. Fortunately,
+  ## OpenSSL provides us with a tool to do that. Simply run:
+  ## openssl dhparam -out dh-params.pem 2048
+  ##
+  ## @doc listeners.<name>.ssl.dhfile
+  ## ValueType: File
+  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/dh-params.pem"
+  #ssl.dhfile: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/dh-params.pem"
+
+  ## A server only does x509-path validation in mode verify_peer,
+  ## as it then sends a certificate request to the client (this
+  ## message is not sent if the verify option is verify_none).
+  ## You can then also want to specify option fail_if_no_peer_cert.
+  ## More information at: http://erlang.org/doc/man/ssl.html
+  ##
+  ## @doc listeners.<name>.ssl.verify
+  ## ValueType: verify_peer | verify_none
+  ## Default: verify_none
+  ssl.verify = verify_none
+
+  ## Used together with {verify, verify_peer} by an SSL server. If set to true,
+  ## the server fails if the client does not have a certificate to send, that is,
+  ## sends an empty certificate.
+  ##
+  ## @doc listeners.<name>.ssl.fail_if_no_peer_cert
+  ## ValueType: Boolean
+  ## Default: true
+  ssl.fail_if_no_peer_cert = false
+
+}
+
+## Socket options for websocket connections
+example_common_websocket_options {
+  ## The path of WebSocket MQTT endpoint
+  ##
+  ## @doc listeners.<name>.websocket.mqtt_path
+  ## ValueType: Path
+  ## Default: "/mqtt"
+  websocket.mqtt_path = "/mqtt"
+
+  ## Whether a WebSocket message is allowed to contain multiple MQTT packets
+  ##
+  ## @doc listeners.<name>.websocket.mqtt_piggyback
+  ## ValueType: single | multiple
+  ## Default: multiple
+  websocket.mqtt_piggyback = multiple
+
+  ## The compress flag for external WebSocket connections.
+  ##
+  ## If this Value is set true,the websocket message would be compressed
+  ##
+  ## @doc listeners.<name>.websocket.compress
+  ## ValueType: Boolean
+  ## Default: false
+  websocket.compress = false
+
+  ## The idle timeout for external WebSocket connections.
+  ##
+  ## @doc listeners.<name>.websocket.idle_timeout
+  ## ValueType: Duration | infinity
+  ## Default: infinity
+  websocket.idle_timeout = infinity
+
+  ## The max frame size for external WebSocket connections.
+  ##
+  ## @doc listeners.<name>.websocket.max_frame_size
+  ## ValueType: Size
+  ## Default: infinity
+  websocket.max_frame_size = infinity
+
+  ## If set to true, the server fails if the client does not
+  ## have a Sec-WebSocket-Protocol to send.
+  ## Set to false for WeChat MiniApp.
+  ##
+  ## @doc listeners.<name>.websocket.fail_if_no_subprotocol
+  ## ValueType: Boolean
+  ## Default: true
+  websocket.fail_if_no_subprotocol = true
+
+  ## Supported subprotocols
+  ##
+  ## @doc listeners.<name>.websocket.supported_subprotocols
+  ## ValueType: String
+  ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5
+  websocket.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5"
+
+  ## Enable origin check in header for websocket connection
+  ##
+  ## @doc listeners.<name>.websocket.check_origin_enable
+  ## ValueType: Boolean
+  ## Default: false
+  websocket.check_origin_enable = false
+
+  ## Allow origin to be absent in header in websocket connection
+  ## when check_origin_enable is true
+  ##
+  ## @doc listeners.<name>.websocket.allow_origin_absence
+  ## ValueType: Boolean
+  ## Default: true
+  websocket.allow_origin_absence = true
+
+  ## Comma separated list of allowed origin in header for websocket connection
+  ##
+  ## @doc listeners.<name>.websocket.check_origins
+  ## ValueType: String
+  ## Examples:
+  ##  local http dashboard url
+  ## check_origins: "http://localhost:18083, http://127.0.0.1:18083"
+  ## Default: ""
+  websocket.check_origins = "http://localhost:18083, http://127.0.0.1:18083"
+
+  ## Specify which HTTP header for real source IP if the EMQ X cluster is
+  ## deployed behind NGINX or HAProxy.
+  ##
+  ## @doc listeners.<name>.websocket.proxy_address_header
+  ## ValueType: String
+  ## Default: X-Forwarded-For
+  websocket.proxy_address_header = X-Forwarded-For
+
+  ## Specify which HTTP header for real source port if the EMQ X cluster is
+  ## deployed behind NGINX or HAProxy.
+  ##
+  ## @doc listeners.<name>.websocket.proxy_port_header
+  ## ValueType: String
+  ## Default: X-Forwarded-Port
+  websocket.proxy_port_header = X-Forwarded-Port
+
+  websocket.deflate_opts {
+    ## The level of deflate options for external WebSocket connections.
+    ##
+    ## @doc listeners.<name>.websocket.deflate_opts.level
+    ## ValueType: none | default | best_compression | best_speed
+    ## Default: default
+    level = default
+
+    ## The mem_level of deflate options for external WebSocket connections.
+    ##
+    ## @doc listeners.<name>.websocket.deflate_opts.mem_level
+    ## ValueType: Integer
+    ## Range: [1,9]
+    ## Default: 8
+    mem_level = 8
+
+    ## The strategy of deflate options for external WebSocket connections.
+    ##
+    ## @doc listeners.<name>.websocket.deflate_opts.strategy
+    ## ValueType: default | filtered | huffman_only | rle
+    ## Default: default
+    strategy = default
+
+    ## The deflate option for external WebSocket connections.
+    ##
+    ## @doc listeners.<name>.websocket.deflate_opts.server_context_takeover
+    ## ValueType: takeover | no_takeover
+    ## Default: takeover
+    server_context_takeover = takeover
+
+    ## The deflate option for external WebSocket connections.
+    ##
+    ## @doc listeners.<name>.websocket.deflate_opts.client_context_takeover
+    ## ValueType: takeover | no_takeover
+    ## Default: takeover
+    client_context_takeover = takeover
+
+    ## The deflate options for external WebSocket connections.
+    ##
+    ##
+    ## @doc listeners.<name>.websocket.deflate_opts.server_max_window_bits
+    ## ValueType: Integer
+    ## Range: [8,15]
+    ## Default: 15
+    server_max_window_bits = 15
+
+    ## The deflate options for external WebSocket connections.
+    ##
+    ## @doc listeners.<name>.websocket.deflate_opts.client_max_window_bits
+    ## ValueType: Integer
+    ## Range: [8,15]
+    ## Default: 15
+    client_max_window_bits = 15
+  }
+}
+
+persistent_session_store {
+    ## Enable/disable internal persistent session store.
+    ##
+    ## @doc persistent_session_store.enabled
+    ## ValueType: Boolean
+    ## Default: false
+    enabled = false
+
+    ## How long are undelivered messages retained in the store
+    ##
+    ## @doc persistent_session_store.max_retain_undelivered
+    ## ValueType: Duration
+    ## Default: 1h
+    max_retain_undelivered = 1h
+
+    ## The time interval in which to try to run garbage collection of persistent session messages
+    ##
+    ## @doc persistent_session_store.message_gc_interval
+    ## ValueType: Duration
+    ## Default: 1h
+    message_gc_interval = 1h
+
+    ## The time interval in which to try to run garbage collection of persistent session transient data
+    ##
+    ## @doc persistent_session_store.session_message_gc_interval
+    ## ValueType: Duration
+    ## Default: 1m
+    session_message_gc_interval = 1m
+}
diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl
index 7bca3ca3c..67542a2bf 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace.erl
@@ -112,7 +112,6 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
             case logger_config:get(ets:whereis(logger), Id) of
                 {ok, #{module := Module} = HandlerConfig0} ->
                     HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
-                    io:format("~p~n", [{Module, Log, HandlerConfig}]),
                     try Module:log(Log, HandlerConfig)
                     catch C:R:S ->
                         case logger:remove_handler(Id) of
diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl
index 01d587028..f3ab7b5b8 100644
--- a/apps/emqx/test/emqx_trace_handler_SUITE.erl
+++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl
@@ -50,15 +50,15 @@ end_per_testcase(_Case, _Config) ->
 t_trace_clientid(_Config) ->
     %% Start tracing
     %% add list clientid
-    ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"),
-    ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"),
-    ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"),
+    ok = emqx_trace_handler:install("CLI-client1", clientid, "client", debug, "tmp/client.log"),
+    ok = emqx_trace_handler:install("CLI-client2", clientid, <<"client2">>, all, "tmp/client2.log"),
+    ok = emqx_trace_handler:install("CLI-client3", clientid, <<"client3">>, all, "tmp/client3.log"),
     {error, {handler_not_added, {file_error, ".", eisdir}}} =
         emqx_trace_handler:install(clientid, <<"client5">>, debug, "."),
     emqx_trace:check(),
-    ok = filesync(<<"client">>, clientid),
-    ok = filesync(<<"client2">>, clientid),
-    ok = filesync(<<"client3">>, clientid),
+    ok = filesync(<<"CLI-client1">>, clientid),
+    ok = filesync(<<"CLI-client2">>, clientid),
+    ok = filesync(<<"CLI-client3">>, clientid),
 
     %% Verify the tracing file exits
     ?assert(filelib:is_regular("tmp/client.log")),
@@ -66,11 +66,11 @@ t_trace_clientid(_Config) ->
     ?assert(filelib:is_regular("tmp/client3.log")),
 
     %% Get current traces
-    ?assertMatch([#{type := clientid, filter := "client", name := <<"client">>,
+    ?assertMatch([#{type := clientid, filter := "client", name := <<"CLI-client1">>,
         level := debug, dst := "tmp/client.log"},
-        #{type := clientid, filter := "client2", name := <<"client2">>
+        #{type := clientid, filter := "client2", name := <<"CLI-client2">>
             , level := debug, dst := "tmp/client2.log"},
-        #{type := clientid, filter := "client3", name := <<"client3">>,
+        #{type := clientid, filter := "client3", name := <<"CLI-client3">>,
             level := debug, dst := "tmp/client3.log"}
     ], emqx_trace_handler:running()),
 
@@ -79,9 +79,9 @@ t_trace_clientid(_Config) ->
     emqtt:connect(T),
     emqtt:publish(T, <<"a/b/c">>, <<"hi">>),
     emqtt:ping(T),
-    ok = filesync(<<"client">>, clientid),
-    ok = filesync(<<"client2">>, clientid),
-    ok = filesync(<<"client3">>, clientid),
+    ok = filesync(<<"CLI-client1">>, clientid),
+    ok = filesync(<<"CLI-client2">>, clientid),
+    ok = filesync(<<"CLI-client3">>, clientid),
 
     %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log".
     {ok, Bin} = file:read_file("tmp/client.log"),
@@ -92,24 +92,24 @@ t_trace_clientid(_Config) ->
     ?assert(filelib:file_size("tmp/client2.log") == 0),
 
     %% Stop tracing
-    ok = emqx_trace_handler:uninstall(clientid, <<"client">>),
-    ok = emqx_trace_handler:uninstall(clientid, <<"client2">>),
-    ok = emqx_trace_handler:uninstall(clientid, <<"client3">>),
+    ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client1">>),
+    ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client2">>),
+    ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client3">>),
 
     emqtt:disconnect(T),
     ?assertEqual([], emqx_trace_handler:running()).
 
 t_trace_clientid_utf8(_) ->
     Utf8Id = <<"client 漢字編碼"/utf8>>,
-    ok = emqx_trace_handler:install(clientid, Utf8Id, debug, "tmp/client-utf8.log"),
+    ok = emqx_trace_handler:install("CLI-UTF8", clientid, Utf8Id, debug, "tmp/client-utf8.log"),
     emqx_trace:check(),
     {ok, T} = emqtt:start_link([{clientid, Utf8Id}]),
     emqtt:connect(T),
     [begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)],
     emqtt:ping(T),
 
-    ok = filesync(Utf8Id, clientid),
-    ok = emqx_trace_handler:uninstall(clientid, Utf8Id),
+    ok = filesync("CLI-UTF8", clientid),
+    ok = emqx_trace_handler:uninstall(clientid, "CLI-UTF8"),
     emqtt:disconnect(T),
     ?assertEqual([], emqx_trace_handler:running()),
     ok.
@@ -119,11 +119,11 @@ t_trace_topic(_Config) ->
     emqtt:connect(T),
 
     %% Start tracing
-    ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"),
-    ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"),
+    ok = emqx_trace_handler:install("CLI-TOPIC-1", topic, <<"x/#">>, all, "tmp/topic_trace_x.log"),
+    ok = emqx_trace_handler:install("CLI-TOPIC-2", topic, <<"y/#">>, all, "tmp/topic_trace_y.log"),
     emqx_trace:check(),
-    ok = filesync(<<"x/#">>, topic),
-    ok = filesync(<<"y/#">>, topic),
+    ok = filesync("CLI-TOPIC-1", topic),
+    ok = filesync("CLI-TOPIC-2", topic),
 
     %% Verify the tracing file exits
     ?assert(filelib:is_regular("tmp/topic_trace_x.log")),
@@ -131,9 +131,9 @@ t_trace_topic(_Config) ->
 
     %% Get current traces
     ?assertMatch([#{type := topic, filter := <<"x/#">>,
-                    level := debug, dst := "tmp/topic_trace_x.log", name := <<"x/#">>},
+                    level := debug, dst := "tmp/topic_trace_x.log", name := <<"CLI-TOPIC-1">>},
                   #{type := topic, filter := <<"y/#">>,
-                      name := <<"y/#">>, level := debug, dst := "tmp/topic_trace_y.log"}
+                      name := <<"CLI-TOPIC-2">>, level := debug, dst := "tmp/topic_trace_y.log"}
                  ],
         emqx_trace_handler:running()),
 
@@ -142,8 +142,8 @@ t_trace_topic(_Config) ->
     emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
     emqtt:subscribe(T, <<"x/y/z">>),
     emqtt:unsubscribe(T, <<"x/y/z">>),
-    ok = filesync(<<"x/#">>, topic),
-    ok = filesync(<<"y/#">>, topic),
+    ok = filesync("CLI-TOPIC-1", topic),
+    ok = filesync("CLI-TOPIC-2", topic),
 
     {ok, Bin} = file:read_file("tmp/topic_trace_x.log"),
     ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@@ -154,8 +154,8 @@ t_trace_topic(_Config) ->
     ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0),
 
     %% Stop tracing
-    ok = emqx_trace_handler:uninstall(topic, <<"x/#">>),
-    ok = emqx_trace_handler:uninstall(topic, <<"y/#">>),
+    ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-1">>),
+    ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-2">>),
     {error, _Reason} = emqx_trace_handler:uninstall(topic, <<"z/#">>),
     ?assertEqual([], emqx_trace_handler:running()),
     emqtt:disconnect(T).
@@ -165,11 +165,12 @@ t_trace_ip_address(_Config) ->
     emqtt:connect(T),
 
     %% Start tracing
-    ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"),
-    ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"),
+    ok = emqx_trace_handler:install("CLI-IP-1", ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"),
+    ok = emqx_trace_handler:install("CLI-IP-2", ip_address,
+        "192.168.1.1", all, "tmp/ip_trace_y.log"),
     emqx_trace:check(),
-    ok = filesync(<<"127.0.0.1">>, ip_address),
-    ok = filesync(<<"192.168.1.1">>, ip_address),
+    ok = filesync(<<"CLI-IP-1">>, ip_address),
+    ok = filesync(<<"CLI-IP-2">>, ip_address),
 
     %% Verify the tracing file exits
     ?assert(filelib:is_regular("tmp/ip_trace_x.log")),
@@ -177,10 +178,10 @@ t_trace_ip_address(_Config) ->
 
     %% Get current traces
     ?assertMatch([#{type := ip_address, filter := "127.0.0.1",
-                    name := <<"127.0.0.1">>,
+                    name := <<"CLI-IP-1">>,
                     level := debug, dst := "tmp/ip_trace_x.log"},
                   #{type := ip_address, filter := "192.168.1.1",
-                      name := <<"192.168.1.1">>,
+                      name := <<"CLI-IP-2">>,
                     level := debug, dst := "tmp/ip_trace_y.log"}
                  ],
         emqx_trace_handler:running()),
@@ -190,8 +191,8 @@ t_trace_ip_address(_Config) ->
     emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
     emqtt:subscribe(T, <<"x/y/z">>),
     emqtt:unsubscribe(T, <<"x/y/z">>),
-    ok = filesync(<<"127.0.0.1">>, ip_address),
-    ok = filesync(<<"192.168.1.1">>, ip_address),
+    ok = filesync(<<"CLI-IP-1">>, ip_address),
+    ok = filesync(<<"CLI-IP-2">>, ip_address),
 
     {ok, Bin} = file:read_file("tmp/ip_trace_x.log"),
     ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@@ -202,8 +203,8 @@ t_trace_ip_address(_Config) ->
     ?assert(filelib:file_size("tmp/ip_trace_y.log") =:= 0),
 
     %% Stop tracing
-    ok = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.1">>),
-    ok = emqx_trace_handler:uninstall(ip_address, <<"192.168.1.1">>),
+    ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-1">>),
+    ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-2">>),
     {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>),
     emqtt:disconnect(T),
     ?assertEqual([], emqx_trace_handler:running()).
@@ -215,7 +216,12 @@ filesync(Name, Type) ->
 
 %% sometime the handler process is not started yet.
 filesync(_Name, _Type, 0) -> ok;
-filesync(Name, Type, Retry) ->
+filesync(Name0, Type, Retry) ->
+    Name =
+        case is_binary(Name0) of
+            true -> Name0;
+            false -> list_to_binary(Name0)
+        end,
     try
         Handler = binary_to_atom(<<"trace_",
             (atom_to_binary(Type))/binary, "_", Name/binary>>),
diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
index 5669e5653..d6902d123 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
@@ -142,11 +142,11 @@ fields(trace) ->
             #{desc => """Filter type""",
                 nullable => false,
                 example => <<"clientid">>})},
-        {topic, hoconsc:mk(emqx_schema:unicode_binary(),
+        {topic, hoconsc:mk(binary(),
             #{desc => """support mqtt wildcard topic.""",
                 nullable => true,
                 example => <<"/dev/#">>})},
-        {clientid, hoconsc:mk(emqx_schema:unicode_binary(),
+        {clientid, hoconsc:mk(binary(),
             #{desc => """mqtt clientid.""",
                 nullable => true,
                 example => <<"dev-001">>})},
diff --git a/apps/emqx_modules/etc/emqx_modules.conf.rendered b/apps/emqx_modules/etc/emqx_modules.conf.rendered
new file mode 100644
index 000000000..be34479e8
--- /dev/null
+++ b/apps/emqx_modules/etc/emqx_modules.conf.rendered
@@ -0,0 +1,50 @@
+
+delayed {
+    enable = true
+    ## 0 is no limit
+    max_delayed_messages = 0
+}
+
+observer_cli {
+    enable = true
+}
+
+telemetry {
+    enable = true
+}
+
+event_message {
+    "$event/client_connected" = true
+    "$event/client_disconnected" = true
+    # "$event/client_subscribed": false
+    # "$event/client_unsubscribed": false
+    # "$event/message_delivered": false
+    # "$event/message_acked": false
+    # "$event/message_dropped": false
+}
+
+topic_metrics: [
+    #{topic: "test/1"}
+]
+
+rewrite: [
+    # {
+    #     action = publish
+    #     source_topic = "x/#"
+    #     re = "^x/y/(.+)$"
+    #     dest_topic = "z/y/$1"
+    # },
+    # {
+    #     action = subscribe
+    #     source_topic = "x1/#"
+    #     re = "^x1/y/(.+)$"
+    #     dest_topic = "z1/y/$1"
+    # },
+    # {
+    #     action = all
+    #     source_topic = "x2/#"
+    #     re = "^x2/y/(.+)$"
+    #     dest_topic = "z2/y/$1"
+    # }
+]
+

From 121d90699244dd33b5731f64d0f4d1224f541766 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 29 Dec 2021 12:49:56 +0800
Subject: [PATCH 4/7] chore(log): add SLOG/3 to add meta info

---
 apps/emqx/include/logger.hrl                  | 22 +++++++++++--------
 apps/emqx/src/emqx_authentication_config.erl  |  4 ++--
 apps/emqx/src/emqx_broker.erl                 | 13 ++++++-----
 apps/emqx/src/emqx_channel.erl                | 12 +++++-----
 apps/emqx/src/emqx_cm.erl                     |  9 +++++---
 apps/emqx/src/emqx_config.erl                 |  4 ++--
 apps/emqx/src/emqx_connection.erl             |  4 ++--
 apps/emqx/src/emqx_flapping.erl               |  4 ++--
 apps/emqx/src/emqx_session.erl                | 12 ++++++----
 apps/emqx/src/emqx_session_router.erl         |  2 +-
 apps/emqx/src/emqx_trace/emqx_trace.erl       |  6 ++++-
 apps/emqx/src/emqx_ws_connection.erl          |  4 ++--
 apps/emqx_authn/test/emqx_authn_api_SUITE.erl |  4 +---
 apps/emqx_bridge/src/emqx_bridge.erl          |  4 ++--
 apps/emqx_conf/src/emqx_cluster_rpc.erl       |  6 ++---
 apps/emqx_conf/src/emqx_conf.erl              |  2 +-
 .../src/emqx_connector_http.erl               |  8 +++----
 .../src/emqx_connector_ldap.erl               |  4 ++--
 .../src/emqx_connector_mongo.erl              |  2 +-
 .../src/emqx_connector_mysql.erl              |  4 ++--
 .../src/emqx_connector_pgsql.erl              |  2 +-
 .../src/emqx_connector_redis.erl              |  4 ++--
 .../src/mqtt/emqx_connector_mqtt_mod.erl      |  8 +++----
 .../src/mqtt/emqx_connector_mqtt_worker.erl   | 12 +++++-----
 24 files changed, 86 insertions(+), 70 deletions(-)

diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl
index 42d598ef9..ddd4349bb 100644
--- a/apps/emqx/include/logger.hrl
+++ b/apps/emqx/include/logger.hrl
@@ -59,15 +59,19 @@
 
 %% structured logging
 -define(SLOG(Level, Data),
-        %% check 'allow' here, only evaluate Data when necessary
-        case logger:allow(Level, ?MODULE) of
-            true ->
-                logger:log(Level, (Data), #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
-                                           , line => ?LINE
-                                           });
-            false ->
-                ok
-        end).
+        ?SLOG(Level, Data, #{})).
+
+%% structured logging, meta is for handler's filter.
+-define(SLOG(Level, Data, Meta),
+%% check 'allow' here, only evaluate Data when necessary
+    case logger:allow(Level, ?MODULE) of
+        true ->
+            logger:log(Level, (Data), Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
+                , line => ?LINE
+            });
+        false ->
+            ok
+    end).
 
 -define(TRACE(Event, Msg, Meta), emqx_trace:log(Event, Msg, Meta)).
 
diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl
index 795dd060e..9767a2265 100644
--- a/apps/emqx/src/emqx_authentication_config.erl
+++ b/apps/emqx/src/emqx_authentication_config.erl
@@ -187,7 +187,7 @@ convert_certs(CertsDir, Config) ->
         {ok, SSL} ->
             new_ssl_config(Config, SSL);
         {error, Reason} ->
-            ?SLOG(error, Reason#{msg => bad_ssl_config}),
+            ?SLOG(error, Reason#{msg => "bad_ssl_config"}),
             throw({bad_ssl_config, Reason})
     end.
 
@@ -199,7 +199,7 @@ convert_certs(CertsDir, NewConfig, OldConfig) ->
             ok = emqx_tls_lib:delete_ssl_files(CertsDir, NewSSL1, OldSSL),
             new_ssl_config(NewConfig, NewSSL1);
         {error, Reason} ->
-            ?SLOG(error, Reason#{msg => bad_ssl_config}),
+            ?SLOG(error, Reason#{msg => "bad_ssl_config"}),
             throw({bad_ssl_config, Reason})
     end.
 
diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl
index dec753fc2..4085b6130 100644
--- a/apps/emqx/src/emqx_broker.erl
+++ b/apps/emqx/src/emqx_broker.erl
@@ -204,8 +204,9 @@ publish(Msg) when is_record(Msg, message) ->
     _ = emqx_trace:publish(Msg),
     emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'),
     case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of
-        #message{headers = #{allow_publish := false}} ->
-            ?TRACE("MQTT", "msg_publish_not_allowed", #{message => emqx_message:to_log_map(Msg)}),
+        #message{headers = #{allow_publish := false}, topic = Topic} ->
+            Message = emqx_message:to_log_map(Msg),
+            ?TRACE("MQTT", "msg_publish_not_allowed", #{message => Message, topic => Topic}),
             [];
         Msg1 = #message{topic = Topic} ->
             emqx_persistent_session:persist_message(Msg1),
@@ -225,7 +226,9 @@ safe_publish(Msg) when is_record(Msg, message) ->
                 reason => Reason,
                 payload => emqx_message:to_log_map(Msg),
                 stacktrace => Stk
-            }),
+            },
+                #{topic => Msg#message.topic}
+            ),
             []
     end.
 
@@ -279,7 +282,7 @@ forward(Node, To, Delivery, async) ->
                 msg => "async_forward_msg_to_node_failed",
                 node => Node,
                 reason => Reason
-            }),
+            }, #{topic => To}),
             {error, badrpc}
     end;
 
@@ -290,7 +293,7 @@ forward(Node, To, Delivery, sync) ->
                 msg => "sync_forward_msg_to_node_failed",
                 node => Node,
                 reason => Reason
-            }),
+            }, #{topic => To}),
             {error, badrpc};
         Result ->
             emqx_metrics:inc('messages.forward'), Result
diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index 0f83b04ff..d0cb07a68 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -552,7 +552,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
                 msg => "cannot_publish_to_topic",
                 topic => Topic,
                 reason => emqx_reason_codes:name(Rc)
-            }),
+            }, #{topic => Topic}),
             case emqx:get_config([authorization, deny_action], ignore) of
                 ignore ->
                     case QoS of
@@ -570,7 +570,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
                 msg => "cannot_publish_to_topic",
                 topic => Topic,
                 reason => emqx_reason_codes:name(Rc)
-            }),
+            }, #{topic => Topic}),
             case QoS of
                 ?QOS_0 ->
                     ok = emqx_metrics:inc('packets.publish.dropped'),
@@ -585,7 +585,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
                 msg => "cannot_publish_to_topic",
                 topic => Topic,
                 reason => emqx_reason_codes:name(Rc)
-            }),
+            }, #{topic => Topic}),
             handle_out(disconnect, Rc, NChannel)
     end.
 
@@ -635,7 +635,7 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2},
                 msg => "dropped_qos2_packet",
                 reason => emqx_reason_codes:name(RC),
                 packet_id => PacketId
-            }),
+            }, #{topic => Msg#message.topic}),
             ok = emqx_metrics:inc('packets.publish.dropped'),
             handle_out(pubrec, {PacketId, RC}, Channel)
     end.
@@ -687,7 +687,7 @@ process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Ac
             ?SLOG(warning, #{
                 msg => "cannot_subscribe_topic_filter",
                 reason => emqx_reason_codes:name(ReasonCode)
-            }),
+            }, #{topic => TopicFilter}),
             process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc])
     end.
 
@@ -703,7 +703,7 @@ do_subscribe(TopicFilter, SubOpts = #{qos := QoS}, Channel =
             ?SLOG(warning, #{
                 msg => "cannot_subscribe_topic_filter",
                 reason => emqx_reason_codes:text(RC)
-            }),
+            }, #{topic => NTopicFilter}),
             {RC, Channel}
     end.
 
diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl
index 162cff2e0..c44cfe15e 100644
--- a/apps/emqx/src/emqx_cm.erl
+++ b/apps/emqx/src/emqx_cm.erl
@@ -448,20 +448,23 @@ kick_session(Action, ClientId, ChanPid) ->
                           , action => Action
                           , error => Error
                           , reason => Reason
-                          })
+                          },
+                #{clientid => unicode:characters_to_list(ClientId, utf8)})
     end.
 
 kick_session(ClientId) ->
     case lookup_channels(ClientId) of
         [] ->
             ?SLOG(warning, #{msg => "kicked_an_unknown_session",
-                             clientid => ClientId}),
+                             clientid => ClientId},
+                #{clientid => unicode:characters_to_list(ClientId, utf8)}),
             ok;
         ChanPids ->
             case length(ChanPids) > 1 of
                 true ->
                     ?SLOG(warning, #{msg => "more_than_one_channel_found",
-                                     chan_pids => ChanPids});
+                                     chan_pids => ChanPids},
+                        #{clientid => unicode:characters_to_list(ClientId, utf8)});
                 false -> ok
             end,
             lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids)
diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl
index f3e6e1366..35a6c048e 100644
--- a/apps/emqx/src/emqx_config.erl
+++ b/apps/emqx/src/emqx_config.erl
@@ -262,7 +262,7 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) ->
         {ok, RawRichConf} ->
             init_load(SchemaMod, RawRichConf);
         {error, Reason} ->
-            ?SLOG(error, #{msg => failed_to_load_hocon_conf,
+            ?SLOG(error, #{msg => "failed_to_load_hocon_conf",
                            reason => Reason,
                            include_dirs => IncDir
                           }),
@@ -396,7 +396,7 @@ save_to_override_conf(RawConf, Opts) ->
             case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
                 ok -> ok;
                 {error, Reason} ->
-                    ?SLOG(error, #{msg => failed_to_write_override_file,
+                    ?SLOG(error, #{msg => "failed_to_write_override_file",
                                    filename => FileName,
                                    reason => Reason}),
                     {error, Reason}
diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 37e15f522..e1dab3260 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -872,7 +872,7 @@ check_limiter(Needs,
                 {ok, Limiter2} ->
                     WhenOk(Data, Msgs, State#state{limiter = Limiter2});
                 {pause, Time, Limiter2} ->
-                    ?SLOG(warning, #{msg => "pause time dueto rate limit",
+                    ?SLOG(warning, #{msg => "pause_time_dueto_rate_limit",
                                      needs => Needs,
                                      time_in_ms => Time}),
 
@@ -912,7 +912,7 @@ retry_limiter(#state{limiter = Limiter} = State) ->
                                 , limiter_timer = undefined
                                 });
             {pause, Time, Limiter2} ->
-                ?SLOG(warning, #{msg => "pause time dueto rate limit",
+                ?SLOG(warning, #{msg => "pause_time_dueto_rate_limit",
                                  types => Types,
                                  time_in_ms => Time}),
 
diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl
index 600144adc..cb3da361a 100644
--- a/apps/emqx/src/emqx_flapping.erl
+++ b/apps/emqx/src/emqx_flapping.erl
@@ -122,7 +122,7 @@ handle_cast({detected, #flapping{clientid   = ClientId,
                 peer_host => fmt_host(PeerHost),
                 detect_cnt => DetectCnt,
                 wind_time_in_ms => WindTime
-            }),
+            }, #{clientid => unicode:characters_to_list(ClientId, utf8)}),
             Now = erlang:system_time(second),
             Banned = #banned{who    = {clientid, ClientId},
                              by     = <<"flapping detector">>,
@@ -138,7 +138,7 @@ handle_cast({detected, #flapping{clientid   = ClientId,
                 peer_host => fmt_host(PeerHost),
                 detect_cnt => DetectCnt,
                 interval => Interval
-            })
+            }, #{clientid => unicode:characters_to_list(ClientId, utf8)})
     end,
     {noreply, State};
 
diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl
index bf79085af..1695ed6ce 100644
--- a/apps/emqx/src/emqx_session.erl
+++ b/apps/emqx/src/emqx_session.erl
@@ -535,16 +535,20 @@ enqueue(Msg, Session = #session{mqueue = Q}) when is_record(Msg, message) ->
     (Dropped =/= undefined) andalso log_dropped(Dropped, Session),
     Session#session{mqueue = NewQ}.
 
-log_dropped(Msg = #message{qos = QoS}, #session{mqueue = Q}) ->
-    case (QoS == ?QOS_0) andalso (not emqx_mqueue:info(store_qos0, Q)) of
+log_dropped(Msg = #message{qos = QoS, topic = Topic}, #session{mqueue = Q}) ->
+    Payload = emqx_message:to_log_map(Msg),
+     #{store_qos0 := StoreQos0} = QueueInfo = emqx_mqueue:info(Q),
+    case (QoS == ?QOS_0) andalso (not StoreQos0) of
         true  ->
             ok = emqx_metrics:inc('delivery.dropped.qos0_msg'),
             ?SLOG(warning, #{msg => "dropped_qos0_msg",
-                             payload => emqx_message:to_log_map(Msg)});
+                             queue => QueueInfo,
+                             payload => Payload}, #{topic => Topic});
         false ->
             ok = emqx_metrics:inc('delivery.dropped.queue_full'),
             ?SLOG(warning, #{msg => "dropped_msg_due_to_mqueue_is_full",
-                             payload => emqx_message:to_log_map(Msg)})
+                             queue => QueueInfo,
+                             payload => Payload}, #{topic => Topic})
     end.
 
 enrich_fun(Session = #session{subscriptions = Subs}) ->
diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl
index aaaedcb12..3d3722c32 100644
--- a/apps/emqx/src/emqx_session_router.erl
+++ b/apps/emqx/src/emqx_session_router.erl
@@ -260,7 +260,7 @@ code_change(_OldVsn, State, _Extra) ->
 init_resume_worker(RemotePid, SessionID, #{ pmon := Pmon } = State) ->
     case emqx_session_router_worker_sup:start_worker(SessionID, RemotePid) of
         {error, What} ->
-            ?SLOG(error, #{msg => "Could not start resume worker", reason => What}),
+            ?SLOG(error, #{msg => "failed_to_start_resume_worker", reason => What}),
             error;
         {ok, Pid} ->
             Pmon1 = emqx_pmon:monitor(Pid, Pmon),
diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl
index 67542a2bf..3869d300d 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace.erl
@@ -98,7 +98,11 @@ log(Event, Msg, Meta0) ->
     case persistent_term:get(?TRACE_FILTER, undefined) of
         undefined -> ok;
         List ->
-            Meta = maps:merge(logger:get_process_metadata(), Meta0),
+            Meta =
+                case logger:get_process_metadata() of
+                    undefined -> Meta0;
+                    ProcMeta -> maps:merge(ProcMeta, Meta0)
+                end,
             Log = #{level => trace, event => Event, meta => Meta, msg => Msg},
             log_filter(List, Log)
     end.
diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl
index f70d7ac6c..e2bdf6c72 100644
--- a/apps/emqx/src/emqx_ws_connection.erl
+++ b/apps/emqx/src/emqx_ws_connection.erl
@@ -549,7 +549,7 @@ check_limiter(Needs,
                 {ok, Limiter2} ->
                     WhenOk(Data, Msgs, State#state{limiter = Limiter2});
                 {pause, Time, Limiter2} ->
-                    ?SLOG(warning, #{msg => "pause time dueto rate limit",
+                    ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit",
                                      needs => Needs,
                                      time_in_ms => Time}),
 
@@ -585,7 +585,7 @@ retry_limiter(#state{limiter = Limiter} = State) ->
                             , limiter_timer = undefined
                             });
         {pause, Time, Limiter2} ->
-            ?SLOG(warning, #{msg => "pause time dueto rate limit",
+            ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit",
                              types => Types,
                              time_in_ms => Time}),
 
diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
index 885811fec..820eeb859 100644
--- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
@@ -43,7 +43,6 @@ groups() ->
     [].
 
 init_per_testcase(_, Config) ->
-    {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
     emqx_authn_test_lib:delete_authenticators(
       [?CONF_NS_ATOM],
       ?GLOBAL),
@@ -56,9 +55,8 @@ init_per_testcase(_, Config) ->
     Config.
 
 init_per_suite(Config) ->
-    _ = application:load(emqx_conf),
     ok = emqx_common_test_helpers:start_apps(
-           [emqx_authn, emqx_dashboard],
+           [emqx_conf, emqx_authn, emqx_dashboard],
            fun set_special_configs/1),
 
     ?AUTHN:delete_chain(?GLOBAL),
diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index d4fc3df2d..a6681d3f1 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -214,7 +214,7 @@ update(Type, Name, {OldConf, Conf}) ->
             case recreate(Type, Name, Conf) of
                 {ok, _} -> maybe_disable_bridge(Type, Name, Conf);
                 {error, not_found} ->
-                    ?SLOG(warning, #{ msg => "updating a non-exist bridge, create a new one"
+                    ?SLOG(warning, #{ msg => "updating_a_non-exist_bridge_need_create_a_new_one"
                                     , type => Type, name => Name, config => Conf}),
                     create(Type, Name, Conf);
                 {error, Reason} -> {update_bridge_failed, Reason}
@@ -242,7 +242,7 @@ create_dry_run(Type, Conf) ->
     end.
 
 remove(Type, Name, _Conf) ->
-    ?SLOG(info, #{msg => "remove bridge", type => Type, name => Name}),
+    ?SLOG(info, #{msg => "remove_bridge", type => Type, name => Name}),
     case emqx_resource:remove_local(resource_id(Type, Name)) of
         ok -> ok;
         {error, not_found} -> ok;
diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl
index 7ebe7645b..514b9156a 100644
--- a/apps/emqx_conf/src/emqx_cluster_rpc.erl
+++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl
@@ -236,7 +236,7 @@ catch_up(#{node := Node, retry_interval := RetryMs} = State, SkipResult) ->
                 false -> RetryMs
             end;
         {aborted, Reason} ->
-            ?SLOG(error, #{msg => "read_next_mfa transaction failed", error => Reason}),
+            ?SLOG(error, #{msg => "read_next_mfa_transaction_failed", error => Reason}),
             RetryMs
     end.
 
@@ -248,7 +248,7 @@ read_next_mfa(Node) ->
                 TnxId = max(LatestId - 1, 0),
                 commit(Node, TnxId),
                 ?SLOG(notice, #{
-                    msg => "New node first catch up and start commit.",
+                    msg => "new_node_first_catch_up_and_start_commit.",
                     node => Node, tnx_id => TnxId}),
                 TnxId;
             [#cluster_rpc_commit{tnx_id = LastAppliedID}] -> LastAppliedID + 1
@@ -277,7 +277,7 @@ do_catch_up(ToTnxId, Node) ->
                 io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)",
                 [Node, LastAppliedId, ToTnxId])),
             ?SLOG(error, #{
-                msg => "catch up failed!",
+                msg => "catch_up_failed!",
                 last_applied_id => LastAppliedId,
                 to_tnx_id => ToTnxId
             }),
diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl
index dec07f35c..b8a8c211d 100644
--- a/apps/emqx_conf/src/emqx_conf.erl
+++ b/apps/emqx_conf/src/emqx_conf.erl
@@ -144,7 +144,7 @@ multicall(M, F, Args) ->
         {retry, TnxId, Res, Nodes} ->
             %% The init MFA return ok, but other nodes failed.
             %% We return ok and alert an alarm.
-            ?SLOG(error, #{msg => "failed to update config in cluster", nodes => Nodes,
+            ?SLOG(error, #{msg => "failed_to_update_config_in_cluster", nodes => Nodes,
                 tnx_id => TnxId, mfa => {M, F, Args}}),
             Res;
         {error, Error} -> %% all MFA return not ok or {ok, term()}.
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 7d7771503..f597a5c1d 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -143,7 +143,7 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
                    retry_interval := RetryInterval,
                    pool_type := PoolType,
                    pool_size := PoolSize} = Config) ->
-    ?SLOG(info, #{msg => "starting http connector",
+    ?SLOG(info, #{msg => "starting_http_connector",
                   connector => InstId, config => Config}),
     {Transport, TransportOpts} = case Scheme of
                                      http ->
@@ -181,13 +181,13 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
     end.
 
 on_stop(InstId, #{pool_name := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping http connector",
+    ?SLOG(info, #{msg => "stopping_http_connector",
                   connector => InstId}),
     ehttpc_sup:stop_pool(PoolName).
 
 on_query(InstId, {send_message, Msg}, AfterQuery, State) ->
     case maps:get(request, State, undefined) of
-        undefined -> ?SLOG(error, #{msg => "request not found", connector => InstId});
+        undefined -> ?SLOG(error, #{msg => "request_not_found", connector => InstId});
         Request ->
             #{method := Method, path := Path, body := Body, headers := Headers,
               request_timeout := Timeout} = process_request(Request, Msg),
@@ -207,7 +207,7 @@ on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery,
                                      _ -> {PoolName, KeyOrNum}
                                  end, Method, NRequest, Timeout) of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "http connector do reqeust failed",
+            ?SLOG(error, #{msg => "http_connector_do_reqeust_failed",
                            request => NRequest, reason => Reason,
                            connector => InstId}),
             emqx_resource:query_failed(AfterQuery);
diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl
index 97e963f18..c188837ba 100644
--- a/apps/emqx_connector/src/emqx_connector_ldap.erl
+++ b/apps/emqx_connector/src/emqx_connector_ldap.erl
@@ -55,7 +55,7 @@ on_start(InstId, #{servers := Servers0,
                    pool_size := PoolSize,
                    auto_reconnect := AutoReconn,
                    ssl := SSL} = Config) ->
-    ?SLOG(info, #{msg => "starting ldap connector",
+    ?SLOG(info, #{msg => "starting_ldap_connector",
                   connector => InstId, config => Config}),
     Servers = [begin proplists:get_value(host, S) end || S <- Servers0],
     SslOpts = case maps:get(enable, SSL) of
@@ -81,7 +81,7 @@ on_start(InstId, #{servers := Servers0,
     {ok, #{poolname => PoolName}}.
 
 on_stop(InstId, #{poolname := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping ldap connector",
+    ?SLOG(info, #{msg => "stopping_ldap_connector",
                   connector => InstId}),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index eacb3ec2d..5321ea459 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -128,7 +128,7 @@ on_start(InstId, Config = #{mongo_type := Type,
     {ok, #{poolname => PoolName, type => Type}}.
 
 on_stop(InstId, #{poolname := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping mongodb connector",
+    ?SLOG(info, #{msg => "stopping_mongodb_connector",
                   connector => InstId}),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index ae8239936..265a6a01e 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -56,7 +56,7 @@ on_start(InstId, #{server := {Host, Port},
                    auto_reconnect := AutoReconn,
                    pool_size := PoolSize,
                    ssl := SSL } = Config) ->
-    ?SLOG(info, #{msg => "starting mysql connector",
+    ?SLOG(info, #{msg => "starting_mysql_connector",
                   connector => InstId, config => Config}),
     SslOpts = case maps:get(enable, SSL) of
         true ->
@@ -76,7 +76,7 @@ on_start(InstId, #{server := {Host, Port},
     {ok, #{poolname => PoolName}}.
 
 on_stop(InstId, #{poolname := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping mysql connector",
+    ?SLOG(info, #{msg => "stopping_mysql_connector",
                   connector => InstId}),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index 9b6f559b4..36bff3386 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -56,7 +56,7 @@ on_start(InstId, #{server := {Host, Port},
                    auto_reconnect := AutoReconn,
                    pool_size := PoolSize,
                    ssl := SSL } = Config) ->
-    ?SLOG(info, #{msg => "starting postgresql connector",
+    ?SLOG(info, #{msg => "starting_postgresql_connector",
                   connector => InstId, config => Config}),
     SslOpts = case maps:get(enable, SSL) of
         true ->
diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl
index 94f4eca3e..48001ca25 100644
--- a/apps/emqx_connector/src/emqx_connector_redis.erl
+++ b/apps/emqx_connector/src/emqx_connector_redis.erl
@@ -87,7 +87,7 @@ on_start(InstId, #{redis_type := Type,
                    pool_size := PoolSize,
                    auto_reconnect := AutoReconn,
                    ssl := SSL } = Config) ->
-    ?SLOG(info, #{msg => "starting redis connector",
+    ?SLOG(info, #{msg => "starting_redis_connector",
                   connector => InstId, config => Config}),
     Servers = case Type of
                 single -> [{servers, [maps:get(server, Config)]}];
@@ -120,7 +120,7 @@ on_start(InstId, #{redis_type := Type,
     {ok, #{poolname => PoolName, type => Type}}.
 
 on_stop(InstId, #{poolname := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping redis connector",
+    ?SLOG(info, #{msg => "stopping_redis_connector",
                   connector => InstId}),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
index 7d5bb1283..3ab410391 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
@@ -158,15 +158,15 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent)
        RC =:= ?RC_NO_MATCHING_SUBSCRIBERS ->
     Parent ! {batch_ack, PktId}, ok;
 handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) ->
-    ?SLOG(warning, #{msg => "publish to remote node falied",
+    ?SLOG(warning, #{msg => "publish_to_remote_node_falied",
         packet_id => PktId, reason_code => RC}).
 
 handle_publish(Msg, undefined) ->
-    ?SLOG(error, #{msg => "cannot publish to local broker as"
-                          " 'ingress' is not configured",
+    ?SLOG(error, #{msg => "cannot_publish_to_local_broker_as"
+                          "_'ingress'_is_not_configured",
                    message => Msg});
 handle_publish(Msg, Vars) ->
-    ?SLOG(debug, #{msg => "publish to local broker",
+    ?SLOG(debug, #{msg => "publish_to_local_broker",
                    message => Msg, vars => Vars}),
     emqx_metrics:inc('bridge.mqtt.message_received_from_remote', 1),
     case Vars of
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
index 5f6f4b69f..e0d5a2d77 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
@@ -188,7 +188,7 @@ callback_mode() -> [state_functions].
 
 %% @doc Config should be a map().
 init(#{name := Name} = ConnectOpts) ->
-    ?SLOG(debug, #{msg => "starting bridge worker",
+    ?SLOG(debug, #{msg => "starting_bridge_worker",
                    name => Name}),
     erlang:process_flag(trap_exit, true),
     Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})),
@@ -335,7 +335,7 @@ common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) ->
     NewQ = replayq:append(Q, [Msg]),
     {keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}};
 common(StateName, Type, Content, #{name := Name} = State) ->
-    ?SLOG(notice, #{msg => "Bridge discarded event",
+    ?SLOG(notice, #{msg => "bridge_discarded_event",
         name => Name, type => Type, state_name => StateName,
         content => Content}),
     {keep_state, State}.
@@ -349,7 +349,7 @@ do_connect(#{connect_opts := ConnectOpts,
             {ok, State#{connection => Conn}};
         {error, Reason} ->
             ConnectOpts1 = obfuscate(ConnectOpts),
-            ?SLOG(error, #{msg => "Failed to connect",
+            ?SLOG(error, #{msg => "failed_to_connect",
                 config => ConnectOpts1, reason => Reason}),
             {error, Reason, State}
     end.
@@ -386,8 +386,8 @@ pop_and_send_loop(#{replayq := Q} = State, N) ->
     end.
 
 do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) ->
-    ?SLOG(error, #{msg => "cannot forward messages to remote broker"
-                          " as 'egress' is not configured",
+    ?SLOG(error, #{msg => "cannot_forward_messages_to_remote_broker"
+                          "_as_'egress'_is_not_configured",
                    messages => Msg});
 do_send(#{inflight := Inflight,
           connection := Connection,
@@ -398,7 +398,7 @@ do_send(#{inflight := Inflight,
                     emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'),
                     emqx_connector_mqtt_msg:to_remote_msg(Message, Vars)
                 end,
-    ?SLOG(debug, #{msg => "publish to remote broker",
+    ?SLOG(debug, #{msg => "publish_to_remote_broker",
         message => Msg, vars => Vars}),
     case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(Msg)]) of
         {ok, Refs} ->

From 04313dc044f7d9ba4d32bac2ba4ca1f9973154c2 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 29 Dec 2021 19:03:54 +0800
Subject: [PATCH 5/7] fix(trace): download trace api not working

---
 .../src/emqx_dashboard_swagger.erl            |  3 ++
 .../src/emqx_mgmt_api_trace.erl               | 26 +++++++---
 .../etc/emqx_modules.conf.rendered            | 50 -------------------
 rebar.config                                  |  2 +-
 4 files changed, 24 insertions(+), 57 deletions(-)
 delete mode 100644 apps/emqx_modules/etc/emqx_modules.conf.rendered

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
index 9a54be9c5..bfc6ae951 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
@@ -312,6 +312,9 @@ responses(Responses, Module) ->
 
 response(Status, Bin, {Acc, RefsAcc, Module}) when is_binary(Bin) ->
     {Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module};
+%% Support swagger raw object(file download).
+response(Status, #{content := _} = Content, {Acc, RefsAcc, Module}) ->
+    {Acc#{integer_to_binary(Status) => Content}, RefsAcc, Module};
 response(Status, ?REF(StructName), {Acc, RefsAcc, Module}) ->
     response(Status, ?R_REF(Module, StructName), {Acc, RefsAcc, Module});
 response(Status, ?R_REF(_Mod, _Name) = RRef, {Acc, RefsAcc, Module}) ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
index d6902d123..15277fa8e 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
@@ -107,9 +107,14 @@ schema("/trace/:name/download") ->
         get => #{
             description => "Download trace log by name",
             parameters => [hoconsc:ref(name)],
-            %% todo zip file octet-stream
             responses => #{
-                200 => <<"TODO octet-stream">>
+                200 =>
+                #{description => "A trace zip file",
+                    content => #{
+                        'application/octet-stream' =>
+                             #{schema => #{type => "string", format => "binary"}}
+                    }
+                }
             }
         }
     };
@@ -126,7 +131,11 @@ schema("/trace/:name/log") ->
             ],
             %% todo response data
             responses => #{
-                200 => <<"TODO">>
+                200 =>
+                [
+                    {items, hoconsc:mk(binary(), #{example => "BinBinBin"})}
+                    | fields(bytes) ++ fields(position)
+                ]
             }
         }
     }.
@@ -209,6 +218,7 @@ fields(position) ->
             default => 0
         })}].
 
+
 -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$").
 
 validate_name(Name) ->
@@ -296,7 +306,12 @@ download_trace_log(get, #{bindings := #{name := Name}}) ->
             ZipFileName = ZipDir ++ binary_to_list(Name) ++ ".zip",
             {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]),
             emqx_trace:delete_files_after_send(ZipFileName, Zips),
-            {200, ZipFile};
+            Headers = #{
+                <<"content-type">> => <<"application/octet-stream">>,
+                <<"content-disposition">> =>
+                iolist_to_binary("attachment; filename=" ++ filename:basename(ZipFile))
+            },
+            {200, Headers, {file, ZipFile}};
         {error, not_found} -> ?NOT_FOUND(Name)
     end.
 
@@ -324,11 +339,10 @@ cluster_call(Mod, Fun, Args, Timeout) ->
     BadNodes =/= [] andalso ?LOG(error, "rpc call failed on ~p ~p", [BadNodes, {Mod, Fun, Args}]),
     GoodRes.
 
-stream_log_file(get, #{bindings := #{name := Name}, query_string := Query} = T) ->
+stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) ->
     Node0 = maps:get(<<"node">>, Query, atom_to_binary(node())),
     Position = maps:get(<<"position">>, Query, 0),
     Bytes = maps:get(<<"bytes">>, Query, 1000),
-    logger:error("~p", [T]),
     case to_node(Node0) of
         {ok, Node} ->
             case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of
diff --git a/apps/emqx_modules/etc/emqx_modules.conf.rendered b/apps/emqx_modules/etc/emqx_modules.conf.rendered
deleted file mode 100644
index be34479e8..000000000
--- a/apps/emqx_modules/etc/emqx_modules.conf.rendered
+++ /dev/null
@@ -1,50 +0,0 @@
-
-delayed {
-    enable = true
-    ## 0 is no limit
-    max_delayed_messages = 0
-}
-
-observer_cli {
-    enable = true
-}
-
-telemetry {
-    enable = true
-}
-
-event_message {
-    "$event/client_connected" = true
-    "$event/client_disconnected" = true
-    # "$event/client_subscribed": false
-    # "$event/client_unsubscribed": false
-    # "$event/message_delivered": false
-    # "$event/message_acked": false
-    # "$event/message_dropped": false
-}
-
-topic_metrics: [
-    #{topic: "test/1"}
-]
-
-rewrite: [
-    # {
-    #     action = publish
-    #     source_topic = "x/#"
-    #     re = "^x/y/(.+)$"
-    #     dest_topic = "z/y/$1"
-    # },
-    # {
-    #     action = subscribe
-    #     source_topic = "x1/#"
-    #     re = "^x1/y/(.+)$"
-    #     dest_topic = "z1/y/$1"
-    # },
-    # {
-    #     action = all
-    #     source_topic = "x2/#"
-    #     re = "^x2/y/(.+)$"
-    #     dest_topic = "z2/y/$1"
-    # }
-]
-
diff --git a/rebar.config b/rebar.config
index 1894d817d..14fe8eb65 100644
--- a/rebar.config
+++ b/rebar.config
@@ -55,7 +55,7 @@
     , {mria, {git, "https://github.com/emqx/mria", {tag, "0.1.5"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
-    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}}
+    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.9"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}}
     , {replayq, "0.3.3"}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}

From b8bb5ff738704ace0f5c6266e1f74b819c10fe53 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 29 Dec 2021 22:33:34 +0800
Subject: [PATCH 6/7] fix(trace): delete duplicate topic from msg

---
 apps/emqx/etc/emqx.conf.rendered              | 1671 -----------------
 apps/emqx/include/logger.hrl                  |    6 +-
 apps/emqx/src/emqx_channel.erl                |    2 -
 apps/emqx/src/emqx_cm.erl                     |    3 +-
 apps/emqx/src/emqx_flapping.erl               |    2 -
 .../src/emqx_trace/emqx_trace_handler.erl     |   15 -
 6 files changed, 4 insertions(+), 1695 deletions(-)
 delete mode 100644 apps/emqx/etc/emqx.conf.rendered

diff --git a/apps/emqx/etc/emqx.conf.rendered b/apps/emqx/etc/emqx.conf.rendered
deleted file mode 100644
index afa640621..000000000
--- a/apps/emqx/etc/emqx.conf.rendered
+++ /dev/null
@@ -1,1671 +0,0 @@
-##==================================================================
-## Listeners
-##==================================================================
-## MQTT/TCP - TCP Listeners for MQTT Protocol
-## syntax: listeners.tcp.<name>
-## example: listeners.tcp.my_tcp_listener
-listeners.tcp.default {
-  ## The IP address and port that the listener will bind.
-  ##
-  ## @doc listeners.tcp.<name>.bind
-  ## ValueType: IPAddress | Port | IPAddrPort
-  ## Required: true
-  ## Examples: 1883, 127.0.0.1:1883, ::1:1883
-  bind = "0.0.0.0:1883"
-
-  ## The configuration zone this listener is using.
-  ## If not set, the global configs are used for this listener.
-  ##
-  ## See `zones.<name>` for more details.
-  ##
-  ## @doc listeners.tcp.<name>.zone
-  ## ValueType: String
-  ## Required: false
-  #zone = default
-
-  ## The size of the acceptor pool for this listener.
-  ##
-  ## @doc listeners.tcp.<name>.acceptors
-  ## ValueType: Number
-  ## Default: 16
-  acceptors = 16
-
-  ## Maximum number of concurrent connections.
-  ##
-  ## @doc listeners.tcp.<name>.max_connections
-  ## ValueType: Number | infinity
-  ## Default: infinity
-  max_connections = 1024000
-
-  ## The access control rules for this listener.
-  ##
-  ## See: https://github.com/emqtt/esockd#allowdeny
-  ##
-  ## @doc listeners.tcp.<name>.access_rules
-  ## ValueType: Array<AccessRules>
-  ## Default: []
-  ## Examples:
-  ##   access_rules: [
-  ##     "deny 192.168.0.0/24",
-  ##     "all all"
-  ##   ]
-  access_rules = [
-    "allow all"
-  ]
-
-  ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed
-  ## behind HAProxy or Nginx.
-  ##
-  ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/
-  ##
-  ## @doc listeners.tcp.<name>.proxy_protocol
-  ## ValueType: Boolean
-  ## Default: false
-  proxy_protocol = false
-
-  ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection
-  ## if no proxy protocol packet received within the timeout.
-  ##
-  ## @doc listeners.tcp.<name>.proxy_protocol_timeout
-  ## ValueType: Duration
-  ## Default: 3s
-  proxy_protocol_timeout = 3s
-
-  ## When publishing or subscribing, prefix all topics with a mountpoint string.
-  ## The prefixed string will be removed from the topic name when the message
-  ## is delivered to the subscriber. The mountpoint is a way that users can use
-  ## to implement isolation of message routing between different listeners.
-  ##
-  ## For example if a clientA subscribes to "t" with `listeners.tcp.<name>.mountpoint`
-  ## set to "some_tenant", then the client accually subscribes to the topic
-  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
-  ## with the clientA) send a message to topic "t", the message is accually route
-  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
-  ## message, with topic name "t".
-  ##
-  ## Set to "" to disable the feature.
-  ##
-  ## Variables in mountpoint string:
-  ##  - ${clientid}: clientid
-  ##  - ${username}: username
-  ##
-  ## @doc listeners.tcp.<name>.mountpoint
-  ## ValueType: String
-  ## Default: ""
-  mountpoint = ""
-
-  ## TCP options
-  ## See ${example_common_tcp_options} for more information
-  tcp.backlog = 1024
-  tcp.buffer = 4KB
-}
-
-## MQTT/SSL - SSL Listeners for MQTT Protocol
-## syntax: listeners.ssl.<name>
-## example: listeners.ssl.my_ssl_listener
-listeners.ssl.default {
-  ## The IP address and port that the listener will bind.
-  ##
-  ## @doc listeners.ssl.<name>.bind
-  ## ValueType: IPAddress | Port | IPAddrPort
-  ## Required: true
-  ## Examples: 8883, 127.0.0.1:8883, ::1:8883
-  bind = "0.0.0.0:8883"
-
-  ## The configuration zone this listener is using.
-  ## If not set, the global configs are used for this listener.
-  ##
-  ## See `zones.<name>` for more details.
-  ##
-  ## @doc listeners.ssl.<name>.zone
-  ## ValueType: String
-  ## Required: false
-  #zone = default
-
-  ## The size of the acceptor pool for this listener.
-  ##
-  ## @doc listeners.ssl.<name>.acceptors
-  ## ValueType: Number
-  ## Default: 16
-  acceptors = 16
-
-  ## Maximum number of concurrent connections.
-  ##
-  ## @doc listeners.ssl.<name>.max_connections
-  ## ValueType: Number | infinity
-  ## Default: infinity
-  max_connections = 512000
-
-  ## The access control rules for this listener.
-  ##
-  ## See: https://github.com/emqtt/esockd#allowdeny
-  ##
-  ## @doc listeners.ssl.<name>.access_rules
-  ## ValueType: Array<AccessRules>
-  ## Default: []
-  ## Examples:
-  ##   access_rules: [
-  ##     "deny 192.168.0.0/24",
-  ##     "all all"
-  ##   ]
-  access_rules = [
-    "allow all"
-  ]
-
-  ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed
-  ## behind HAProxy or Nginx.
-  ##
-  ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/
-  ##
-  ## @doc listeners.ssl.<name>.proxy_protocol
-  ## ValueType: Boolean
-  ## Default: true
-  proxy_protocol = false
-
-  ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection
-  ## if no proxy protocol packet received within the timeout.
-  ##
-  ## @doc listeners.ssl.<name>.proxy_protocol_timeout
-  ## ValueType: Duration
-  ## Default: 3s
-  proxy_protocol_timeout = 3s
-
-  ## When publishing or subscribing, prefix all topics with a mountpoint string.
-  ## The prefixed string will be removed from the topic name when the message
-  ## is delivered to the subscriber. The mountpoint is a way that users can use
-  ## to implement isolation of message routing between different listeners.
-  ##
-  ## For example if a clientA subscribes to "t" with `listeners.ssl.<name>.mountpoint`
-  ## set to "some_tenant", then the client accually subscribes to the topic
-  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
-  ## with the clientA) send a message to topic "t", the message is accually route
-  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
-  ## message, with topic name "t".
-  ##
-  ## Set to "" to disable the feature.
-  ##
-  ## Variables in mountpoint string:
-  ##  - ${clientid}: clientid
-  ##  - ${username}: username
-  ##
-  ## @doc listeners.ssl.<name>.mountpoint
-  ## ValueType: String
-  ## Default: ""
-  mountpoint = ""
-
-  ## SSL options
-  ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
-  ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
-  ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem"
-
-  # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
-  # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256"
-  # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA"
-  # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA"
-  # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config
-  # ssl.ciphers = ""
-
-  ## TCP options
-  ## See ${example_common_tcp_options} for more information
-  tcp.backlog = 1024
-  tcp.buffer = 4KB
-}
-
-## MQTT/QUIC - QUIC Listeners for MQTT Protocol
-## syntax: listeners.quic.<name>
-## example: listeners.quic.my_quic_listener
-listeners.quic.default {
-  ## The IP address and port that the listener will bind.
-  ##
-  ## @doc listeners.quic.<name>.bind
-  ## ValueType: IPAddress | Port | IPAddrPort
-  ## Required: true
-  ## Examples: 14567, 127.0.0.1:14567, ::1:14567
-  bind = "0.0.0.0:14567"
-
-  ## The configuration zone this listener is using.
-  ## If not set, the global configs are used for this listener.
-  ##
-  ## See `zones.<name>` for more details.
-  ## NOTE: This is a cluster-wide configuration.
-  ## It requires all nodes to be stopped before changing it.
-  ##
-  ## @doc listeners.quic.<name>.zone
-  ## ValueType: String
-  ## Required: false
-  #zone = default
-
-  ## The size of the acceptor pool for this listener.
-  ##
-  ## @doc listeners.quic.<name>.acceptors
-  ## ValueType: Number
-  ## Default: 16
-  acceptors = 16
-
-  ## Maximum number of concurrent connections.
-  ##
-  ## @doc listeners.quic.<name>.max_connections
-  ## ValueType: Number | infinity
-  ## Default: infinity
-  max_connections = 1024000
-
-  ## Path to the file containing the user's private PEM-encoded key.
-  ##
-  ## @doc listeners.quic.<name>.keyfile
-  ## ValueType: String
-  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
-  keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
-
-  ## Path to a file containing the user certificate.
-  ##
-  ## @doc listeners.quic.<name>.certfile
-  ## ValueType: String
-  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
-  certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
-
-  ## When publishing or subscribing, prefix all topics with a mountpoint string.
-  ## The prefixed string will be removed from the topic name when the message
-  ## is delivered to the subscriber. The mountpoint is a way that users can use
-  ## to implement isolation of message routing between different listeners.
-  ##
-  ## For example if a clientA subscribes to "t" with `listeners.quic.<name>.mountpoint`
-  ## set to "some_tenant", then the client accually subscribes to the topic
-  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
-  ## with the clientA) send a message to topic "t", the message is accually route
-  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
-  ## message, with topic name "t".
-  ##
-  ## Set to "" to disable the feature.
-  ##
-  ## Variables in mountpoint string:
-  ##  - ${clientid}: clientid
-  ##  - ${username}: username
-  ##
-  ## @doc listeners.quic.<name>.mountpoint
-  ## ValueType: String
-  ## Default: ""
-  mountpoint = ""
-}
-
-## MQTT/WS - Websocket Listeners for MQTT Protocol
-## syntax: listeners.ws.<name>
-## example: listeners.ws.my_ws_listener
-listeners.ws.default {
-  ## The IP address and port that the listener will bind.
-  ##
-  ## @doc listeners.ws.<name>.bind
-  ## ValueType: IPAddress | Port | IPAddrPort
-  ## Required: true
-  ## Examples: 8083, 127.0.0.1:8083, ::1:8083
-  bind = "0.0.0.0:8083"
-
-  ## The configuration zone this listener is using.
-  ## If not set, the global configs are used for this listener.
-  ##
-  ## See `zones.<name>` for more details.
-  ##
-  ## @doc listeners.ws.<name>.zone
-  ## ValueType: String
-  ## Required: false
-  #zone = default
-
-  ## The size of the acceptor pool for this listener.
-  ##
-  ## @doc listeners.ws.<name>.acceptors
-  ## ValueType: Number
-  ## Default: 16
-  acceptors = 16
-
-  ## Maximum number of concurrent connections.
-  ##
-  ## @doc listeners.ws.<name>.max_connections
-  ## ValueType: Number | infinity
-  ## Default: infinity
-  max_connections = 1024000
-
-  ## The access control rules for this listener.
-  ##
-  ## See: https://github.com/emqtt/esockd#allowdeny
-  ##
-  ## @doc listeners.ws.<name>.access_rules
-  ## ValueType: Array<AccessRules>
-  ## Default: []
-  ## Examples:
-  ##   access_rules: [
-  ##     "deny 192.168.0.0/24",
-  ##     "all all"
-  ##   ]
-  access_rules = [
-    "allow all"
-  ]
-
-  ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed
-  ## behind HAProxy or Nginx.
-  ##
-  ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/
-  ##
-  ## @doc listeners.ws.<name>.proxy_protocol
-  ## ValueType: Boolean
-  ## Default: true
-  proxy_protocol = false
-
-  ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection
-  ## if no proxy protocol packet received within the timeout.
-  ##
-  ## @doc listeners.ws.<name>.proxy_protocol_timeout
-  ## ValueType: Duration
-  ## Default: 3s
-  proxy_protocol_timeout = 3s
-
-  ## When publishing or subscribing, prefix all topics with a mountpoint string.
-  ## The prefixed string will be removed from the topic name when the message
-  ## is delivered to the subscriber. The mountpoint is a way that users can use
-  ## to implement isolation of message routing between different listeners.
-  ##
-  ## For example if a clientA subscribes to "t" with `listeners.ws.<name>.mountpoint`
-  ## set to "some_tenant", then the client accually subscribes to the topic
-  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
-  ## with the clientA) send a message to topic "t", the message is accually route
-  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
-  ## message, with topic name "t".
-  ##
-  ## Set to "" to disable the feature.
-  ##
-  ## Variables in mountpoint string:
-  ##  - ${clientid}: clientid
-  ##  - ${username}: username
-  ##
-  ## @doc listeners.ws.<name>.mountpoint
-  ## ValueType: String
-  ## Default: ""
-  mountpoint = ""
-
-  ## TCP options
-  ## See ${example_common_tcp_options} for more information
-  tcp.backlog = 1024
-  tcp.buffer = 4KB
-
-  ## Websocket options
-  ## See ${example_common_websocket_options} for more information
-  websocket.idle_timeout = 86400s
-}
-
-## MQTT/WSS - WebSocket Secure Listeners for MQTT Protocol
-## syntax: listeners.wss.<name>
-## example: listeners.wss.my_wss_listener
-listeners.wss.default {
-  ## The IP address and port that the listener will bind.
-  ##
-  ## @doc listeners.wss.<name>.bind
-  ## ValueType: IPAddress | Port | IPAddrPort
-  ## Required: true
-  ## Examples: 8084, 127.0.0.1:8084, ::1:8084
-  bind = "0.0.0.0:8084"
-
-  ## The configuration zone this listener is using.
-  ## If not set, the global configs are used for this listener.
-  ##
-  ## See `zones.<name>` for more details.
-  ##
-  ## @doc listeners.wss.<name>.zone
-  ## ValueType: String
-  ## Required: false
-  #zone = default
-
-  ## The size of the acceptor pool for this listener.
-  ##
-  ## @doc listeners.wss.<name>.acceptors
-  ## ValueType: Number
-  ## Default: 16
-  acceptors = 16
-
-  ## Maximum number of concurrent connections.
-  ##
-  ## @doc listeners.wss.<name>.max_connections
-  ## ValueType: Number | infinity
-  ## Default: infinity
-  max_connections = 512000
-
-  ## The access control rules for this listener.
-  ##
-  ## See: https://github.com/emqtt/esockd#allowdeny
-  ##
-  ## @doc listeners.wss.<name>.access_rules
-  ## ValueType: Array<AccessRules>
-  ## Default: []
-  ## Examples:
-  ##   access_rules: [
-  ##     "deny 192.168.0.0/24",
-  ##     "all all"
-  ##   ]
-  access_rules = [
-    "allow all"
-  ]
-
-  ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed
-  ## behind HAProxy or Nginx.
-  ##
-  ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/
-  ##
-  ## @doc listeners.wss.<name>.proxy_protocol
-  ## ValueType: Boolean
-  ## Default: true
-  proxy_protocol = false
-
-  ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection
-  ## if no proxy protocol packet received within the timeout.
-  ##
-  ## @doc listeners.wss.<name>.proxy_protocol_timeout
-  ## ValueType: Duration
-  ## Default: 3s
-  proxy_protocol_timeout = 3s
-
-  ## When publishing or subscribing, prefix all topics with a mountpoint string.
-  ## The prefixed string will be removed from the topic name when the message
-  ## is delivered to the subscriber. The mountpoint is a way that users can use
-  ## to implement isolation of message routing between different listeners.
-  ##
-  ## For example if a clientA subscribes to "t" with `listeners.wss.<name>.mountpoint`
-  ## set to "some_tenant", then the client accually subscribes to the topic
-  ## "some_tenant/t". Similarly if another clientB (connected to the same listener
-  ## with the clientA) send a message to topic "t", the message is accually route
-  ## to all the clients subscribed "some_tenant/t", so clientA will receive the
-  ## message, with topic name "t".
-  ##
-  ## Set to "" to disable the feature.
-  ##
-  ## Variables in mountpoint string:
-  ##  - ${clientid}: clientid
-  ##  - ${username}: username
-  ##
-  ## @doc listeners.wss.<name>.mountpoint
-  ## ValueType: String
-  ## Default: ""
-  mountpoint = ""
-
-  ## SSL options
-  ## See ${example_common_ssl_options} for more information
-  ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
-  ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
-  ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem"
-
-  ## TCP options
-  ## See ${example_common_tcp_options} for more information
-  tcp.backlog = 1024
-  tcp.buffer = 4KB
-
-  ## Websocket options
-  ## See ${example_common_websocket_options} for more information
-  websocket.idle_timeout = 86400s
-
-}
-
-## Enable per connection statistics.
-##
-## @doc stats.enable
-## ValueType: Boolean
-## Default: true
-stats.enable = true
-
-authorization {
-  ## Behaviour after not matching a rule.
-  ##
-  ## @doc authorization.no_match
-  ## ValueType: allow | deny
-  ## Default: allow
-  no_match: allow
-
-  ## The action when authorization check reject current operation
-  ##
-  ## @doc authorization.deny_action
-  ## ValueType: ignore | disconnect
-  ## Default: ignore
-  deny_action: ignore
-
-  ## Whether to enable Authorization cache.
-  ##
-  ## If enabled, Authorization roles for each client will be cached in the memory
-  ##
-  ## @doc authorization.cache.enable
-  ## ValueType: Boolean
-  ## Default: true
-  cache.enable: true
-
-  ## The maximum count of Authorization entries can be cached for a client.
-  ##
-  ## @doc authorization.cache.max_size
-  ## ValueType: Integer
-  ## Range: [0, 1048576]
-  ## Default: 32
-  cache.max_size: 32
-
-  ## The time after which an Authorization cache entry will be deleted
-  ##
-  ## @doc authorization.cache.ttl
-  ## ValueType: Duration
-  ## Default: 1m
-  cache.ttl: 1m
-}
-
-mqtt {
-  ## How long time the MQTT connection will be disconnected if the
-  ## TCP connection is established but MQTT CONNECT has not been
-  ## received.
-  ##
-  ## @doc mqtt.idle_timeout
-  ## ValueType: Duration
-  ## Default: 15s
-  idle_timeout = 15s
-
-  ## Maximum MQTT packet size allowed.
-  ##
-  ## @doc mqtt.max_packet_size
-  ## ValueType: Bytes
-  ## Default: 1MB
-  max_packet_size = 1MB
-
-  ## Maximum length of MQTT clientId allowed.
-  ##
-  ## @doc mqtt.max_clientid_len
-  ## ValueType: Integer
-  ## Range: [23, 65535]
-  ## Default: 65535
-  max_clientid_len = 65535
-
-  ## Maximum topic levels allowed.
-  ##
-  ## @doc mqtt.max_topic_levels
-  ## ValueType: Integer
-  ## Range: [1, 65535]
-  ## Default: 128
-  ## Depth so big may lead to subscribing performance issues
-  max_topic_levels = 128
-
-  ## Maximum QoS allowed.
-  ##
-  ## @doc mqtt.max_qos_allowed
-  ## ValueType: 0 | 1 | 2
-  ## Default: 2
-  max_qos_allowed = 2
-
-  ## Maximum Topic Alias, 0 means no topic alias supported.
-  ##
-  ## @doc mqtt.max_topic_alias
-  ## ValueType: Integer
-  ## Range: [0, 65535]
-  ## Default: 65535
-  max_topic_alias = 65535
-
-  ## Whether the Server supports MQTT retained messages.
-  ##
-  ## @doc mqtt.retain_available
-  ## ValueType: Boolean
-  ## Default: true
-  retain_available = true
-
-  ## Whether the Server supports MQTT Wildcard Subscriptions
-  ##
-  ## @doc mqtt.wildcard_subscription
-  ## ValueType: Boolean
-  ## Default: true
-  wildcard_subscription = true
-
-  ## Whether the Server supports MQTT Shared Subscriptions.
-  ##
-  ## @doc mqtt.shared_subscription
-  ## ValueType: Boolean
-  ## Default: true
-  shared_subscription = true
-
-  ## Whether to ignore loop delivery of messages.(for mqtt v3.1.1)
-  ##
-  ## @doc mqtt.ignore_loop_deliver
-  ## ValueType: Boolean
-  ## Default: false
-  ignore_loop_deliver = false
-
-  ## Whether to parse the MQTT frame in strict mode
-  ##
-  ## @doc mqtt.strict_mode
-  ## ValueType: Boolean
-  ## Default: false
-  strict_mode = false
-
-  ## Specify the response information returned to the client
-  ##
-  ## This feature is disabled if is set to ""
-  ##
-  ## @doc mqtt.response_information
-  ## ValueType: String
-  ## Default: ""
-  response_information = ""
-
-  ## Server Keep Alive of MQTT 5.0
-  ##
-  ## @doc mqtt.server_keepalive
-  ## ValueType: Number | disabled
-  ## Default: disabled
-  server_keepalive = disabled
-
-  ## The backoff for MQTT keepalive timeout. The broker will kick a connection out
-  ## until 'Keepalive * backoff * 2' timeout.
-  ##
-  ## @doc mqtt.keepalive_backoff
-  ## ValueType: Float
-  ## Range: (0.5, 1]
-  ## Default: 0.75
-  keepalive_backoff = 0.75
-
-  ## Maximum number of subscriptions allowed.
-  ##
-  ## @doc mqtt.max_subscriptions
-  ## ValueType: Integer | infinity
-  ## Range: [1, infinity)
-  ## Default: infinity
-  max_subscriptions = infinity
-
-  ## Force to upgrade QoS according to subscription.
-  ##
-  ## @doc mqtt.upgrade_qos
-  ## ValueType: Boolean
-  ## Default: false
-  upgrade_qos = false
-
-  ## Maximum size of the Inflight Window storing QoS1/2 messages delivered but unacked.
-  ##
-  ## @doc mqtt.max_inflight
-  ## ValueType: Integer
-  ## Range: [1, 65535]
-  ## Default: 32
-  max_inflight = 32
-
-  ## Retry interval for QoS1/2 message delivering.
-  ##
-  ## @doc mqtt.retry_interval
-  ## ValueType: Duration
-  ## Default: 30s
-  retry_interval = 30s
-
-  ## Maximum QoS2 packets (Client -> Broker) awaiting PUBREL.
-  ##
-  ## @doc mqtt.max_awaiting_rel
-  ## ValueType: Integer | infinity
-  ## Range: [1, infinity)
-  ## Default: 100
-  max_awaiting_rel = 100
-
-  ## The QoS2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout.
-  ##
-  ## @doc mqtt.await_rel_timeout
-  ## ValueType: Duration
-  ## Default: 300s
-  await_rel_timeout = 300s
-
-  ## Default session expiry interval for MQTT V3.1.1 connections.
-  ##
-  ## @doc mqtt.session_expiry_interval
-  ## ValueType: Duration
-  ## Default: 2h
-  session_expiry_interval = 2h
-
-  ## Maximum queue length. Enqueued messages when persistent client disconnected,
-  ## or inflight window is full.
-  ##
-  ## @doc mqtt.max_mqueue_len
-  ## ValueType: Integer | infinity
-  ## Range: [0, infinity)
-  ## Default: 1000
-  max_mqueue_len = 1000
-
-  ## Topic priorities.
-  ##
-  ## There's no priority table by default, hence all messages
-  ## are treated equal.
-  ##
-  ## Priority number [1-255]
-  ##
-  ## NOTE: comma and equal signs are not allowed for priority topic names
-  ## NOTE: Messages for topics not in the priority table are treated as
-  ## either highest or lowest priority depending on the configured
-  ## value for mqtt.mqueue_default_priority
-  ##
-  ## @doc mqtt.mqueue_priorities
-  ## ValueType: Map | disabled
-  ## Examples:
-  ##  To configure "topic/1" > "topic/2":
-  ##    mqueue_priorities: {"topic/1": 10, "topic/2": 8}
-  ## Default: disabled
-  mqueue_priorities = disabled
-
-  ## Default to highest priority for topics not matching priority table
-  ##
-  ## @doc mqtt.mqueue_default_priority
-  ## ValueType: highest | lowest
-  ## Default: lowest
-  mqueue_default_priority = lowest
-
-  ## Whether to enqueue QoS0 messages.
-  ##
-  ## @doc mqtt.mqueue_store_qos0
-  ## ValueType: Boolean
-  ## Default: true
-  mqueue_store_qos0 = true
-
-  ## Whether use username replace client id
-  ##
-  ## @doc mqtt.use_username_as_clientid
-  ## ValueType: Boolean
-  ## Default: false
-  use_username_as_clientid = false
-
-  ## Use the CN, DN or CRT field from the client certificate as a username.
-  ## Only works for SSL connection.
-  ##
-  ## @doc mqtt.peer_cert_as_username
-  ## ValueType: cn | dn | crt | disabled
-  ## Default: disabled
-  peer_cert_as_username = disabled
-
-  ## Use the CN, DN or CRT field from the client certificate as a clientid.
-  ## Only works for SSL connection.
-  ##
-  ## @doc mqtt.peer_cert_as_clientid
-  ## ValueType: cn | dn | crt | disabled
-  ## Default: disabled
-  peer_cert_as_clientid = disabled
-}
-
-flapping_detect {
-  ## Enable Flapping Detection.
-  ##
-  ## This config controls the allowed maximum number of CONNECT received
-  ## from the same clientid in a time frame defined by `window_time`.
-  ## After the limit is reached, successive CONNECT requests are forbidden
-  ## (banned) until the end of the time period defined by `ban_time`.
-  ##
-  ## @doc flapping_detect.enable
-  ## ValueType: Boolean
-  ## Default: true
-  enable = false
-
-  ## The max disconnect allowed of a MQTT Client in `window_time`
-  ##
-  ## @doc flapping_detect.max_count
-  ## ValueType: Integer
-  ## Default: 15
-  max_count = 15
-
-  ## The time window for flapping detect
-  ##
-  ## @doc flapping_detect.window_time
-  ## ValueType: Duration
-  ## Default: 1m
-  window_time = 1m
-
-  ## How long the clientid will be banned
-  ##
-  ## @doc flapping_detect.ban_time
-  ## ValueType: Duration
-  ## Default: 5m
-  ban_time = 5m
-
-}
-
-force_shutdown {
-  ## Enable force_shutdown
-  ##
-  ## @doc force_shutdown.enable
-  ## ValueType: Boolean
-  ## Default: true
-  enable = true
-
-  ## Max message queue length
-  ## @doc force_shutdown.max_message_queue_len
-  ## ValueType: Integer
-  ## Range: (0, infinity)
-  ## Default: 1000
-  max_message_queue_len = 1000
-
-  ## Total heap size
-  ##
-  ## @doc force_shutdown.max_heap_size
-  ## ValueType: Size
-  ## Default: 32MB
-  max_heap_size = 32MB
-}
-
-overload_protection {
- ## React on system overload or not
- ## @doc overload_protection.enable
- ## ValueType: Boolean
- ## Default: false
- enable = false
-
- ## Backoff delay in ms
- ## @doc overload_protection.backoff_delay
- ## ValueType: Integer
- ## Range: (0, infinity)
- ## Default: 1
- backoff_delay = 1
-
- ## Backoff GC enabled
- ## @doc overload_protection.backoff_gc
- ## ValueType: Boolean
- ## Default: false
- backoff_gc = false
-
- ## Backoff hibernation enabled
- ## @doc overload_protection.backoff_hibernation
- ## ValueType: Boolean
- ## Default: true
- backoff_hibernation = true
-
- ## Backoff hibernation enabled
- ## @doc overload_protection.backoff_hibernation
- ## ValueType: Boolean
- ## Default: true
- backoff_new_conn = true
-}
-
-force_gc {
-  ## Force the MQTT connection process GC after this number of
-  ## messages or bytes passed through.
-  ##
-  ## @doc force_gc.enable
-  ## ValueType: Boolean
-  ## Default: true
-  enable = true
-
-  ## GC the process after how many messages received
-  ## @doc force_gc.max_message_queue_len
-  ## ValueType: Integer
-  ## Range: (0, infinity)
-  ## Default: 16000
-  count = 16000
-
-  ## GC the process after how much bytes passed through
-  ##
-  ## @doc force_gc.bytes
-  ## ValueType: Size
-  ## Default: 16MB
-  bytes = 16MB
-}
-
-conn_congestion {
-  ## Whether to alarm the congested connections.
-  ##
-  ## Sometimes the mqtt connection (usually an MQTT subscriber) may
-  ## get "congested" because there're too many packets to sent.
-  ## The socket trys to buffer the packets until the buffer is
-  ## full. If more packets comes after that, the packets will be
-  ## "pending" in a queue and we consider the connection is
-  ## "congested".
-  ##
-  ## Enable this to send an alarm when there's any bytes pending in
-  ## the queue. You could set the `sndbuf` to a larger value if the
-  ## alarm is triggered too often.
-  ##
-  ## The name of the alarm is of format "conn_congestion/<ClientID>/<Username>".
-  ## Where the <ClientID> is the client-id of the congested MQTT connection.
-  ## And the <Username> is the username or "unknown_user" of not provided by the client.
-  ##
-  ## @doc conn_congestion.enable_alarm
-  ## ValueType: Boolean
-  ## Default: true
-  enable_alarm = true
-
-  ## Won't clear the congested alarm in how long time.
-  ## The alarm is cleared only when there're no pending bytes in
-  ## the queue, and also it has been `min_alarm_sustain_duration`
-  ## time since the last time we considered the connection is "congested".
-  ##
-  ## This is to avoid clearing and sending the alarm again too often.
-  ##
-  ## @doc conn_congestion.min_alarm_sustain_duration
-  ## ValueType: Duration
-  ## Default: 1m
-  min_alarm_sustain_duration = 1m
-}
-
-rate_limit {
-  ## Maximum connections per second.
-  ##
-  ## @doc zones.<name>.max_conn_rate
-  ## ValueType: Number | infinity
-  ## Default: 1000
-  ## Examples:
-  ##   max_conn_rate: 1000
-  max_conn_rate = 1000
-
-  ## Message limit for the a external MQTT connection.
-  ##
-  ## @doc rate_limit.conn_messages_in
-  ## ValueType: String | infinity
-  ## Default: infinity
-  ## Examples: 100 messages per 10 seconds.
-  ##   conn_messages_in: "100,10s"
-  conn_messages_in = "100,10s"
-
-  ## Limit the rate of receiving packets for a MQTT connection.
-  ## The rate is counted by bytes of packets per second.
-  ##
-  ## The connection won't accept more messages if the messages come
-  ## faster than the limit.
-  ##
-  ## @doc rate_limit.conn_bytes_in
-  ## ValueType: String | infinity
-  ## Default: infinity
-  ## Examples: 100KB incoming per 10 seconds.
-  ##   conn_bytes_in: "100KB,10s"
-  ##
-  conn_bytes_in = "100KB,10s"
-}
-
-quota {
-  ## Messages quota for the each of external MQTT connection.
-  ## This value consumed by the number of recipient on a message.
-  ##
-  ## @doc quota.conn_messages_routing
-  ## ValueType: String | infinity
-  ## Default: infinity
-  ## Examples: 100 messaegs per 1s:
-  ##   quota.conn_messages_routing: "100,1s"
-  conn_messages_routing = "100,1s"
-
-  ## Messages quota for the all of external MQTT connections.
-  ## This value consumed by the number of recipient on a message.
-  ##
-  ## @doc quota.overall_messages_routing
-  ## ValueType: String | infinity
-  ## Default: infinity
-  ## Examples: 200000 messages per 1s:
-  ##    quota.overall_messages_routing: "200000,1s"
-  ##
-  overall_messages_routing = "200000,1s"
-}
-
-##==================================================================
-## Zones
-##==================================================================
-## A zone contains a set of configurations for listeners.
-##
-## A zone can be used by a listener via `listener.<type>.<name>.zone`.
-##
-## The configs defined in zones will override the global configs with the same key.
-##
-## For example given the following config:
-##
-## ```
-## a {
-##     b: 1, c: 1
-## }
-##
-## zone.my_zone {
-##   a {
-##     b:2
-##   }
-## }
-## ```
-##
-## The global config "a" is overridden by the configs "a" inside the zone "my_zone".
-## If there is a listener uses the zone "my_zone", the value of config "a" will be:
-## `{b:2, c: 1}`.
-## Note that although the default value of `a.c` is `0`, the global value is used.
-## i.e. configs in the zone have no default values. To overridde `a.c` we must configure
-## it explicitly in the zone.
-##
-## All the global configs that can be overridden in zones are:
-##   - `stats.*`
-##   - `mqtt.*`
-##   - `authorization.*`
-##   - `flapping_detect.*`
-##   - `force_shutdown.*`
-##   - `conn_congestion.*`
-##   - `rate_limit.*`
-##   - `quota.*`
-##   - `force_gc.*`
-##
-## syntax: zones.<zone-name>
-## example: zones.my_zone
-zones.default {
-
-}
-
-##==================================================================
-## Broker
-##==================================================================
-broker {
-  ## System interval of publishing $SYS messages.
-  ##
-  ## @doc broker.sys_msg_interval
-  ## ValueType: Duration | disabled
-  ## Default: 1m
-  sys_msg_interval = 1m
-
-  ## System heartbeat interval of publishing following heart beat message:
-  ##  - "$SYS/brokers/<node>/uptime"
-  ##  - "$SYS/brokers/<node>/datetime"
-  ##
-  ## @doc broker.sys_heartbeat_interval
-  ## ValueType: Duration
-  ## Default: 30s | disabled
-  sys_heartbeat_interval = 30s
-
-  ## Session locking strategy in a cluster.
-  ##
-  ## @doc broker.session_locking_strategy
-  ## ValueType: local | one | quorum | all
-  ##   - local: only lock the session locally on the current node
-  ##   - one: select only one remove node to lock the session
-  ##   - quorum: select some nodes to lock the session
-  ##   - all: lock the session on all of the nodes in the cluster
-  ## Default: quorum
-  session_locking_strategy = quorum
-
-  ## Dispatch strategy for shared subscription
-  ##
-  ## @doc broker.shared_subscription_strategy
-  ## ValueType: random | round_robin | sticky | hash
-  ##   - random: dispatch the message to a random selected subscriber
-  ##   - round_robin: select the subscribers in a round-robin manner
-  ##   - sticky: always use the last selected subscriber to dispatch,
-  ##     until the susbcriber disconnected.
-  ##   - hash: select the subscribers by the hash of clientIds
-  ## Default: round_robin
-  shared_subscription_strategy = round_robin
-
-  ## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages
-  ## This should allow messages to be dispatched to a different subscriber in
-  ## the group in case the picked (based on shared_subscription_strategy) one # is offline
-  ##
-  ## @doc broker.shared_dispatch_ack_enabled
-  ## ValueType: Boolean
-  ## Default: false
-  shared_dispatch_ack_enabled = false
-
-  ## Enable batch clean for deleted routes.
-  ##
-  ## @doc broker.route_batch_clean
-  ## ValueType: Boolean
-  ## Default: true
-  route_batch_clean = true
-
-  ## Performance toggle for subscribe/unsubscribe wildcard topic.
-  ## Change this toggle only when there are many wildcard topics.
-  ##
-  ## NOTE: when changing from/to 'global' lock, it requires all
-  ## nodes in the cluster to be stopped before the change.
-  ##
-  ## @doc broker.perf.route_lock_type
-  ## ValueType: key | tab | global
-  ##  - key:   mnesia translational updates with per-key locks. recommended for single node setup.
-  ##  - tab:   mnesia translational updates with table lock. recommended for multi-nodes setup.
-  ##  - global: global lock protected updates. recommended for larger cluster.
-  ## Default: key
-  perf.route_lock_type = key
-
-  ## Enable trie path compaction.
-  ## Enabling it significantly improves wildcard topic subscribe
-  ## rate, if wildcard topics have unique prefixes like:
-  ## 'sensor//+/', where ID is unique per subscriber.
-  ##
-  ## Topic match performance (when publishing) may degrade if messages
-  ## are mostly published to topics with large number of levels.
-  ##
-  ## NOTE: This is a cluster-wide configuration.
-  ## It requires all nodes to be stopped before changing it.
-  ##
-  ## @doc broker.perf.trie_compaction
-  ## ValueType: Boolean
-  ## Default: true
-  perf.trie_compaction = true
-}
-
-##==================================================================
-## System Monitor
-##==================================================================
-sysmon {
-  ## The time interval for the periodic process limit check
-  ##
-  ## @doc sysmon.vm.process_check_interval
-  ## ValueType: Duration
-  ## Default: 30s
-  vm.process_check_interval = 30s
-
-  ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is set.
-  ##
-  ## @doc sysmon.vm.process_high_watermark
-  ## ValueType: Percentage
-  ## Default: 80%
-  vm.process_high_watermark = 80%
-
-  ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is clear.
-  ##
-  ## @doc sysmon.vm.process_low_watermark
-  ## ValueType: Percentage
-  ## Default: 60%
-  vm.process_low_watermark = 60%
-
-  ## Enable Long GC monitoring.
-  ## Notice: don't enable the monitor in production for:
-  ## https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421
-  ##
-  ## @doc sysmon.vm.long_gc
-  ## ValueType: Duration | disabled
-  ## Default: disabled
-  vm.long_gc = disabled
-
-  ## Enable Long Schedule(ms) monitoring.
-  ##
-  ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2
-  ##
-  ## @doc sysmon.vm.long_schedule
-  ## ValueType: Duration | disabled
-  ## Default: disabled
-  vm.long_schedule = 240ms
-
-  ## Enable Large Heap monitoring.
-  ##
-  ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2
-  ##
-  ## @doc sysmon.vm.large_heap
-  ## ValueType: Size | disabled
-  ## Default: 32MB
-  vm.large_heap = 32MB
-
-  ## Enable Busy Port monitoring.
-  ##
-  ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2
-  ##
-  ## @doc sysmon.vm.busy_port
-  ## ValueType: Boolean
-  ## Default: true
-  vm.busy_port = true
-
-  ## Enable Busy Dist Port monitoring.
-  ##
-  ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2
-  ##
-  ## @doc sysmon.vm.busy_dist_port
-  ## ValueType: Boolean
-  ## Default: true
-  vm.busy_dist_port = true
-
-  ## The time interval for the periodic cpu check
-  ##
-  ## @doc sysmon.os.cpu_check_interval
-  ## ValueType: Duration
-  ## Default: 60s
-  os.cpu_check_interval = 60s
-
-  ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is set.
-  ##
-  ## @doc sysmon.os.cpu_high_watermark
-  ## ValueType: Percentage
-  ## Default: 80%
-  os.cpu_high_watermark = 80%
-
-  ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is clear.
-  ##
-  ## @doc sysmon.os.cpu_low_watermark
-  ## ValueType: Percentage
-  ## Default: 60%
-  os.cpu_low_watermark = 60%
-
-  ## The time interval for the periodic memory check
-  ##
-  ## @doc sysmon.os.mem_check_interval
-  ## ValueType: Duration | disabled
-  ## Default: 60s
-  os.mem_check_interval = 60s
-
-  ## The threshold, as percentage of system memory, for how much system memory can be allocated before the corresponding alarm is set.
-  ##
-  ## @doc sysmon.os.sysmem_high_watermark
-  ## ValueType: Percentage
-  ## Default: 70%
-  os.sysmem_high_watermark = 70%
-
-  ## The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before the corresponding alarm is set.
-  ##
-  ## @doc sysmon.os.procmem_high_watermark
-  ## ValueType: Percentage
-  ## Default: 5%
-  os.procmem_high_watermark = 5%
-}
-
-##==================================================================
-## Alarm
-##==================================================================
-alarm {
-  ## Specifies the actions to take when an alarm is activated
-  ##
-  ## @doc alarm.actions
-  ## ValueType: Array<AlarmAction>
-  ## Default: [log, publish]
-  actions = [log, publish]
-
-  ## The maximum number of deactivated alarms
-  ##
-  ## @doc alarm.size_limit
-  ## ValueType: Integer
-  ## Default: 1000
-  size_limit = 1000
-
-  ## Validity Period of deactivated alarms
-  ##
-  ## @doc alarm.validity_period
-  ## ValueType: Duration
-  ## Default: 24h
-  validity_period = 24h
-}
-
-## Config references for listeners
-
-## Socket options for TCP connections
-## See: http://erlang.org/doc/man/inet.html
-example_common_tcp_options {
-  ## Specify the {active, N} option for this Socket.
-  ##
-  ## See: https://erlang.org/doc/man/inet.html#setopts-2
-  ##
-  ## @doc listeners.<name>.tcp.active_n
-  ## ValueType: Number
-  ## Default: 100
-  tcp.active_n = 100
-
-  ## TCP backlog defines the maximum length that the queue of
-  ## pending connections can grow to.
-  ##
-  ## @doc listeners.<name>.tcp.backlog
-  ## ValueType: Number
-  ## Range: [0, 1048576]
-  ## Default: 1024
-  tcp.backlog = 1024
-
-  ## The TCP send timeout for the connections.
-  ##
-  ## @doc listeners.<name>.tcp.send_timeout
-  ## ValueType: Duration
-  ## Default: 15s
-  tcp.send_timeout = 15s
-
-  ## Close the connection if send timeout.
-  ##
-  ## @doc listeners.<name>.tcp.send_timeout_close
-  ## ValueType: Boolean
-  ## Default: true
-  tcp.send_timeout_close = true
-
-  ## The TCP receive buffer(os kernel) for the connections.
-  ##
-  ## @doc listeners.<name>.tcp.recbuf
-  ## ValueType: Size
-  ## Default: notset
-  #tcp.recbuf: 2KB
-
-  ## The TCP send buffer(os kernel) for the connections.
-  ##
-  ## @doc listeners.<name>.tcp.sndbuf
-  ## ValueType: Size
-  ## Default: notset
-  #tcp.sndbuf: 4KB
-
-  ## The size of the user-level software buffer used by the driver.
-  ##
-  ## @doc listeners.<name>.tcp.buffer
-  ## ValueType: Size
-  ## Default: notset
-  #tcp.buffer: 4KB
-
-  ## The socket is set to a busy state when the amount of data queued internally
-  ## by the ERTS socket implementation reaches this limit.
-  ##
-  ## @doc listeners.<name>.tcp.high_watermark
-  ## ValueType: Size
-  ## Default: 1MB
-  tcp.high_watermark = 1MB
-
-  ## The TCP_NODELAY flag for the connections.
-  ##
-  ## @doc listeners.<name>.tcp.nodelay
-  ## ValueType: Boolean
-  ## Default: false
-  tcp.nodelay = false
-
-  ## The SO_REUSEADDR flag for the connections.
-  ##
-  ## @doc listeners.<name>.tcp.reuseaddr
-  ## ValueType: Boolean
-  ## Default: true
-  tcp.reuseaddr = true
-}
-
-## Socket options for SSL connections
-## See: http://erlang.org/doc/man/ssl.html
-example_common_ssl_options {
-
-  ## A performance optimization setting, it allows clients to reuse
-  ## pre-existing sessions, instead of initializing new ones.
-  ## Read more about it here.
-  ##
-  ## @doc listeners.<name>.ssl.reuse_sessions
-  ## ValueType: Boolean
-  ## Default: true
-  ssl.reuse_sessions = true
-
-  ## SSL parameter renegotiation is a feature that allows a client and a server
-  ## to renegotiate the parameters of the SSL connection on the fly.
-  ## RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation,
-  ## you drop support for the insecure renegotiation, prone to MitM attacks.
-  ##
-  ## @doc listeners.<name>.ssl.secure_renegotiate
-  ## ValueType: Boolean
-  ## Default: true
-  ssl.secure_renegotiate = true
-
-  ## In protocols that support client-initiated renegotiation,
-  ## the cost of resources of such an operation is higher for the server than the client.
-  ## This can act as a vector for denial of service attacks.
-  ## The SSL application already takes measures to counter-act such attempts,
-  ## but client-initiated renegotiation can be strictly disabled by setting this option to false.
-  ## The default value is true. Note that disabling renegotiation can result in
-  ## long-lived connections becoming unusable due to limits on
-  ## the number of messages the underlying cipher suite can encipher.
-  ssl.client_renegotiation = true
-
-  ## An important security setting, it forces the cipher to be set based
-  ## on the server-specified order instead of the client-specified order,
-  ## hence enforcing the (usually more properly configured) security
-  ## ordering of the server administrator.
-  ##
-  ## @doc listeners.<name>.ssl.honor_cipher_order
-  ## ValueType: Boolean
-  ## Default: true
-  ssl.honor_cipher_order = true
-
-  # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
-  # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256"
-  # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA"
-  # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA"
-  # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config
-  # NOTE: by default, ALL ciphers are enabled
-  # ssl.ciphers = ""
-
-  ## TLS Handshake timeout.
-  ##
-  ## @doc listeners.<name>.ssl.handshake_timeout
-  ## ValueType: Duration
-  ## Default: 15s
-  ssl.handshake_timeout = 15s
-
-  ## Maximum number of non-self-issued intermediate certificates that
-  ## can follow the peer certificate in a valid certification path.
-  ##
-  ## @doc listeners.<name>.ssl.depth
-  ## ValueType: Integer
-  ## Default: 10
-  ssl.depth = 10
-
-  ## Path to the file containing the user's private PEM-encoded key.
-  ##
-  ## @doc listeners.<name>.ssl.keyfile
-  ## ValueType: File
-  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
-  ssl.keyfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/key.pem"
-
-  ## Path to a file containing the user certificate.
-  ##
-  ## @doc listeners.<name>.ssl.certfile
-  ## ValueType: File
-  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
-  ssl.certfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cert.pem"
-
-  ## Path to the file containing PEM-encoded CA certificates. The CA certificates
-  ## are used during server authentication and when building the client certificate chain.
-  ##
-  ## @doc listeners.<name>.ssl.cacertfile
-  ## ValueType: File
-  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem"
-  ssl.cacertfile = "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/cacert.pem"
-
-  ## Maximum number of non-self-issued intermediate certificates that
-  ## can follow the peer certificate in a valid certification path.
-  ##
-  ## @doc listeners.<name>.ssl.depth
-  ## ValueType: Number
-  ## Default: 10
-  ssl.depth = 10
-
-  ## String containing the user's password. Only used if the private keyfile
-  ## is password-protected.
-  ##
-  ## See: listener.ssl.$name.key_password
-  ##
-  ## @doc listeners.<name>.ssl.depth
-  ## ValueType: String
-  ## Default: ""
-  #ssl.key_password: ""
-
-  ## The Ephemeral Diffie-Helman key exchange is a very effective way of
-  ## ensuring Forward Secrecy by exchanging a set of keys that never hit
-  ## the wire. Since the DH key is effectively signed by the private key,
-  ## it needs to be at least as strong as the private key. In addition,
-  ## the default DH groups that most of the OpenSSL installations have
-  ## are only a handful (since they are distributed with the OpenSSL
-  ## package that has been built for the operating system it’s running on)
-  ## and hence predictable (not to mention, 1024 bits only).
-  ## In order to escape this situation, first we need to generate a fresh,
-  ## strong DH group, store it in a file and then use the option above,
-  ## to force our SSL application to use the new DH group. Fortunately,
-  ## OpenSSL provides us with a tool to do that. Simply run:
-  ## openssl dhparam -out dh-params.pem 2048
-  ##
-  ## @doc listeners.<name>.ssl.dhfile
-  ## ValueType: File
-  ## Default: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/dh-params.pem"
-  #ssl.dhfile: "/Users/zhongwen/github/emqx/emqx-v5/apps/emqx/etc/certs/dh-params.pem"
-
-  ## A server only does x509-path validation in mode verify_peer,
-  ## as it then sends a certificate request to the client (this
-  ## message is not sent if the verify option is verify_none).
-  ## You can then also want to specify option fail_if_no_peer_cert.
-  ## More information at: http://erlang.org/doc/man/ssl.html
-  ##
-  ## @doc listeners.<name>.ssl.verify
-  ## ValueType: verify_peer | verify_none
-  ## Default: verify_none
-  ssl.verify = verify_none
-
-  ## Used together with {verify, verify_peer} by an SSL server. If set to true,
-  ## the server fails if the client does not have a certificate to send, that is,
-  ## sends an empty certificate.
-  ##
-  ## @doc listeners.<name>.ssl.fail_if_no_peer_cert
-  ## ValueType: Boolean
-  ## Default: true
-  ssl.fail_if_no_peer_cert = false
-
-}
-
-## Socket options for websocket connections
-example_common_websocket_options {
-  ## The path of WebSocket MQTT endpoint
-  ##
-  ## @doc listeners.<name>.websocket.mqtt_path
-  ## ValueType: Path
-  ## Default: "/mqtt"
-  websocket.mqtt_path = "/mqtt"
-
-  ## Whether a WebSocket message is allowed to contain multiple MQTT packets
-  ##
-  ## @doc listeners.<name>.websocket.mqtt_piggyback
-  ## ValueType: single | multiple
-  ## Default: multiple
-  websocket.mqtt_piggyback = multiple
-
-  ## The compress flag for external WebSocket connections.
-  ##
-  ## If this Value is set true,the websocket message would be compressed
-  ##
-  ## @doc listeners.<name>.websocket.compress
-  ## ValueType: Boolean
-  ## Default: false
-  websocket.compress = false
-
-  ## The idle timeout for external WebSocket connections.
-  ##
-  ## @doc listeners.<name>.websocket.idle_timeout
-  ## ValueType: Duration | infinity
-  ## Default: infinity
-  websocket.idle_timeout = infinity
-
-  ## The max frame size for external WebSocket connections.
-  ##
-  ## @doc listeners.<name>.websocket.max_frame_size
-  ## ValueType: Size
-  ## Default: infinity
-  websocket.max_frame_size = infinity
-
-  ## If set to true, the server fails if the client does not
-  ## have a Sec-WebSocket-Protocol to send.
-  ## Set to false for WeChat MiniApp.
-  ##
-  ## @doc listeners.<name>.websocket.fail_if_no_subprotocol
-  ## ValueType: Boolean
-  ## Default: true
-  websocket.fail_if_no_subprotocol = true
-
-  ## Supported subprotocols
-  ##
-  ## @doc listeners.<name>.websocket.supported_subprotocols
-  ## ValueType: String
-  ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5
-  websocket.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5"
-
-  ## Enable origin check in header for websocket connection
-  ##
-  ## @doc listeners.<name>.websocket.check_origin_enable
-  ## ValueType: Boolean
-  ## Default: false
-  websocket.check_origin_enable = false
-
-  ## Allow origin to be absent in header in websocket connection
-  ## when check_origin_enable is true
-  ##
-  ## @doc listeners.<name>.websocket.allow_origin_absence
-  ## ValueType: Boolean
-  ## Default: true
-  websocket.allow_origin_absence = true
-
-  ## Comma separated list of allowed origin in header for websocket connection
-  ##
-  ## @doc listeners.<name>.websocket.check_origins
-  ## ValueType: String
-  ## Examples:
-  ##  local http dashboard url
-  ## check_origins: "http://localhost:18083, http://127.0.0.1:18083"
-  ## Default: ""
-  websocket.check_origins = "http://localhost:18083, http://127.0.0.1:18083"
-
-  ## Specify which HTTP header for real source IP if the EMQ X cluster is
-  ## deployed behind NGINX or HAProxy.
-  ##
-  ## @doc listeners.<name>.websocket.proxy_address_header
-  ## ValueType: String
-  ## Default: X-Forwarded-For
-  websocket.proxy_address_header = X-Forwarded-For
-
-  ## Specify which HTTP header for real source port if the EMQ X cluster is
-  ## deployed behind NGINX or HAProxy.
-  ##
-  ## @doc listeners.<name>.websocket.proxy_port_header
-  ## ValueType: String
-  ## Default: X-Forwarded-Port
-  websocket.proxy_port_header = X-Forwarded-Port
-
-  websocket.deflate_opts {
-    ## The level of deflate options for external WebSocket connections.
-    ##
-    ## @doc listeners.<name>.websocket.deflate_opts.level
-    ## ValueType: none | default | best_compression | best_speed
-    ## Default: default
-    level = default
-
-    ## The mem_level of deflate options for external WebSocket connections.
-    ##
-    ## @doc listeners.<name>.websocket.deflate_opts.mem_level
-    ## ValueType: Integer
-    ## Range: [1,9]
-    ## Default: 8
-    mem_level = 8
-
-    ## The strategy of deflate options for external WebSocket connections.
-    ##
-    ## @doc listeners.<name>.websocket.deflate_opts.strategy
-    ## ValueType: default | filtered | huffman_only | rle
-    ## Default: default
-    strategy = default
-
-    ## The deflate option for external WebSocket connections.
-    ##
-    ## @doc listeners.<name>.websocket.deflate_opts.server_context_takeover
-    ## ValueType: takeover | no_takeover
-    ## Default: takeover
-    server_context_takeover = takeover
-
-    ## The deflate option for external WebSocket connections.
-    ##
-    ## @doc listeners.<name>.websocket.deflate_opts.client_context_takeover
-    ## ValueType: takeover | no_takeover
-    ## Default: takeover
-    client_context_takeover = takeover
-
-    ## The deflate options for external WebSocket connections.
-    ##
-    ##
-    ## @doc listeners.<name>.websocket.deflate_opts.server_max_window_bits
-    ## ValueType: Integer
-    ## Range: [8,15]
-    ## Default: 15
-    server_max_window_bits = 15
-
-    ## The deflate options for external WebSocket connections.
-    ##
-    ## @doc listeners.<name>.websocket.deflate_opts.client_max_window_bits
-    ## ValueType: Integer
-    ## Range: [8,15]
-    ## Default: 15
-    client_max_window_bits = 15
-  }
-}
-
-persistent_session_store {
-    ## Enable/disable internal persistent session store.
-    ##
-    ## @doc persistent_session_store.enabled
-    ## ValueType: Boolean
-    ## Default: false
-    enabled = false
-
-    ## How long are undelivered messages retained in the store
-    ##
-    ## @doc persistent_session_store.max_retain_undelivered
-    ## ValueType: Duration
-    ## Default: 1h
-    max_retain_undelivered = 1h
-
-    ## The time interval in which to try to run garbage collection of persistent session messages
-    ##
-    ## @doc persistent_session_store.message_gc_interval
-    ## ValueType: Duration
-    ## Default: 1h
-    message_gc_interval = 1h
-
-    ## The time interval in which to try to run garbage collection of persistent session transient data
-    ##
-    ## @doc persistent_session_store.session_message_gc_interval
-    ## ValueType: Duration
-    ## Default: 1m
-    session_message_gc_interval = 1m
-}
diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl
index ddd4349bb..2398a7dee 100644
--- a/apps/emqx/include/logger.hrl
+++ b/apps/emqx/include/logger.hrl
@@ -63,12 +63,12 @@
 
 %% structured logging, meta is for handler's filter.
 -define(SLOG(Level, Data, Meta),
-%% check 'allow' here, only evaluate Data when necessary
+%% check 'allow' here, only evaluate Data and Meta when necessary
     case logger:allow(Level, ?MODULE) of
         true ->
-            logger:log(Level, (Data), Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
+            logger:log(Level, (Data), (Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
                 , line => ?LINE
-            });
+            }));
         false ->
             ok
     end).
diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index d0cb07a68..290299aee 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -550,7 +550,6 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
         {error, Rc = ?RC_NOT_AUTHORIZED, NChannel} ->
             ?SLOG(warning, #{
                 msg => "cannot_publish_to_topic",
-                topic => Topic,
                 reason => emqx_reason_codes:name(Rc)
             }, #{topic => Topic}),
             case emqx:get_config([authorization, deny_action], ignore) of
@@ -568,7 +567,6 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
         {error, Rc = ?RC_QUOTA_EXCEEDED, NChannel} ->
             ?SLOG(warning, #{
                 msg => "cannot_publish_to_topic",
-                topic => Topic,
                 reason => emqx_reason_codes:name(Rc)
             }, #{topic => Topic}),
             case QoS of
diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl
index c44cfe15e..a2101efa1 100644
--- a/apps/emqx/src/emqx_cm.erl
+++ b/apps/emqx/src/emqx_cm.erl
@@ -455,8 +455,7 @@ kick_session(Action, ClientId, ChanPid) ->
 kick_session(ClientId) ->
     case lookup_channels(ClientId) of
         [] ->
-            ?SLOG(warning, #{msg => "kicked_an_unknown_session",
-                             clientid => ClientId},
+            ?SLOG(warning, #{msg => "kicked_an_unknown_session"},
                 #{clientid => unicode:characters_to_list(ClientId, utf8)}),
             ok;
         ChanPids ->
diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl
index cb3da361a..df7c7523f 100644
--- a/apps/emqx/src/emqx_flapping.erl
+++ b/apps/emqx/src/emqx_flapping.erl
@@ -118,7 +118,6 @@ handle_cast({detected, #flapping{clientid   = ClientId,
         true -> %% Flapping happened:(
             ?SLOG(warning, #{
                 msg => "flapping_detected",
-                client_id => ClientId,
                 peer_host => fmt_host(PeerHost),
                 detect_cnt => DetectCnt,
                 wind_time_in_ms => WindTime
@@ -134,7 +133,6 @@ handle_cast({detected, #flapping{clientid   = ClientId,
         false ->
             ?SLOG(warning, #{
                 msg => "client_disconnected",
-                client_id => ClientId,
                 peer_host => fmt_host(PeerHost),
                 detect_cnt => DetectCnt,
                 interval => Interval
diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
index d1fbd20b8..f8a5f3828 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
@@ -35,7 +35,6 @@
         , filter_topic/2
         , filter_ip_address/2
         ]).
--export([template/1]).
 
 -export([handler_id/2]).
 -export([payload_encode/0]).
@@ -158,20 +157,6 @@ formatter(#{type := _Type}) ->
         }
     }.
 
-%% Don't log clientid since clientid only supports exact match, all client ids are the same.
-%% if clientid is not latin characters. the logger_formatter restricts the output must be `~tp`
-%% (actually should use `~ts`), the utf8 characters clientid will become very difficult to read.
-template(clientid) ->
-    [time, " [", level, "] ", {peername, [peername, " "], []}, msg, "\n"];
-template(_) ->
-    [time, " [", level, "] ",
-        {clientid,
-            [{peername, [clientid, "@", peername, " "], [clientid, " "]}],
-            [{peername, [peername, " "], []}]
-        },
-        msg, "\n"
-    ].
-
 filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) ->
     Init = #{id => Id, level => Level, dst => Dst},
     case Filters of

From 489fb7f806b69ff9b59f8310c5da2e8fa93936aa Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Wed, 29 Dec 2021 22:54:38 +0800
Subject: [PATCH 7/7] fix(trace): copy binary:encode_hex/2 from binary.erl

---
 apps/emqx/include/logger.hrl                  | 15 +++-
 apps/emqx/src/emqx_broker.erl                 |  4 +-
 apps/emqx/src/emqx_cm.erl                     | 27 +++---
 apps/emqx/src/emqx_connection.erl             |  2 +-
 apps/emqx/src/emqx_flapping.erl               |  4 +-
 apps/emqx/src/emqx_logger.erl                 | 10 +--
 apps/emqx/src/emqx_logger_textfmt.erl         | 73 +++++++++++++--
 apps/emqx/src/emqx_packet.erl                 | 88 +++++++++++++++++--
 apps/emqx/src/emqx_schema.erl                 | 10 +--
 apps/emqx/src/emqx_trace/emqx_trace.erl       | 47 +++++-----
 .../src/emqx_trace/emqx_trace_formatter.erl   | 21 ++---
 .../src/emqx_trace/emqx_trace_handler.erl     | 11 +--
 apps/emqx/test/emqx_trace_handler_SUITE.erl   | 19 ++--
 apps/emqx_authn/test/emqx_authn_api_SUITE.erl |  6 +-
 apps/emqx_conf/src/emqx_conf_schema.erl       | 11 +--
 .../src/emqx_connector_pgsql.erl              |  3 +-
 .../emqx_management/src/emqx_mgmt_api_app.erl |  2 +-
 .../src/emqx_mgmt_api_trace.erl               |  5 +-
 apps/emqx_management/src/emqx_mgmt_cli.erl    | 38 ++++----
 19 files changed, 265 insertions(+), 131 deletions(-)

diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl
index 2398a7dee..d549e2ccb 100644
--- a/apps/emqx/include/logger.hrl
+++ b/apps/emqx/include/logger.hrl
@@ -67,13 +67,24 @@
     case logger:allow(Level, ?MODULE) of
         true ->
             logger:log(Level, (Data), (Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
-                , line => ?LINE
+                                            , line => ?LINE
             }));
         false ->
             ok
     end).
 
--define(TRACE(Event, Msg, Meta), emqx_trace:log(Event, Msg, Meta)).
+-define(TRACE_FILTER, emqx_trace_filter).
+
+%% Only evaluate when necessary
+-define(TRACE(Event, Msg, Meta),
+    begin
+    case persistent_term:get(?TRACE_FILTER, undefined) of
+        undefined -> ok;
+        [] -> ok;
+        List ->
+           emqx_trace:log(List, Event, Msg, Meta)
+    end
+    end).
 
 %% print to 'user' group leader
 -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl
index 4085b6130..9dbfb0b43 100644
--- a/apps/emqx/src/emqx_broker.erl
+++ b/apps/emqx/src/emqx_broker.erl
@@ -205,8 +205,8 @@ publish(Msg) when is_record(Msg, message) ->
     emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'),
     case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of
         #message{headers = #{allow_publish := false}, topic = Topic} ->
-            Message = emqx_message:to_log_map(Msg),
-            ?TRACE("MQTT", "msg_publish_not_allowed", #{message => Message, topic => Topic}),
+            ?TRACE("MQTT", "msg_publish_not_allowed", #{message => emqx_message:to_log_map(Msg),
+                topic => Topic}),
             [];
         Msg1 = #message{topic = Topic} ->
             emqx_persistent_session:persist_message(Msg1),
diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl
index a2101efa1..eae8dd43d 100644
--- a/apps/emqx/src/emqx_cm.erl
+++ b/apps/emqx/src/emqx_cm.erl
@@ -375,7 +375,7 @@ discard_session(ClientId) when is_binary(ClientId) ->
 -spec kick_or_kill(kick | discard, module(), pid()) -> ok.
 kick_or_kill(Action, ConnMod, Pid) ->
     try
-        %% this is essentailly a gen_server:call implemented in emqx_connection
+        %% this is essentially a gen_server:call implemented in emqx_connection
         %% and emqx_ws_connection.
         %% the handle_call is implemented in emqx_channel
         ok = apply(ConnMod, call, [Pid, Action, ?T_KICK])
@@ -390,19 +390,12 @@ kick_or_kill(Action, ConnMod, Pid) ->
             ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action});
         _ : {timeout, {gen_server, call, _}} ->
             ?tp(warning, "session_kick_timeout",
-                #{pid => Pid,
-                  action => Action,
-                  stale_channel => stale_channel_info(Pid)
-                 }),
+                #{pid => Pid, action => Action, stale_channel => stale_channel_info(Pid)}),
             ok = force_kill(Pid);
         _ : Error : St ->
             ?tp(error, "session_kick_exception",
-                #{pid => Pid,
-                  action => Action,
-                  reason => Error,
-                  stacktrace => St,
-                  stale_channel => stale_channel_info(Pid)
-                 }),
+                #{pid => Pid, action => Action, reason => Error, stacktrace => St,
+                    stale_channel => stale_channel_info(Pid)}),
             ok = force_kill(Pid)
     end.
 
@@ -449,21 +442,21 @@ kick_session(Action, ClientId, ChanPid) ->
                           , error => Error
                           , reason => Reason
                           },
-                #{clientid => unicode:characters_to_list(ClientId, utf8)})
+                #{clientid => ClientId})
     end.
 
 kick_session(ClientId) ->
     case lookup_channels(ClientId) of
         [] ->
             ?SLOG(warning, #{msg => "kicked_an_unknown_session"},
-                #{clientid => unicode:characters_to_list(ClientId, utf8)}),
+                #{clientid => ClientId}),
             ok;
         ChanPids ->
             case length(ChanPids) > 1 of
                 true ->
                     ?SLOG(warning, #{msg => "more_than_one_channel_found",
                                      chan_pids => ChanPids},
-                        #{clientid => unicode:characters_to_list(ClientId, utf8)});
+                        #{clientid => ClientId});
                 false -> ok
             end,
             lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids)
@@ -480,12 +473,12 @@ with_channel(ClientId, Fun) ->
         Pids  -> Fun(lists:last(Pids))
     end.
 
-%% @doc Get all registed channel pids. Debugg/test interface
+%% @doc Get all registered channel pids. Debug/test interface
 all_channels() ->
     Pat = [{{'_', '$1'}, [], ['$1']}],
     ets:select(?CHAN_TAB, Pat).
 
-%% @doc Get all registed clientIDs. Debugg/test interface
+%% @doc Get all registered clientIDs. Debug/test interface
 all_client_ids() ->
     Pat = [{{'$1', '_'}, [], ['$1']}],
     ets:select(?CHAN_TAB, Pat).
@@ -513,7 +506,7 @@ lookup_channels(local, ClientId) ->
 rpc_call(Node, Fun, Args, Timeout) ->
     case rpc:call(Node, ?MODULE, Fun, Args, 2 * Timeout) of
         {badrpc, Reason} ->
-            %% since eqmx app 4.3.10, the 'kick' and 'discard' calls hanndler
+            %% since emqx app 4.3.10, the 'kick' and 'discard' calls handler
             %% should catch all exceptions and always return 'ok'.
             %% This leaves 'badrpc' only possible when there is problem
             %% calling the remote node.
diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index e1dab3260..d334ac23e 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -754,7 +754,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
             <<>> -> ?SLOG(warning, #{
                         msg => "packet_is_discarded",
                         reason => "frame_is_too_large",
-                        packet => emqx_packet:format(Packet, null)
+                        packet => emqx_packet:format(Packet, hidden)
                     }),
                     ok = emqx_metrics:inc('delivery.dropped.too_large'),
                     ok = emqx_metrics:inc('delivery.dropped'),
diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl
index df7c7523f..b34819e53 100644
--- a/apps/emqx/src/emqx_flapping.erl
+++ b/apps/emqx/src/emqx_flapping.erl
@@ -121,7 +121,7 @@ handle_cast({detected, #flapping{clientid   = ClientId,
                 peer_host => fmt_host(PeerHost),
                 detect_cnt => DetectCnt,
                 wind_time_in_ms => WindTime
-            }, #{clientid => unicode:characters_to_list(ClientId, utf8)}),
+            }, #{clientid => ClientId}),
             Now = erlang:system_time(second),
             Banned = #banned{who    = {clientid, ClientId},
                              by     = <<"flapping detector">>,
@@ -136,7 +136,7 @@ handle_cast({detected, #flapping{clientid   = ClientId,
                 peer_host => fmt_host(PeerHost),
                 detect_cnt => DetectCnt,
                 interval => Interval
-            }, #{clientid => unicode:characters_to_list(ClientId, utf8)})
+            }, #{clientid => ClientId})
     end,
     {noreply, State};
 
diff --git a/apps/emqx/src/emqx_logger.erl b/apps/emqx/src/emqx_logger.erl
index 79ac5e6b8..66274a711 100644
--- a/apps/emqx/src/emqx_logger.erl
+++ b/apps/emqx/src/emqx_logger.erl
@@ -197,15 +197,7 @@ critical(Metadata, Format, Args) when is_map(Metadata) ->
 set_metadata_clientid(<<>>) ->
     ok;
 set_metadata_clientid(ClientId) ->
-    try
-        %% try put string format client-id metadata so
-        %% so the log is not like <<"...">>
-        Id = unicode:characters_to_list(ClientId, utf8),
-        set_proc_metadata(#{clientid => Id})
-    catch
-        _: _->
-            ok
-    end.
+    set_proc_metadata(#{clientid => ClientId}).
 
 -spec(set_metadata_peername(peername_str()) -> ok).
 set_metadata_peername(Peername) ->
diff --git a/apps/emqx/src/emqx_logger_textfmt.erl b/apps/emqx/src/emqx_logger_textfmt.erl
index 986c0fd8a..4e7dbcf14 100644
--- a/apps/emqx/src/emqx_logger_textfmt.erl
+++ b/apps/emqx/src/emqx_logger_textfmt.erl
@@ -18,22 +18,77 @@
 
 -export([format/2]).
 -export([check_config/1]).
+-export([try_format_unicode/1]).
 
 check_config(X) -> logger_formatter:check_config(X).
 
-format(#{msg := {report, Report}, meta := Meta} = Event, Config) when is_map(Report) ->
-    logger_formatter:format(Event#{msg := {report, enrich(Report, Meta)}}, Config);
-format(#{msg := Msg, meta := Meta} = Event, Config) ->
-    NewMsg = enrich_fmt(Msg, Meta),
-    logger_formatter:format(Event#{msg := NewMsg}, Config).
+format(#{msg := {report, Report0}, meta := Meta} = Event, Config) when is_map(Report0) ->
+    Report1 = enrich_report_mfa(Report0, Meta),
+    Report2 = enrich_report_clientid(Report1, Meta),
+    Report3 = enrich_report_peername(Report2, Meta),
+    Report4 = enrich_report_topic(Report3, Meta),
+    logger_formatter:format(Event#{msg := {report, Report4}}, Config);
+format(#{msg := {string, String}} = Event, Config) ->
+    format(Event#{msg => {"~ts ", String}}, Config);
+format(#{msg := Msg0, meta := Meta} = Event, Config) ->
+    Msg1 = enrich_client_info(Msg0, Meta),
+    Msg2 = enrich_mfa(Msg1, Meta),
+    Msg3 = enrich_topic(Msg2, Meta),
+    logger_formatter:format(Event#{msg := Msg3}, Config).
 
-enrich(Report, #{mfa := Mfa, line := Line}) ->
+try_format_unicode(Char) ->
+    List =
+        try
+            case unicode:characters_to_list(Char) of
+                {error, _, _} -> error;
+                {incomplete, _, _} -> error;
+                Binary -> Binary
+            end
+        catch _:_ ->
+            error
+        end,
+    case List of
+        error -> io_lib:format("~0p", [Char]);
+        _ -> List
+    end.
+
+enrich_report_mfa(Report, #{mfa := Mfa, line := Line}) ->
     Report#{mfa => mfa(Mfa), line => Line};
-enrich(Report, _) -> Report.
+enrich_report_mfa(Report, _) -> Report.
 
-enrich_fmt({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) ->
+enrich_report_clientid(Report, #{clientid := ClientId}) ->
+    Report#{clientid => try_format_unicode(ClientId)};
+enrich_report_clientid(Report, _) -> Report.
+
+enrich_report_peername(Report, #{peername := Peername}) ->
+    Report#{peername => Peername};
+enrich_report_peername(Report, _) -> Report.
+
+%% clientid and peername always in emqx_conn's process metadata.
+%% topic can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2
+enrich_report_topic(Report, #{topic := Topic}) ->
+    Report#{topic => try_format_unicode(Topic)};
+enrich_report_topic(Report = #{topic := Topic}, _) ->
+    Report#{topic => try_format_unicode(Topic)};
+enrich_report_topic(Report, _) -> Report.
+
+enrich_mfa({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) ->
     {Fmt ++ " mfa: ~ts line: ~w", Args ++ [mfa(Mfa), Line]};
-enrich_fmt(Msg, _) ->
+enrich_mfa(Msg, _) ->
+    Msg.
+
+enrich_client_info({Fmt, Args}, #{clientid := ClientId, peername := Peer}) when is_list(Fmt) ->
+    {" ~ts@~ts " ++ Fmt, [ClientId, Peer | Args] };
+enrich_client_info({Fmt, Args}, #{clientid := ClientId}) when is_list(Fmt) ->
+    {" ~ts " ++ Fmt, [ClientId | Args]};
+enrich_client_info({Fmt, Args}, #{peername := Peer}) when is_list(Fmt) ->
+    {" ~ts " ++ Fmt, [Peer | Args]};
+enrich_client_info(Msg, _) ->
+    Msg.
+
+enrich_topic({Fmt, Args}, #{topic := Topic}) when is_list(Fmt) ->
+    {" topic: ~ts" ++ Fmt, [Topic | Args]};
+enrich_topic(Msg, _) ->
     Msg.
 
 mfa({M, F, A}) -> atom_to_list(M) ++ ":" ++ atom_to_list(F) ++ "/" ++ integer_to_list(A).
diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl
index 53e4eabfe..23b8390e5 100644
--- a/apps/emqx/src/emqx_packet.erl
+++ b/apps/emqx/src/emqx_packet.erl
@@ -48,6 +48,8 @@
         , format/2
         ]).
 
+-export([encode_hex/1]).
+
 -define(TYPE_NAMES,
         { 'CONNECT'
         , 'CONNACK'
@@ -440,7 +442,7 @@ will_msg(#mqtt_packet_connect{clientid     = ClientId,
 format(Packet) -> format(Packet, emqx_trace_handler:payload_encode()).
 
 %% @doc Format packet
--spec(format(emqx_types:packet(), hex | text | null) -> iolist()).
+-spec(format(emqx_types:packet(), hex | text | hidden) -> iolist()).
 format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, PayloadEncode) ->
     HeaderIO = format_header(Header),
     case format_variable(Variable, Payload, PayloadEncode) of
@@ -504,11 +506,13 @@ format_variable(#mqtt_packet_puback{packet_id = PacketId,
 
 format_variable(#mqtt_packet_subscribe{packet_id     = PacketId,
                                        topic_filters = TopicFilters}, _) ->
-    io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, TopicFilters]);
+    [io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=",
+        format_topic_filters(TopicFilters)];
 
 format_variable(#mqtt_packet_unsubscribe{packet_id     = PacketId,
                                          topic_filters = Topics}, _) ->
-    io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, Topics]);
+    [io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=",
+        format_topic_filters(Topics)];
 
 format_variable(#mqtt_packet_suback{packet_id = PacketId,
                                     reason_codes = ReasonCodes}, _) ->
@@ -527,9 +531,83 @@ format_password(undefined) -> "undefined";
 format_password(_Password) -> "******".
 
 format_payload(Payload, text) -> ["Payload=", io_lib:format("~ts", [Payload])];
-format_payload(Payload, hex) -> ["Payload(hex)=", binary:encode_hex(Payload)];
-format_payload(_, null) -> "Payload=******".
+format_payload(Payload, hex) -> ["Payload(hex)=", encode_hex(Payload)];
+format_payload(_, hidden) -> "Payload=******".
 
 i(true)  -> 1;
 i(false) -> 0;
 i(I) when is_integer(I) -> I.
+
+format_topic_filters(Filters) ->
+    ["[",
+        lists:join(",",
+            lists:map(
+                fun({TopicFilter, SubOpts}) ->
+                    io_lib:format("~ts(~p)", [TopicFilter, SubOpts]);
+                    (TopicFilter) ->
+                        io_lib:format("~ts", [TopicFilter])
+                end, Filters)),
+        "]"].
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% Hex encoding functions
+%% Copy from binary:encode_hex/1 (was only introduced in OTP24).
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+-define(HEX(X), (hex(X)):16).
+-compile({inline,[hex/1]}).
+-spec encode_hex(Bin) -> Bin2 when
+    Bin :: binary(),
+    Bin2 :: <<_:_*16>>.
+encode_hex(Data) when byte_size(Data) rem 8 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G),?HEX(H)>> || <<A,B,C,D,E,F,G,H>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 7 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G)>> || <<A,B,C,D,E,F,G>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 6 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F)>> || <<A,B,C,D,E,F>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 5 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E)>> || <<A,B,C,D,E>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 4 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D)>> || <<A,B,C,D>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 3 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C)>> || <<A,B,C>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 2 =:= 0 ->
+    << <<?HEX(A),?HEX(B)>> || <<A,B>> <= Data >>;
+encode_hex(Data) when is_binary(Data) ->
+    << <<?HEX(N)>> || <<N>> <= Data >>;
+encode_hex(Bin) ->
+    erlang:error(badarg, [Bin]).
+
+hex(X) ->
+    element(
+        X+1, {16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, 16#3041,
+            16#3042, 16#3043, 16#3044, 16#3045, 16#3046,
+            16#3130, 16#3131, 16#3132, 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3141,
+            16#3142, 16#3143, 16#3144, 16#3145, 16#3146,
+            16#3230, 16#3231, 16#3232, 16#3233, 16#3234, 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3241,
+            16#3242, 16#3243, 16#3244, 16#3245, 16#3246,
+            16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, 16#3337, 16#3338, 16#3339, 16#3341,
+            16#3342, 16#3343, 16#3344, 16#3345, 16#3346,
+            16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, 16#3439, 16#3441,
+            16#3442, 16#3443, 16#3444, 16#3445, 16#3446,
+            16#3530, 16#3531, 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3541,
+            16#3542, 16#3543, 16#3544, 16#3545, 16#3546,
+            16#3630, 16#3631, 16#3632, 16#3633, 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3641,
+            16#3642, 16#3643, 16#3644, 16#3645, 16#3646,
+            16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, 16#3736, 16#3737, 16#3738, 16#3739, 16#3741,
+            16#3742, 16#3743, 16#3744, 16#3745, 16#3746,
+            16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, 16#3838, 16#3839, 16#3841,
+            16#3842, 16#3843, 16#3844, 16#3845, 16#3846,
+            16#3930, 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, 16#3941,
+            16#3942, 16#3943, 16#3944, 16#3945, 16#3946,
+            16#4130, 16#4131, 16#4132, 16#4133, 16#4134, 16#4135, 16#4136, 16#4137, 16#4138, 16#4139, 16#4141,
+            16#4142, 16#4143, 16#4144, 16#4145, 16#4146,
+            16#4230, 16#4231, 16#4232, 16#4233, 16#4234, 16#4235, 16#4236, 16#4237, 16#4238, 16#4239, 16#4241,
+            16#4242, 16#4243, 16#4244, 16#4245, 16#4246,
+            16#4330, 16#4331, 16#4332, 16#4333, 16#4334, 16#4335, 16#4336, 16#4337, 16#4338, 16#4339, 16#4341,
+            16#4342, 16#4343, 16#4344, 16#4345, 16#4346,
+            16#4430, 16#4431, 16#4432, 16#4433, 16#4434, 16#4435, 16#4436, 16#4437, 16#4438, 16#4439, 16#4441,
+            16#4442, 16#4443, 16#4444, 16#4445, 16#4446,
+            16#4530, 16#4531, 16#4532, 16#4533, 16#4534, 16#4535, 16#4536, 16#4537, 16#4538, 16#4539, 16#4541,
+            16#4542, 16#4543, 16#4544, 16#4545, 16#4546,
+            16#4630, 16#4631, 16#4632, 16#4633, 16#4634, 16#4635, 16#4636, 16#4637, 16#4638, 16#4639, 16#4641,
+            16#4642, 16#4643, 16#4644, 16#4645, 16#4646}).
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 123ee5379..fc90ba1bb 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -985,13 +985,13 @@ fields("latency_stats") ->
                                   desc => "the number of smaples for calculate the average latency of delivery"})}
     ];
 fields("trace") ->
-    [ {"payload_encode", sc(hoconsc:enum([hex, text, null]), #{
+    [ {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{
         default => text,
         desc => """
-Determine the format of the payload format in the trace file.</br>
-- `text`: Text-based protocol or plain text protocol. It is recommended when payload is json encode.</br>
-- `hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.</br>
-- `null`: Don't show payload in trace log file.
+Determine the format of the payload format in the trace file.<br>
+`text`: Text-based protocol or plain text protocol. It is recommended when payload is json encode.<br>
+`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.<br>
+`hidden`: payload is obfuscated as `******`
         """
     })}
     ].
diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl
index 3869d300d..5af0d156e 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace.erl
@@ -26,7 +26,7 @@
 -export([ publish/1
         , subscribe/3
         , unsubscribe/2
-        , log/3
+        , log/4
         ]).
 
 -export([ start_link/0
@@ -52,8 +52,7 @@
 
 -define(TRACE, ?MODULE).
 -define(MAX_SIZE, 30).
--define(TRACE_FILTER, emqx_trace_filter).
--define(OWN_KEYS,[level,filters,filter_default,handlers]).
+-define(OWN_KEYS, [level, filters, filter_default, handlers]).
 
 -ifdef(TEST).
 -export([ log_file/2
@@ -94,18 +93,14 @@ unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore;
 unsubscribe(Topic, SubOpts) ->
     ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}).
 
-log(Event, Msg, Meta0) ->
-    case persistent_term:get(?TRACE_FILTER, undefined) of
-        undefined -> ok;
-        List ->
-            Meta =
-                case logger:get_process_metadata() of
-                    undefined -> Meta0;
-                    ProcMeta -> maps:merge(ProcMeta, Meta0)
-                end,
-            Log = #{level => trace, event => Event, meta => Meta, msg => Msg},
-            log_filter(List, Log)
-    end.
+log(List, Event, Msg, Meta0) ->
+    Meta =
+        case logger:get_process_metadata() of
+            undefined -> Meta0;
+            ProcMeta -> maps:merge(ProcMeta, Meta0)
+        end,
+    Log = #{level => trace, event => Event, meta => Meta, msg => Msg},
+    log_filter(List, Log).
 
 log_filter([], _Log) -> ok;
 log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
@@ -196,7 +191,7 @@ update(Name, Enable) ->
     transaction(Tran).
 
 check() ->
-    erlang:send(?MODULE, {mnesia_table_event, check}).
+    gen_server:call(?MODULE, check).
 
 -spec get_trace_filename(Name :: binary()) ->
     {ok, FileName :: string()} | {error, not_found}.
@@ -241,6 +236,9 @@ init([]) ->
     update_trace_handler(),
     {ok, #{timer => TRef, monitors => #{}}}.
 
+handle_call(check, _From, State) ->
+    {_, NewState} = handle_info({mnesia_table_event, check}, State),
+    {reply, ok, NewState};
 handle_call(Req, _From, State) ->
     ?SLOG(error, #{unexpected_call => Req}),
     {reply, ok, State}.
@@ -259,8 +257,7 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor
             lists:foreach(fun file:delete/1, Files),
             {noreply, State#{monitors => NewMonitors}}
     end;
-handle_info({timeout, TRef, update_trace},
-    #{timer := TRef} = State) ->
+handle_info({timeout, TRef, update_trace}, #{timer := TRef} = State) ->
     Traces = get_enable_trace(),
     NextTRef = update_trace(Traces),
     update_trace_handler(),
@@ -344,10 +341,10 @@ disable_finished(Traces) ->
 
 start_trace(Traces, Started0) ->
     Started = lists:map(fun(#{name := Name}) -> Name end, Started0),
-    lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) ->
+    lists:foldl(fun(#?TRACE{name = Name} = Trace,
+        {Running, StartedAcc}) ->
         case lists:member(Name, StartedAcc) of
-            true ->
-                {[Name | Running], StartedAcc};
+            true -> {[Name | Running], StartedAcc};
             false ->
                 case start_trace(Trace) of
                     ok -> {[Name | Running], [Name | StartedAcc]};
@@ -366,9 +363,11 @@ start_trace(Trace) ->
     emqx_trace_handler:install(Who, debug, log_file(Name, Start)).
 
 stop_trace(Finished, Started) ->
-    lists:foreach(fun(#{name := Name, type := Type}) ->
+    lists:foreach(fun(#{name := Name, type := Type, filter := Filter}) ->
         case lists:member(Name, Finished) of
-            true -> emqx_trace_handler:uninstall(Type, Name);
+            true ->
+                ?TRACE("API", "trace_stopping", #{Type => Filter}),
+                emqx_trace_handler:uninstall(Type, Name);
             false -> ok
         end
                   end, Started).
@@ -455,7 +454,7 @@ to_trace(#{type := ip_address, ip_address := Filter} = Trace, Rec) ->
     case validate_ip_address(Filter) of
         ok ->
             Trace0 = maps:without([type, ip_address], Trace),
-            to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = Filter});
+            to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = binary_to_list(Filter)});
         Error -> Error
     end;
 to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])};
diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
index a9c4ca31d..2ef142d38 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
@@ -25,7 +25,7 @@
 format(#{level := trace, event := Event, meta := Meta, msg := Msg},
     #{payload_encode := PEncode}) ->
     Time = calendar:system_time_to_rfc3339(erlang:system_time(second)),
-    ClientId = maps:get(clientid, Meta, ""),
+    ClientId = to_iolist(maps:get(clientid, Meta, "")),
     Peername = maps:get(peername, Meta, ""),
     MetaBin = format_meta(Meta, PEncode),
     [Time, " [", Event, "] ", ClientId, "@", Peername, " msg: ", Msg, MetaBin, "\n"];
@@ -39,10 +39,7 @@ format_meta(Meta0, Encode) ->
     Meta1 = maps:without([msg, clientid, peername, packet, payload], Meta0),
     case Meta1 =:= #{} of
         true -> [Packet, Payload];
-        false ->
-            Meta2 = lists:map(fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end,
-                maps:to_list(Meta1)),
-            [Packet, ", ", lists:join(",", Meta2), Payload]
+        false -> [Packet, ", ", map_to_iolist(Meta1), Payload]
     end.
 
 format_packet(undefined, _) -> "";
@@ -50,12 +47,16 @@ format_packet(Packet, Encode) -> [", packet: ", emqx_packet:format(Packet, Encod
 
 format_payload(undefined, _) -> "";
 format_payload(Payload, text) -> [", payload: ", io_lib:format("~ts", [Payload])];
-format_payload(Payload, hex) -> [", payload(hex): ", binary:encode_hex(Payload)];
-format_payload(_, null) -> ", payload=******".
+format_payload(Payload, hex) -> [", payload(hex): ", emqx_packet:encode_hex(Payload)];
+format_payload(_, hidden) -> ", payload=******".
 
 to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom);
 to_iolist(Int) when is_integer(Int) -> integer_to_list(Int);
 to_iolist(Float) when is_float(Float) -> float_to_list(Float, [{decimals, 2}]);
-to_iolist(Bin)when is_binary(Bin)  -> unicode:characters_to_binary(Bin);
-to_iolist(List) when is_list(List) -> unicode:characters_to_list(List);
-to_iolist(Term) -> io_lib:format("~0p", [Term]).
+to_iolist(SubMap) when is_map(SubMap) -> ["[", map_to_iolist(SubMap), "]"];
+to_iolist(Char) -> emqx_logger_textfmt:try_format_unicode(Char).
+
+map_to_iolist(Map) ->
+    lists:join(",",
+        lists:map(fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end,
+            maps:to_list(Map))).
diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
index f8a5f3828..320421309 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl
@@ -119,11 +119,11 @@ uninstall(HandlerId) ->
 running() ->
     lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)).
 
--spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop.
+-spec filter_clientid(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop.
 filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log;
 filter_clientid(_Log, _ExpectId) -> stop.
 
--spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop.
+-spec filter_topic(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop.
 filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) ->
     case emqx_topic:match(Topic, TopicFilter) of
         true -> Log;
@@ -140,7 +140,7 @@ filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) ->
 filter_ip_address(_Log, _ExpectId) -> stop.
 
 filters(#{type := clientid, filter := Filter, name := Name}) ->
-    [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}];
+    [{clientid, {fun ?MODULE:filter_clientid/2, {Filter, Name}}}];
 filters(#{type := topic, filter := Filter, name := Name}) ->
     [{topic, {fun ?MODULE:filter_topic/2, {ensure_bin(Filter), Name}}}];
 filters(#{type := ip_address, filter := Filter, name := Name}) ->
@@ -149,8 +149,9 @@ filters(#{type := ip_address, filter := Filter, name := Name}) ->
 formatter(#{type := _Type}) ->
     {emqx_trace_formatter,
         #{
-            template => [],
-            single_line => false,
+            %% template is for ?SLOG message not ?TRACE.
+            template => [time," [",level,"] ", msg,"\n"],
+            single_line => true,
             max_size => unlimited,
             depth => unlimited,
             payload_encode => payload_encode()
diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl
index f3ab7b5b8..1224fdac9 100644
--- a/apps/emqx/test/emqx_trace_handler_SUITE.erl
+++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl
@@ -32,19 +32,22 @@ all() -> [t_trace_clientid, t_trace_topic, t_trace_ip_address, t_trace_clientid_
 
 init_per_suite(Config) ->
     emqx_common_test_helpers:boot_modules(all),
-    emqx_common_test_helpers:start_apps([emqx_modules]),
+    emqx_common_test_helpers:start_apps([]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_modules]).
+    emqx_common_test_helpers:stop_apps([]).
 
 init_per_testcase(t_trace_clientid, Config) ->
+    init(),
     Config;
 init_per_testcase(_Case, Config) ->
     _ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()],
+    init(),
     Config.
 
 end_per_testcase(_Case, _Config) ->
+    terminate(),
     ok.
 
 t_trace_clientid(_Config) ->
@@ -66,11 +69,11 @@ t_trace_clientid(_Config) ->
     ?assert(filelib:is_regular("tmp/client3.log")),
 
     %% Get current traces
-    ?assertMatch([#{type := clientid, filter := "client", name := <<"CLI-client1">>,
+    ?assertMatch([#{type := clientid, filter := <<"client">>, name := <<"CLI-client1">>,
         level := debug, dst := "tmp/client.log"},
-        #{type := clientid, filter := "client2", name := <<"CLI-client2">>
+        #{type := clientid, filter := <<"client2">>, name := <<"CLI-client2">>
             , level := debug, dst := "tmp/client2.log"},
-        #{type := clientid, filter := "client3", name := <<"CLI-client3">>,
+        #{type := clientid, filter := <<"client3">>, name := <<"CLI-client3">>,
             level := debug, dst := "tmp/client3.log"}
     ], emqx_trace_handler:running()),
 
@@ -231,3 +234,9 @@ filesync(Name0, Type, Retry) ->
         ct:sleep(100),
         filesync(Name, Type, Retry - 1)
     end.
+
+init() ->
+    emqx_trace:start_link().
+
+terminate() ->
+    catch ok = gen_server:stop(emqx_trace, normal, 5000).
diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
index 820eeb859..31e5e52e1 100644
--- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
@@ -43,6 +43,7 @@ groups() ->
     [].
 
 init_per_testcase(_, Config) ->
+    {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
     emqx_authn_test_lib:delete_authenticators(
       [?CONF_NS_ATOM],
       ?GLOBAL),
@@ -55,8 +56,9 @@ init_per_testcase(_, Config) ->
     Config.
 
 init_per_suite(Config) ->
+    _ = application:load(emqx_conf),
     ok = emqx_common_test_helpers:start_apps(
-           [emqx_conf, emqx_authn, emqx_dashboard],
+           [emqx_authn, emqx_dashboard],
            fun set_special_configs/1),
 
     ?AUTHN:delete_chain(?GLOBAL),
@@ -65,7 +67,7 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_authn, emqx_dashboard]),
+    emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authn]),
     ok.
 
 set_special_configs(emqx_dashboard) ->
diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index 1ea368826..385953b87 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -792,16 +792,7 @@ do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth) ->
         }};
 do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth) ->
     {emqx_logger_textfmt,
-        #{template =>
-            [time," [",level,"] ",
-                {clientid,
-                    [{peername,
-                        [clientid,"@",peername," "],
-                        [clientid, " "]}],
-                    [{peername,
-                        [peername," "],
-                        []}]},
-                msg,"\n"],
+        #{template => [time," [",level,"] ", msg,"\n"],
           chars_limit => CharsLimit,
           single_line => SingleLine,
           time_offset => TimeOffSet,
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index 5d9d0e1d9..81435a1c5 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -88,7 +88,8 @@ on_query(InstId, QueryParams, AfterQuery, #{poolname := PoolName} = State) ->
                           {prepared_query, Name, SQL} -> {prepared_query, [Name, SQL, []]};
                           {prepared_query, Name, SQL, Params} -> {prepared_query, [Name, SQL, Params]}
                       end,
-    ?TRACE("QUERY", "postgresql_connector_received", #{connector => InstId, command => Command, args => Args, state => State}}),
+    ?TRACE("QUERY", "postgresql_connector_received",
+        #{connector => InstId, command => Command, args => Args, state => State}),
     case Result = ecpool:pick_and_do(PoolName, {?MODULE, Command, Args}, no_handover) of
         {error, Reason} ->
             ?SLOG(error, #{
diff --git a/apps/emqx_management/src/emqx_mgmt_api_app.erl b/apps/emqx_management/src/emqx_mgmt_api_app.erl
index dfce3cf30..489d679be 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_app.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_app.erl
@@ -149,7 +149,7 @@ api_key(post, #{body := App}) ->
     Desc = unicode:characters_to_binary(Desc0, unicode),
     case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of
         {ok, NewApp} -> {200, format(NewApp)};
-        {error, Reason} -> {400, Reason}
+        {error, Reason} -> {400, io_lib:format("~p", [Reason])}
     end.
 
 api_key_by_name(get, #{bindings := #{name := Name}}) ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
index 15277fa8e..296cecea2 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
@@ -129,11 +129,10 @@ schema("/trace/:name/log") ->
                 hoconsc:ref(position),
                 hoconsc:ref(node)
             ],
-            %% todo response data
             responses => #{
                 200 =>
                 [
-                    {items, hoconsc:mk(binary(), #{example => "BinBinBin"})}
+                    {items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}
                     | fields(bytes) ++ fields(position)
                 ]
             }
@@ -307,7 +306,7 @@ download_trace_log(get, #{bindings := #{name := Name}}) ->
             {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]),
             emqx_trace:delete_files_after_send(ZipFileName, Zips),
             Headers = #{
-                <<"content-type">> => <<"application/octet-stream">>,
+                <<"content-type">> => <<"application/x-zip">>,
                 <<"content-disposition">> =>
                 iolist_to_binary("attachment; filename=" ++ filename:basename(ZipFile))
             },
diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl
index 0084855ab..3a723f33c 100644
--- a/apps/emqx_management/src/emqx_mgmt_cli.erl
+++ b/apps/emqx_management/src/emqx_mgmt_cli.erl
@@ -18,6 +18,7 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -include("emqx_mgmt.hrl").
 
@@ -386,19 +387,19 @@ trace(["list"]) ->
         emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst])
                   end, emqx_trace_handler:running());
 
-trace(["stop", Operation, ClientId]) ->
-    case trace_type(Operation) of
-        {ok, Type} -> trace_off(Type, ClientId);
+trace(["stop", Operation, Filter0]) ->
+    case trace_type(Operation, Filter0) of
+        {ok, Type, Filter} -> trace_off(Type, Filter);
         error -> trace([])
     end;
 
 trace(["start", Operation, ClientId, LogFile]) ->
     trace(["start", Operation, ClientId, LogFile, "all"]);
 
-trace(["start", Operation, Filter, LogFile, Level]) ->
-    case trace_type(Operation) of
-        {ok, Type} ->
-            trace_on(name(Filter), Type, Filter,
+trace(["start", Operation, Filter0, LogFile, Level]) ->
+    case trace_type(Operation, Filter0) of
+        {ok, Type, Filter} ->
+            trace_on(name(Filter0), Type, Filter,
                 list_to_existing_atom(Level), LogFile);
         error -> trace([])
     end;
@@ -428,13 +429,14 @@ trace_on(Name, Type, Filter, Level, LogFile) ->
             emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Filter, Name, Error])
     end.
 
-trace_off(Who, Filter) ->
-    case emqx_trace_handler:uninstall(Who, name(Filter)) of
+trace_off(Type, Filter) ->
+    ?TRACE("CLI", "trace_stopping", #{Type => Filter}),
+    case emqx_trace_handler:uninstall(Type, name(Filter)) of
         ok ->
             emqx_trace:check(),
-            emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Filter]);
+            emqx_ctl:print("stop tracing ~s ~s successfully~n", [Type, Filter]);
         {error, Error} ->
-            emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Filter, Error])
+            emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Type, Filter, Error])
     end.
 
 %%--------------------------------------------------------------------
@@ -463,9 +465,9 @@ traces(["delete", Name]) ->
 traces(["start", Name, Operation, Filter]) ->
     traces(["start", Name, Operation, Filter, "900"]);
 
-traces(["start", Name, Operation, Filter, DurationS]) ->
-    case trace_type(Operation) of
-        {ok, Type} ->  trace_cluster_on(Name, Type, Filter, DurationS);
+traces(["start", Name, Operation, Filter0, DurationS]) ->
+    case trace_type(Operation, Filter0) of
+        {ok, Type, Filter} ->  trace_cluster_on(Name, Type, Filter, DurationS);
         error -> traces([])
     end;
 
@@ -507,10 +509,10 @@ trace_cluster_off(Name) ->
         {error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error])
     end.
 
-trace_type("client") -> {ok, clientid};
-trace_type("topic") -> {ok, topic};
-trace_type("ip_address") -> {ok, ip_address};
-trace_type(_) -> error.
+trace_type("client", ClientId) -> {ok, clientid, list_to_binary(ClientId)};
+trace_type("topic", Topic) -> {ok, topic, list_to_binary(Topic)};
+trace_type("ip_address", IP) -> {ok, ip_address, IP};
+trace_type(_, _) -> error.
 
 %%--------------------------------------------------------------------
 %% @doc Listeners Command