Merge remote-tracking branch 'upstream/release-54'

.ci/docker-compose-file/credentials.env

@@ -0,0 +1,7 @@
+MONGO_USERNAME=emqx
+MONGO_PASSWORD=passw0rd
+MONGO_AUTHSOURCE=admin
+
+# See "Environment Variables" @ https://hub.docker.com/_/mongo
+MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME}
+MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD}

.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml

@@ -9,6 +9,9 @@ services:
       - emqx_bridge
     ports:
       - "27017:27017"
+    env_file:
+      - .env
+      - credentials.env
     command:
       --ipv6
       --bind_ip_all

.ci/docker-compose-file/docker-compose.yaml

@@ -5,6 +5,7 @@ services:
     container_name: erlang
     image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04}
     env_file:
+      - credentials.env
       - conf.env
     environment:
       GITHUB_ACTIONS: ${GITHUB_ACTIONS:-}

apps/emqx/include/emqx.hrl

@@ -39,9 +39,6 @@
 %% System topic
 -define(SYSTOP, <<"$SYS/">>).
 
-%% Queue topic
--define(QUEUE, <<"$queue/">>).
-
 %%--------------------------------------------------------------------
 %% alarms
 %%--------------------------------------------------------------------

apps/emqx/include/emqx_mqtt.hrl

@@ -55,6 +55,17 @@
 %% MQTT-3.1.1 and MQTT-5.0 [MQTT-4.7.3-3]
 -define(MAX_TOPIC_LEN, 65535).
 
+%%--------------------------------------------------------------------
+%% MQTT Share-Sub Internal
+%%--------------------------------------------------------------------
+
+-record(share, {group :: emqx_types:group(), topic :: emqx_types:topic()}).
+
+%% guards
+-define(IS_TOPIC(T),
+    (is_binary(T) orelse is_record(T, share))
+).
+
 %%--------------------------------------------------------------------
 %% MQTT QoS Levels
 %%--------------------------------------------------------------------
@@ -661,13 +672,10 @@ end).
 -define(PACKET(Type), #mqtt_packet{header = #mqtt_packet_header{type = Type}}).
 
 -define(SHARE, "$share").
+-define(QUEUE, "$queue").
 -define(SHARE(Group, Topic), emqx_topic:join([<<?SHARE>>, Group, Topic])).
--define(IS_SHARE(Topic),
+
-    case Topic of
+-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
-        <<?SHARE, _/binary>> -> true;
-        _ -> false
-    end
-).
 
 -define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty).
 -define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty).

apps/emqx/include/emqx_trace.hrl

@@ -32,6 +32,5 @@
 
 -define(SHARD, ?COMMON_SHARD).
 -define(MAX_SIZE, 30).
--define(OWN_KEYS, [level, filters, filter_default, handlers]).
 
 -endif.

apps/emqx/include/http_api.hrl

@@ -17,6 +17,7 @@
 %% HTTP API Auth
 -define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
 -define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET').
+-define(API_KEY_NOT_ALLOW_MSG, <<"This API Key don't have permission to access this resource">>).
 
 %% Bad Request
 -define(BAD_REQUEST, 'BAD_REQUEST').

apps/emqx/include/logger.hrl

@@ -40,7 +40,9 @@
     end
 ).
 
+-define(AUDIT_HANDLER, emqx_audit).
 -define(TRACE_FILTER, emqx_trace_filter).
+-define(OWN_KEYS, [level, filters, filter_default, handlers]).
 
 -define(TRACE(Tag, Msg, Meta), ?TRACE(debug, Tag, Msg, Meta)).
 
@@ -61,25 +63,35 @@
     )
 end).
 
--define(AUDIT(_Level_, _From_, _Meta_), begin
+-ifdef(EMQX_RELEASE_EDITION).
-    case emqx_config:get([log, audit], #{enable => false}) of
+
-        #{enable := false} ->
+-if(?EMQX_RELEASE_EDITION == ee).
+
+-define(AUDIT(_LevelFun_, _MetaFun_), begin
+    case logger_config:get(logger, ?AUDIT_HANDLER) of
+        {error, {not_found, _}} ->
             ok;
-        #{enable := true, level := _AllowLevel_} ->
+        {ok, Handler = #{level := _AllowLevel_}} ->
+            _Level_ = _LevelFun_,
             case logger:compare_levels(_AllowLevel_, _Level_) of
                 _R_ when _R_ == lt; _R_ == eq ->
-                    emqx_trace:log(
+                    emqx_audit:log(_Level_, _MetaFun_, Handler);
-                        _Level_,
+                _ ->
-                        [{emqx_audit, fun(L, _) -> L end, undefined, undefined}],
-                        _Msg = undefined,
-                        _Meta_#{from => _From_}
-                    );
-                gt ->
                     ok
             end
     end
 end).
 
+-else.
+%% Only for compile pass, ce edition will not call it
+-define(AUDIT(_L_, _M_), _ = {_L_, _M_}).
+-endif.
+
+-else.
+%% Only for compile pass, ce edition will not call it
+-define(AUDIT(_L_, _M_), _ = {_L_, _M_}).
+-endif.
+
 %% print to 'user' group leader
 -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
 -define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)).

apps/emqx/src/bhvrs/emqx_db_backup.erl

@@ -16,4 +16,21 @@
 
 -module(emqx_db_backup).
 
+-type traverse_break_reason() :: over | migrate.
+
 -callback backup_tables() -> [mria:table()].
+
+%% validate the backup
+%% return `ok` to traverse the next item
+%% return `{ok, over}` to finish the traverse
+%% return `{ok, migrate}` to call the migration callback
+-callback validate_mnesia_backup(tuple()) ->
+    ok
+    | {ok, traverse_break_reason()}
+    | {error, term()}.
+
+-callback migrate_mnesia_backup(tuple()) -> {ok, tuple()} | {error, term()}.
+
+-optional_callbacks([validate_mnesia_backup/1, migrate_mnesia_backup/1]).
+
+-export_type([traverse_break_reason/0]).

apps/emqx/src/config/emqx_config_logger.erl

@@ -23,8 +23,9 @@
 -export([post_config_update/5]).
 -export([filter_audit/2]).
 
+-include("logger.hrl").
+
 -define(LOG, [log]).
--define(AUDIT_HANDLER, emqx_audit).
 
 add_handler() ->
     ok = emqx_config_handler:add_handler(?LOG, ?MODULE),
@@ -95,6 +96,10 @@ update_log_handlers(NewHandlers) ->
     ok = application:set_env(kernel, logger, NewHandlers),
     ok.
 
+%% Don't remove audit log handler here, we need record this removed action into audit log file.
+%% we will remove audit log handler after audit log is record in emqx_audit:log/3.
+update_log_handler({removed, ?AUDIT_HANDLER}) ->
+    ok;
 update_log_handler({removed, Id}) ->
     log_to_console("Config override: ~s is removed~n", [id_for_log(Id)]),
     logger:remove_handler(Id);

apps/emqx/src/emqx_broker.erl

@@ -118,18 +118,20 @@ create_tabs() ->
 %% Subscribe API
 %%------------------------------------------------------------------------------
 
--spec subscribe(emqx_types:topic()) -> ok.
+-spec subscribe(emqx_types:topic() | emqx_types:share()) -> ok.
-subscribe(Topic) when is_binary(Topic) ->
+subscribe(Topic) when ?IS_TOPIC(Topic) ->
     subscribe(Topic, undefined).
 
--spec subscribe(emqx_types:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok.
+-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid() | emqx_types:subopts()) ->
-subscribe(Topic, SubId) when is_binary(Topic), ?IS_SUBID(SubId) ->
+    ok.
+subscribe(Topic, SubId) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId) ->
     subscribe(Topic, SubId, ?DEFAULT_SUBOPTS);
-subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) ->
+subscribe(Topic, SubOpts) when ?IS_TOPIC(Topic), is_map(SubOpts) ->
     subscribe(Topic, undefined, SubOpts).
 
--spec subscribe(emqx_types:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok.
+-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid(), emqx_types:subopts()) ->
-subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) ->
+    ok.
+subscribe(Topic, SubId, SubOpts0) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) ->
     SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0),
     _ = emqx_trace:subscribe(Topic, SubId, SubOpts),
     SubPid = self(),
@@ -151,13 +153,13 @@ with_subid(undefined, SubOpts) ->
 with_subid(SubId, SubOpts) ->
     maps:put(subid, SubId, SubOpts).
 
-%% @private
 do_subscribe(Topic, SubPid, SubOpts) ->
     true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}),
-    Group = maps:get(share, SubOpts, undefined),
+    do_subscribe2(Topic, SubPid, SubOpts).
-    do_subscribe(Group, Topic, SubPid, SubOpts).
 
-do_subscribe(undefined, Topic, SubPid, SubOpts) ->
+do_subscribe2(Topic, SubPid, SubOpts) when is_binary(Topic) ->
+    %% FIXME: subscribe shard bug
+    %% https://emqx.atlassian.net/browse/EMQX-10214
     case emqx_broker_helper:get_sub_shard(SubPid, Topic) of
         0 ->
             true = ets:insert(?SUBSCRIBER, {Topic, SubPid}),
@@ -168,34 +170,40 @@ do_subscribe(undefined, Topic, SubPid, SubOpts) ->
             true = ets:insert(?SUBOPTION, {{Topic, SubPid}, maps:put(shard, I, SubOpts)}),
             call(pick({Topic, I}), {subscribe, Topic, I})
     end;
-%% Shared subscription
+do_subscribe2(Topic = #share{group = Group, topic = RealTopic}, SubPid, SubOpts) when
-do_subscribe(Group, Topic, SubPid, SubOpts) ->
+    is_binary(RealTopic)
+->
     true = ets:insert(?SUBOPTION, {{Topic, SubPid}, SubOpts}),
-    emqx_shared_sub:subscribe(Group, Topic, SubPid).
+    emqx_shared_sub:subscribe(Group, RealTopic, SubPid).
 
 %%--------------------------------------------------------------------
 %% Unsubscribe API
 %%--------------------------------------------------------------------
 
--spec unsubscribe(emqx_types:topic()) -> ok.
+-spec unsubscribe(emqx_types:topic() | emqx_types:share()) -> ok.
-unsubscribe(Topic) when is_binary(Topic) ->
+unsubscribe(Topic) when ?IS_TOPIC(Topic) ->
     SubPid = self(),
     case ets:lookup(?SUBOPTION, {Topic, SubPid}) of
         [{_, SubOpts}] ->
-            _ = emqx_broker_helper:reclaim_seq(Topic),
             _ = emqx_trace:unsubscribe(Topic, SubOpts),
             do_unsubscribe(Topic, SubPid, SubOpts);
         [] ->
             ok
     end.
 
+-spec do_unsubscribe(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) ->
+    ok.
 do_unsubscribe(Topic, SubPid, SubOpts) ->
     true = ets:delete(?SUBOPTION, {Topic, SubPid}),
     true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}),
-    Group = maps:get(share, SubOpts, undefined),
+    do_unsubscribe2(Topic, SubPid, SubOpts).
-    do_unsubscribe(Group, Topic, SubPid, SubOpts).
 
-do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
+-spec do_unsubscribe2(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) ->
+    ok.
+do_unsubscribe2(Topic, SubPid, SubOpts) when
+    is_binary(Topic), is_pid(SubPid), is_map(SubOpts)
+->
+    _ = emqx_broker_helper:reclaim_seq(Topic),
     case maps:get(shard, SubOpts, 0) of
         0 ->
             true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}),
@@ -205,7 +213,9 @@ do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
             true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}),
             cast(pick({Topic, I}), {unsubscribed, Topic, I})
     end;
-do_unsubscribe(Group, Topic, SubPid, _SubOpts) ->
+do_unsubscribe2(#share{group = Group, topic = Topic}, SubPid, _SubOpts) when
+    is_binary(Group), is_binary(Topic), is_pid(SubPid)
+->
     emqx_shared_sub:unsubscribe(Group, Topic, SubPid).
 
 %%--------------------------------------------------------------------
@@ -306,7 +316,9 @@ aggre([], true, Acc) ->
     lists:usort(Acc).
 
 %% @doc Forward message to another node.
--spec forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode :: sync | async) ->
+-spec forward(
+    node(), emqx_types:topic() | emqx_types:share(), emqx_types:delivery(), RpcMode :: sync | async
+) ->
     emqx_types:deliver_result().
 forward(Node, To, Delivery, async) ->
     true = emqx_broker_proto_v1:forward_async(Node, To, Delivery),
@@ -329,7 +341,8 @@ forward(Node, To, Delivery, sync) ->
             Result
     end.
 
--spec dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result().
+-spec dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) ->
+    emqx_types:deliver_result().
 dispatch(Topic, Delivery = #delivery{}) when is_binary(Topic) ->
     case emqx:is_running() of
         true ->
@@ -353,7 +366,11 @@ inc_dropped_cnt(Msg) ->
     end.
 
 -compile({inline, [subscribers/1]}).
--spec subscribers(emqx_types:topic() | {shard, emqx_types:topic(), non_neg_integer()}) ->
+-spec subscribers(
+    emqx_types:topic()
+    | emqx_types:share()
+    | {shard, emqx_types:topic() | emqx_types:share(), non_neg_integer()}
+) ->
     [pid()].
 subscribers(Topic) when is_binary(Topic) ->
     lookup_value(?SUBSCRIBER, Topic, []);
@@ -372,7 +389,7 @@ subscriber_down(SubPid) ->
                 SubOpts when is_map(SubOpts) ->
                     _ = emqx_broker_helper:reclaim_seq(Topic),
                     true = ets:delete(?SUBOPTION, {Topic, SubPid}),
-                    do_unsubscribe(undefined, Topic, SubPid, SubOpts);
+                    do_unsubscribe2(Topic, SubPid, SubOpts);
                 undefined ->
                     ok
             end
@@ -386,7 +403,7 @@ subscriber_down(SubPid) ->
 %%--------------------------------------------------------------------
 
 -spec subscriptions(pid() | emqx_types:subid()) ->
-    [{emqx_types:topic(), emqx_types:subopts()}].
+    [{emqx_types:topic() | emqx_types:share(), emqx_types:subopts()}].
 subscriptions(SubPid) when is_pid(SubPid) ->
     [
         {Topic, lookup_value(?SUBOPTION, {Topic, SubPid}, #{})}
@@ -400,20 +417,22 @@ subscriptions(SubId) ->
             []
     end.
 
--spec subscriptions_via_topic(emqx_types:topic()) -> [emqx_types:subopts()].
+-spec subscriptions_via_topic(emqx_types:topic() | emqx_types:share()) -> [emqx_types:subopts()].
 subscriptions_via_topic(Topic) ->
     MatchSpec = [{{{Topic, '_'}, '_'}, [], ['$_']}],
     ets:select(?SUBOPTION, MatchSpec).
 
--spec subscribed(pid() | emqx_types:subid(), emqx_types:topic()) -> boolean().
+-spec subscribed(
+    pid() | emqx_types:subid(), emqx_types:topic() | emqx_types:share()
+) -> boolean().
 subscribed(SubPid, Topic) when is_pid(SubPid) ->
     ets:member(?SUBOPTION, {Topic, SubPid});
 subscribed(SubId, Topic) when ?IS_SUBID(SubId) ->
     SubPid = emqx_broker_helper:lookup_subpid(SubId),
     ets:member(?SUBOPTION, {Topic, SubPid}).
 
--spec get_subopts(pid(), emqx_types:topic()) -> maybe(emqx_types:subopts()).
+-spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> maybe(emqx_types:subopts()).
-get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) ->
+get_subopts(SubPid, Topic) when is_pid(SubPid), ?IS_TOPIC(Topic) ->
     lookup_value(?SUBOPTION, {Topic, SubPid});
 get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->
     case emqx_broker_helper:lookup_subpid(SubId) of
@@ -423,7 +442,7 @@ get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->
             undefined
     end.
 
--spec set_subopts(emqx_types:topic(), emqx_types:subopts()) -> boolean().
+-spec set_subopts(emqx_types:topic() | emqx_types:share(), emqx_types:subopts()) -> boolean().
 set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) ->
     set_subopts(self(), Topic, NewOpts).
 
@@ -437,7 +456,7 @@ set_subopts(SubPid, Topic, NewOpts) ->
             false
     end.
 
--spec topics() -> [emqx_types:topic()].
+-spec topics() -> [emqx_types:topic() | emqx_types:share()].
 topics() ->
     emqx_router:topics().
 
@@ -542,7 +561,8 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
--spec do_dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result().
+-spec do_dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) ->
+    emqx_types:deliver_result().
 do_dispatch(Topic, #delivery{message = Msg}) ->
     DispN = lists:foldl(
         fun(Sub, N) ->
@@ -560,6 +580,8 @@ do_dispatch(Topic, #delivery{message = Msg}) ->
             {ok, DispN}
     end.
 
+%% Donot dispatch to share subscriber here.
+%% we do it in `emqx_shared_sub.erl` with configured strategy
 do_dispatch(SubPid, Topic, Msg) when is_pid(SubPid) ->
     case erlang:is_process_alive(SubPid) of
         true ->

apps/emqx/src/emqx_channel.erl

@@ -476,60 +476,27 @@ handle_in(
             ok = emqx_metrics:inc('packets.pubcomp.missed'),
             {ok, Channel}
     end;
-handle_in(
+handle_in(SubPkt = ?SUBSCRIBE_PACKET(PacketId, _Properties, _TopicFilters0), Channel0) ->
-    SubPkt = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
+    Pipe = pipeline(
-    Channel = #channel{clientinfo = ClientInfo}
+        [
-) ->
+            fun check_subscribe/2,
-    case emqx_packet:check(SubPkt) of
+            fun enrich_subscribe/2,
-        ok ->
+            %% TODO && FIXME (EMQX-10786): mount topic before authz check.
-            TopicFilters0 = parse_topic_filters(TopicFilters),
+            fun check_sub_authzs/2,
-            TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0),
+            fun check_sub_caps/2
-            TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel),
-            HasAuthzDeny = lists:any(
-                fun({_TopicFilter, ReasonCode}) ->
-                    ReasonCode =:= ?RC_NOT_AUTHORIZED
-                end,
-                TupleTopicFilters0
-            ),
-            DenyAction = emqx:get_config([authorization, deny_action], ignore),
-            case DenyAction =:= disconnect andalso HasAuthzDeny of
-                true ->
-                    handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel);
-                false ->
-                    TopicFilters2 = [
-                        TopicFilter
-                     || {TopicFilter, ?RC_SUCCESS} <- TupleTopicFilters0
         ],
-                    TopicFilters3 = run_hooks(
+        SubPkt,
-                        'client.subscribe',
+        Channel0
-                        [ClientInfo, Properties],
-                        TopicFilters2
     ),
-                    {TupleTopicFilters1, NChannel} = process_subscribe(
+    case Pipe of
-                        TopicFilters3,
+        {ok, NPkt = ?SUBSCRIBE_PACKET(_PacketId, TFChecked), Channel} ->
-                        Properties,
+            {TFSubedWithNRC, NChannel} = process_subscribe(run_sub_hooks(NPkt, Channel), Channel),
-                        Channel
+            ReasonCodes = gen_reason_codes(TFChecked, TFSubedWithNRC),
-                    ),
+            handle_out(suback, {PacketId, ReasonCodes}, NChannel);
-                    TupleTopicFilters2 =
+        {error, {disconnect, RC}, Channel} ->
-                        lists:foldl(
+            %% funcs in pipeline always cause action: `disconnect`
-                            fun
+            %% And Only one ReasonCode in DISCONNECT packet
-                                ({{Topic, Opts = #{deny_subscription := true}}, _QoS}, Acc) ->
+            handle_out(disconnect, RC, Channel)
-                                    Key = {Topic, maps:without([deny_subscription], Opts)},
-                                    lists:keyreplace(Key, 1, Acc, {Key, ?RC_UNSPECIFIED_ERROR});
-                                (Tuple = {Key, _Value}, Acc) ->
-                                    lists:keyreplace(Key, 1, Acc, Tuple)
-                            end,
-                            TupleTopicFilters0,
-                            TupleTopicFilters1
-                        ),
-                    ReasonCodes2 = [
-                        ReasonCode
-                     || {_TopicFilter, ReasonCode} <- TupleTopicFilters2
-                    ],
-                    handle_out(suback, {PacketId, ReasonCodes2}, NChannel)
-            end;
-        {error, ReasonCode} ->
-            handle_out(disconnect, ReasonCode, Channel)
     end;
 handle_in(
     Packet = ?UNSUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
@@ -540,7 +507,7 @@ handle_in(
             TopicFilters1 = run_hooks(
                 'client.unsubscribe',
                 [ClientInfo, Properties],
-                parse_topic_filters(TopicFilters)
+                parse_raw_topic_filters(TopicFilters)
             ),
             {ReasonCodes, NChannel} = process_unsubscribe(TopicFilters1, Properties, Channel),
             handle_out(unsuback, {PacketId, ReasonCodes}, NChannel);
@@ -782,32 +749,14 @@ after_message_acked(ClientInfo, Msg, PubAckProps) ->
 %% Process Subscribe
 %%--------------------------------------------------------------------
 
--compile({inline, [process_subscribe/3]}).
+process_subscribe(TopicFilters, Channel) ->
-process_subscribe(TopicFilters, SubProps, Channel) ->
+    process_subscribe(TopicFilters, Channel, []).
-    process_subscribe(TopicFilters, SubProps, Channel, []).
 
-process_subscribe([], _SubProps, Channel, Acc) ->
+process_subscribe([], Channel, Acc) ->
     {lists:reverse(Acc), Channel};
-process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Acc) ->
+process_subscribe([Filter = {TopicFilter, SubOpts} | More], Channel, Acc) ->
-    case check_sub_caps(TopicFilter, SubOpts, Channel) of
+    {NReasonCode, NChannel} = do_subscribe(TopicFilter, SubOpts, Channel),
-        ok ->
+    process_subscribe(More, NChannel, [{Filter, NReasonCode} | Acc]).
-            {ReasonCode, NChannel} = do_subscribe(
-                TopicFilter,
-                SubOpts#{sub_props => SubProps},
-                Channel
-            ),
-            process_subscribe(More, SubProps, NChannel, [{Topic, ReasonCode} | Acc]);
-        {error, ReasonCode} ->
-            ?SLOG(
-                warning,
-                #{
-                    msg => "cannot_subscribe_topic_filter",
-                    reason => emqx_reason_codes:name(ReasonCode)
-                },
-                #{topic => TopicFilter}
-            ),
-            process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc])
-    end.
 
 do_subscribe(
     TopicFilter,
@@ -818,11 +767,13 @@ do_subscribe(
             session = Session
         }
 ) ->
+    %% TODO && FIXME (EMQX-10786): mount topic before authz check.
     NTopicFilter = emqx_mountpoint:mount(MountPoint, TopicFilter),
-    NSubOpts = enrich_subopts(maps:merge(?DEFAULT_SUBOPTS, SubOpts), Channel),
+    case emqx_session:subscribe(ClientInfo, NTopicFilter, SubOpts, Session) of
-    case emqx_session:subscribe(ClientInfo, NTopicFilter, NSubOpts, Session) of
         {ok, NSession} ->
-            {QoS, Channel#channel{session = NSession}};
+            %% TODO && FIXME (EMQX-11216): QoS as ReasonCode(max granted QoS) for now
+            RC = QoS,
+            {RC, Channel#channel{session = NSession}};
         {error, RC} ->
             ?SLOG(
                 warning,
@@ -835,6 +786,30 @@ do_subscribe(
             {RC, Channel}
     end.
 
+gen_reason_codes(TFChecked, TFSubedWitNhRC) ->
+    do_gen_reason_codes([], TFChecked, TFSubedWitNhRC).
+
+%% Initial RC is `RC_SUCCESS | RC_NOT_AUTHORIZED`, generated by check_sub_authzs/2
+%% And then TF with `RC_SUCCESS` will passing through `process_subscribe/2` and
+%% NRC should override the initial RC.
+do_gen_reason_codes(Acc, [], []) ->
+    lists:reverse(Acc);
+do_gen_reason_codes(
+    Acc,
+    [{_, ?RC_SUCCESS} | RestTF],
+    [{_, NRC} | RestWithNRC]
+) ->
+    %% will passing through `process_subscribe/2`
+    %% use NRC to override IintialRC
+    do_gen_reason_codes([NRC | Acc], RestTF, RestWithNRC);
+do_gen_reason_codes(
+    Acc,
+    [{_, InitialRC} | Rest],
+    RestWithNRC
+) ->
+    %% InitialRC is not `RC_SUCCESS`, use it.
+    do_gen_reason_codes([InitialRC | Acc], Rest, RestWithNRC).
+
 %%--------------------------------------------------------------------
 %% Process Unsubscribe
 %%--------------------------------------------------------------------
@@ -1213,13 +1188,8 @@ handle_call(Req, Channel) ->
     ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}.
 
 handle_info({subscribe, TopicFilters}, Channel) ->
-    {_, NChannel} = lists:foldl(
+    NTopicFilters = enrich_subscribe(TopicFilters, Channel),
-        fun({TopicFilter, SubOpts}, {_, ChannelAcc}) ->
+    {_TopicFiltersWithRC, NChannel} = process_subscribe(NTopicFilters, Channel),
-            do_subscribe(TopicFilter, SubOpts, ChannelAcc)
-        end,
-        {[], Channel},
-        parse_topic_filters(TopicFilters)
-    ),
     {ok, NChannel};
 handle_info({unsubscribe, TopicFilters}, Channel) ->
     {_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel),
@@ -1857,49 +1827,156 @@ check_pub_caps(
 ) ->
     emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic => Topic}).
 
+%%--------------------------------------------------------------------
+%% Check Subscribe Packet
+
+check_subscribe(SubPkt, _Channel) ->
+    case emqx_packet:check(SubPkt) of
+        ok -> ok;
+        {error, RC} -> {error, {disconnect, RC}}
+    end.
+
 %%--------------------------------------------------------------------
 %% Check Sub Authorization
 
-check_sub_authzs(TopicFilters, Channel) ->
-    check_sub_authzs(TopicFilters, Channel, []).
-
 check_sub_authzs(
-    [TopicFilter = {Topic, _} | More],
+    ?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters0),
-    Channel = #channel{clientinfo = ClientInfo},
+    Channel = #channel{clientinfo = ClientInfo}
-    Acc
 ) ->
+    CheckResult = do_check_sub_authzs(TopicFilters0, ClientInfo),
+    HasAuthzDeny = lists:any(
+        fun({{_TopicFilter, _SubOpts}, ReasonCode}) ->
+            ReasonCode =:= ?RC_NOT_AUTHORIZED
+        end,
+        CheckResult
+    ),
+    DenyAction = emqx:get_config([authorization, deny_action], ignore),
+    case DenyAction =:= disconnect andalso HasAuthzDeny of
+        true ->
+            {error, {disconnect, ?RC_NOT_AUTHORIZED}, Channel};
+        false ->
+            {ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}
+    end.
+
+do_check_sub_authzs(TopicFilters, ClientInfo) ->
+    do_check_sub_authzs(ClientInfo, TopicFilters, []).
+
+do_check_sub_authzs(_ClientInfo, [], Acc) ->
+    lists:reverse(Acc);
+do_check_sub_authzs(ClientInfo, [TopicFilter = {Topic, _SubOpts} | More], Acc) ->
+    %% subsclibe authz check only cares the real topic filter when shared-sub
+    %% e.g. only check <<"t/#">> for <<"$share/g/t/#">>
     Action = authz_action(TopicFilter),
-    case emqx_access_control:authorize(ClientInfo, Action, Topic) of
+    case
+        emqx_access_control:authorize(
+            ClientInfo,
+            Action,
+            emqx_topic:get_shared_real_topic(Topic)
+        )
+    of
+        %% TODO: support maximum QoS granted
+        %% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7]
+        %% Not implemented yet:
+        %% {allow, RC} -> do_check_sub_authzs(ClientInfo, More, [{TopicFilter, RC} | Acc]);
         allow ->
-            check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]);
+            do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_SUCCESS} | Acc]);
         deny ->
-            check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
+            do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
-    end;
+    end.
-check_sub_authzs([], _Channel, Acc) ->
-    lists:reverse(Acc).
 
 %%--------------------------------------------------------------------
 %% Check Sub Caps
 
-check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = ClientInfo}) ->
+check_sub_caps(
-    emqx_mqtt_caps:check_sub(ClientInfo, TopicFilter, SubOpts).
+    ?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters),
+    Channel = #channel{clientinfo = ClientInfo}
+) ->
+    CheckResult = do_check_sub_caps(ClientInfo, TopicFilters),
+    {ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}.
+
+do_check_sub_caps(ClientInfo, TopicFilters) ->
+    do_check_sub_caps(ClientInfo, TopicFilters, []).
+
+do_check_sub_caps(_ClientInfo, [], Acc) ->
+    lists:reverse(Acc);
+do_check_sub_caps(ClientInfo, [TopicFilter = {{Topic, SubOpts}, ?RC_SUCCESS} | More], Acc) ->
+    case emqx_mqtt_caps:check_sub(ClientInfo, Topic, SubOpts) of
+        ok ->
+            do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]);
+        {error, NRC} ->
+            ?SLOG(
+                warning,
+                #{
+                    msg => "cannot_subscribe_topic_filter",
+                    reason => emqx_reason_codes:name(NRC)
+                },
+                #{topic => Topic}
+            ),
+            do_check_sub_caps(ClientInfo, More, [{{Topic, SubOpts}, NRC} | Acc])
+    end;
+do_check_sub_caps(ClientInfo, [TopicFilter = {{_Topic, _SubOpts}, _OtherRC} | More], Acc) ->
+    do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]).
 
 %%--------------------------------------------------------------------
-%% Enrich SubId
+%% Run Subscribe Hooks
 
-enrich_subopts_subid(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
+run_sub_hooks(
-    [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
+    ?SUBSCRIBE_PACKET(_PacketId, Properties, TopicFilters0),
-enrich_subopts_subid(_Properties, TopicFilters) ->
+    _Channel = #channel{clientinfo = ClientInfo}
-    TopicFilters.
+) ->
+    TopicFilters = [
+        TopicFilter
+     || {TopicFilter, ?RC_SUCCESS} <- TopicFilters0
+    ],
+    _NTopicFilters = run_hooks('client.subscribe', [ClientInfo, Properties], TopicFilters).
 
 %%--------------------------------------------------------------------
 %% Enrich SubOpts
 
-enrich_subopts(SubOpts, _Channel = ?IS_MQTT_V5) ->
+%% for api subscribe without sub-authz check and sub-caps check.
-    SubOpts;
+enrich_subscribe(TopicFilters, Channel) when is_list(TopicFilters) ->
-enrich_subopts(SubOpts, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) ->
+    do_enrich_subscribe(#{}, TopicFilters, Channel);
+%% for mqtt clients sent subscribe packet.
+enrich_subscribe(?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), Channel) ->
+    NTopicFilters = do_enrich_subscribe(Properties, TopicFilters, Channel),
+    {ok, ?SUBSCRIBE_PACKET(PacketId, Properties, NTopicFilters), Channel}.
+
+do_enrich_subscribe(Properties, TopicFilters, Channel) ->
+    _NTopicFilters = run_fold(
+        [
+            %% TODO: do try catch with reason code here
+            fun(TFs, _) -> parse_raw_topic_filters(TFs) end,
+            fun enrich_subopts_subid/2,
+            fun enrich_subopts_porps/2,
+            fun enrich_subopts_flags/2
+        ],
+        TopicFilters,
+        #{sub_props => Properties, channel => Channel}
+    ).
+
+enrich_subopts_subid(TopicFilters, #{sub_props := #{'Subscription-Identifier' := SubId}}) ->
+    [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
+enrich_subopts_subid(TopicFilters, _State) ->
+    TopicFilters.
+
+enrich_subopts_porps(TopicFilters, #{sub_props := SubProps}) ->
+    [{Topic, SubOpts#{sub_props => SubProps}} || {Topic, SubOpts} <- TopicFilters].
+
+enrich_subopts_flags(TopicFilters, #{channel := Channel}) ->
+    do_enrich_subopts_flags(TopicFilters, Channel).
+
+do_enrich_subopts_flags(TopicFilters, ?IS_MQTT_V5) ->
+    [{Topic, merge_default_subopts(SubOpts)} || {Topic, SubOpts} <- TopicFilters];
+do_enrich_subopts_flags(TopicFilters, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) ->
+    Rap = flag(IsBridge),
     NL = flag(get_mqtt_conf(Zone, ignore_loop_deliver)),
-    SubOpts#{rap => flag(IsBridge), nl => NL}.
+    [
+        {Topic, (merge_default_subopts(SubOpts))#{rap => Rap, nl => NL}}
+     || {Topic, SubOpts} <- TopicFilters
+    ].
+
+merge_default_subopts(SubOpts) ->
+    maps:merge(?DEFAULT_SUBOPTS, SubOpts).
 
 %%--------------------------------------------------------------------
 %% Enrich ConnAck Caps
@@ -2089,8 +2166,8 @@ maybe_shutdown(Reason, _Intent = shutdown, Channel) ->
 %%--------------------------------------------------------------------
 %% Parse Topic Filters
 
--compile({inline, [parse_topic_filters/1]}).
+%% [{<<"$share/group/topic">>, _SubOpts = #{}} | _]
-parse_topic_filters(TopicFilters) ->
+parse_raw_topic_filters(TopicFilters) ->
     lists:map(fun emqx_topic:parse/1, TopicFilters).
 
 %%--------------------------------------------------------------------

apps/emqx/src/emqx_mountpoint.erl

@@ -17,6 +17,7 @@
 -module(emqx_mountpoint).
 
 -include("emqx.hrl").
+-include("emqx_mqtt.hrl").
 -include("emqx_placeholder.hrl").
 -include("types.hrl").
 
@@ -34,38 +35,54 @@
 -spec mount(maybe(mountpoint()), Any) -> Any when
     Any ::
         emqx_types:topic()
+        | emqx_types:share()
         | emqx_types:message()
         | emqx_types:topic_filters().
 mount(undefined, Any) ->
     Any;
-mount(MountPoint, Topic) when is_binary(Topic) ->
+mount(MountPoint, Topic) when ?IS_TOPIC(Topic) ->
-    prefix(MountPoint, Topic);
+    prefix_maybe_share(MountPoint, Topic);
-mount(MountPoint, Msg = #message{topic = Topic}) ->
+mount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
-    Msg#message{topic = prefix(MountPoint, Topic)};
+    Msg#message{topic = prefix_maybe_share(MountPoint, Topic)};
 mount(MountPoint, TopicFilters) when is_list(TopicFilters) ->
-    [{prefix(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
+    [{prefix_maybe_share(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
 
-%% @private
+-spec prefix_maybe_share(maybe(mountpoint()), Any) -> Any when
--compile({inline, [prefix/2]}).
+    Any ::
-prefix(MountPoint, Topic) ->
+        emqx_types:topic()
-    <<MountPoint/binary, Topic/binary>>.
+        | emqx_types:share().
+prefix_maybe_share(MountPoint, Topic) when
+    is_binary(MountPoint) andalso is_binary(Topic)
+->
+    <<MountPoint/binary, Topic/binary>>;
+prefix_maybe_share(MountPoint, #share{group = Group, topic = Topic}) when
+    is_binary(MountPoint) andalso is_binary(Topic)
+->
+    #share{group = Group, topic = prefix_maybe_share(MountPoint, Topic)}.
 
 -spec unmount(maybe(mountpoint()), Any) -> Any when
     Any ::
         emqx_types:topic()
+        | emqx_types:share()
         | emqx_types:message().
 unmount(undefined, Any) ->
     Any;
-unmount(MountPoint, Topic) when is_binary(Topic) ->
+unmount(MountPoint, Topic) when ?IS_TOPIC(Topic) ->
+    unmount_maybe_share(MountPoint, Topic);
+unmount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
+    Msg#message{topic = unmount_maybe_share(MountPoint, Topic)}.
+
+unmount_maybe_share(MountPoint, Topic) when
+    is_binary(MountPoint) andalso is_binary(Topic)
+->
     case string:prefix(Topic, MountPoint) of
         nomatch -> Topic;
         Topic1 -> Topic1
     end;
-unmount(MountPoint, Msg = #message{topic = Topic}) ->
+unmount_maybe_share(MountPoint, TopicFilter = #share{topic = Topic}) when
-    case string:prefix(Topic, MountPoint) of
+    is_binary(MountPoint) andalso is_binary(Topic)
-        nomatch -> Msg;
+->
-        Topic1 -> Msg#message{topic = Topic1}
+    TopicFilter#share{topic = unmount_maybe_share(MountPoint, Topic)}.
-    end.
 
 -spec replvar(maybe(mountpoint()), map()) -> maybe(mountpoint()).
 replvar(undefined, _Vars) ->

apps/emqx/src/emqx_mqtt_caps.erl

@@ -102,16 +102,19 @@ do_check_pub(_Flags, _Caps) ->
 
 -spec check_sub(
     emqx_types:clientinfo(),
-    emqx_types:topic(),
+    emqx_types:topic() | emqx_types:share(),
     emqx_types:subopts()
 ) ->
     ok_or_error(emqx_types:reason_code()).
 check_sub(ClientInfo = #{zone := Zone}, Topic, SubOpts) ->
     Caps = emqx_config:get_zone_conf(Zone, [mqtt]),
     Flags = #{
+        %% TODO: qos check
+        %% (max_qos_allowed, Map) ->
+        %% max_qos_allowed => maps:get(max_qos_allowed, Caps, 2),
         topic_levels => emqx_topic:levels(Topic),
         is_wildcard => emqx_topic:wildcard(Topic),
-        is_shared => maps:is_key(share, SubOpts),
+        is_shared => erlang:is_record(Topic, share),
         is_exclusive => maps:get(is_exclusive, SubOpts, false)
     },
     do_check_sub(Flags, Caps, ClientInfo, Topic).
@@ -126,13 +129,19 @@ do_check_sub(#{is_shared := true}, #{shared_subscription := false}, _, _) ->
     {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED};
 do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := false}, _, _) ->
     {error, ?RC_TOPIC_FILTER_INVALID};
-do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) ->
+do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) when
+    is_binary(Topic)
+->
     case emqx_exclusive_subscription:check_subscribe(ClientInfo, Topic) of
         deny ->
             {error, ?RC_QUOTA_EXCEEDED};
         _ ->
             ok
     end;
+%% for max_qos_allowed
+%% see: RC_GRANTED_QOS_0, RC_GRANTED_QOS_1, RC_GRANTED_QOS_2
+%% do_check_sub(_, _) ->
+%%     {ok, RC};
 do_check_sub(_Flags, _Caps, _, _) ->
     ok.
 

apps/emqx/src/emqx_reason_codes.erl

@@ -177,6 +177,7 @@ compat(connack, 16#9D) -> ?CONNACK_SERVER;
 compat(connack, 16#9F) -> ?CONNACK_SERVER;
 compat(suback, Code) when Code =< ?QOS_2 -> Code;
 compat(suback, Code) when Code >= 16#80 -> 16#80;
+%% TODO: 16#80(qos0) 16#81(qos1) 16#82(qos2) for mqtt-v3.1.1
 compat(unsuback, _Code) -> undefined;
 compat(_Other, _Code) -> undefined.
 

apps/emqx/src/emqx_schema_secret.erl

@@ -0,0 +1,85 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc HOCON schema that defines _secret_ concept.
+-module(emqx_schema_secret).
+
+-include_lib("emqx/include/logger.hrl").
+-include_lib("typerefl/include/types.hrl").
+
+-export([mk/1]).
+
+%% HOCON Schema API
+-export([convert_secret/2]).
+
+%% @doc Secret value.
+-type t() :: binary().
+
+%% @doc Source of the secret value.
+%% * "file://...": file path to a file containing secret value.
+%% * other binaries: secret value itself.
+-type source() :: iodata().
+
+-type secret() :: binary() | function().
+-reflect_type([secret/0]).
+
+-define(SCHEMA, #{
+    required => false,
+    format => <<"password">>,
+    sensitive => true,
+    converter => fun ?MODULE:convert_secret/2
+}).
+
+-dialyzer({nowarn_function, source/1}).
+
+%%
+
+-spec mk(#{atom() => _}) -> hocon_schema:field_schema().
+mk(Overrides = #{}) ->
+    hoconsc:mk(secret(), maps:merge(?SCHEMA, Overrides)).
+
+convert_secret(undefined, #{}) ->
+    undefined;
+convert_secret(Secret, #{make_serializable := true}) ->
+    unicode:characters_to_binary(source(Secret));
+convert_secret(Secret, #{}) when is_function(Secret, 0) ->
+    Secret;
+convert_secret(Secret, #{}) when is_integer(Secret) ->
+    wrap(integer_to_binary(Secret));
+convert_secret(Secret, #{}) ->
+    try unicode:characters_to_binary(Secret) of
+        String when is_binary(String) ->
+            wrap(String);
+        {error, _, _} ->
+            throw(invalid_string)
+    catch
+        error:_ ->
+            throw(invalid_type)
+    end.
+
+-spec wrap(source()) -> emqx_secret:t(t()).
+wrap(<<"file://", Filename/binary>>) ->
+    emqx_secret:wrap_load({file, Filename});
+wrap(Secret) ->
+    emqx_secret:wrap(Secret).
+
+-spec source(emqx_secret:t(t())) -> source().
+source(Secret) when is_function(Secret) ->
+    source(emqx_secret:term(Secret));
+source({file, Filename}) ->
+    <<"file://", Filename/binary>>;
+source(Secret) ->
+    Secret.

apps/emqx/src/emqx_secret.erl

@@ -19,23 +19,52 @@
 -module(emqx_secret).
 
 %% API:
--export([wrap/1, unwrap/1]).
+-export([wrap/1, wrap_load/1, unwrap/1, term/1]).
 
 -export_type([t/1]).
 
 -opaque t(T) :: T | fun(() -> t(T)).
 
+%% Secret loader module.
+%% Any changes related to processing of secrets should be made there.
+-define(LOADER, emqx_secret_loader).
+
 %%================================================================================
 %% API funcions
 %%================================================================================
 
+%% @doc Wrap a term in a secret closure.
+%% This effectively hides the term from any term formatting / printing code.
+-spec wrap(T) -> t(T).
 wrap(Term) ->
     fun() ->
         Term
     end.
 
+%% @doc Wrap a loader function call over a term in a secret closure.
+%% This is slightly more flexible form of `wrap/1` with the same basic purpose.
+-spec wrap_load(emqx_secret_loader:source()) -> t(_).
+wrap_load(Source) ->
+    fun() ->
+        apply(?LOADER, load, [Source])
+    end.
+
+%% @doc Unwrap a secret closure, revealing the secret.
+%% This is either `Term` or `Module:Function(Term)` depending on how it was wrapped.
+-spec unwrap(t(T)) -> T.
 unwrap(Term) when is_function(Term, 0) ->
     %% Handle potentially nested funs
     unwrap(Term());
 unwrap(Term) ->
     Term.
+
+%% @doc Inspect the term wrapped in a secret closure.
+-spec term(t(_)) -> _Term.
+term(Wrap) when is_function(Wrap, 0) ->
+    case erlang:fun_info(Wrap, module) of
+        {module, ?MODULE} ->
+            {env, Env} = erlang:fun_info(Wrap, env),
+            lists:last(Env);
+        _ ->
+            error(badarg, [Wrap])
+    end.

apps/emqx/src/emqx_secret_loader.erl

@@ -0,0 +1,42 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_secret_loader).
+
+%% API
+-export([load/1]).
+-export([file/1]).
+
+-export_type([source/0]).
+
+-type source() :: {file, file:filename_all()}.
+
+-spec load(source()) -> binary() | no_return().
+load({file, Filename}) ->
+    file(Filename).
+
+-spec file(file:filename_all()) -> binary() | no_return().
+file(Filename) ->
+    case file:read_file(Filename) of
+        {ok, Secret} ->
+            string:trim(Secret, trailing);
+        {error, Reason} ->
+            throw(#{
+                msg => failed_to_read_secret_file,
+                path => Filename,
+                reason => emqx_utils:explain_posix(Reason)
+            })
+    end.

apps/emqx/src/emqx_session.erl

@@ -267,7 +267,7 @@ destroy(Session) ->
 
 -spec subscribe(
     clientinfo(),
-    emqx_types:topic(),
+    emqx_types:topic() | emqx_types:share(),
     emqx_types:subopts(),
     t()
 ) ->
@@ -287,7 +287,7 @@ subscribe(ClientInfo, TopicFilter, SubOpts, Session) ->
 
 -spec unsubscribe(
     clientinfo(),
-    emqx_types:topic(),
+    emqx_types:topic() | emqx_types:share(),
     emqx_types:subopts(),
     t()
 ) ->
@@ -418,7 +418,13 @@ enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) ->
     end.
 
 enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) ->
-    SubOpts = ?IMPL(Session):get_subscription(Topic, Session),
+    SubOpts =
+        case Msg of
+            #message{headers = #{redispatch_to := ?REDISPATCH_TO(Group, T)}} ->
+                ?IMPL(Session):get_subscription(emqx_topic:make_shared_record(Group, T), Session);
+            _ ->
+                ?IMPL(Session):get_subscription(Topic, Session)
+        end,
     enrich_message(ClientInfo, Msg, SubOpts, UpgradeQoS).
 
 enrich_message(

apps/emqx/src/emqx_session_mem.erl

@@ -316,7 +316,7 @@ unsubscribe(
             {error, ?RC_NO_SUBSCRIPTION_EXISTED}
     end.
 
--spec get_subscription(emqx_types:topic(), session()) ->
+-spec get_subscription(emqx_types:topic() | emqx_types:share(), session()) ->
     emqx_types:subopts() | undefined.
 get_subscription(Topic, #session{subscriptions = Subs}) ->
     maps:get(Topic, Subs, undefined).

apps/emqx/src/emqx_shared_sub.erl

@@ -95,7 +95,6 @@
 -define(ACK, shared_sub_ack).
 -define(NACK(Reason), {shared_sub_nack, Reason}).
 -define(NO_ACK, no_ack).
--define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
 -define(SUBSCRIBER_DOWN, noproc).
 
 -type redispatch_to() :: ?REDISPATCH_TO(emqx_types:group(), emqx_types:topic()).
@@ -234,19 +233,16 @@ without_group_ack(Msg) ->
 get_group_ack(Msg) ->
     emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK).
 
-with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) ->
+%% always add `redispatch_to` header to the message
-    Msg;
+%% for QOS_0 msgs, redispatch_to is not needed and filtered out in is_redispatch_needed/1
 with_redispatch_to(Msg, Group, Topic) ->
     emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg).
 
-%% @hidden Redispatch is needed only for the messages with redispatch_to header added.
+%% @hidden Redispatch is needed only for the messages which not QOS_0
-is_redispatch_needed(#message{} = Msg) ->
+is_redispatch_needed(#message{qos = ?QOS_0}) ->
-    case get_redispatch_to(Msg) of
+    false;
-        ?REDISPATCH_TO(_, _) ->
+is_redispatch_needed(#message{headers = #{redispatch_to := ?REDISPATCH_TO(_, _)}}) ->
-            true;
+    true.
-        _ ->
-            false
-    end.
 
 %% @doc Redispatch shared deliveries to other members in the group.
 redispatch(Messages0) ->

apps/emqx/src/emqx_topic.erl

@@ -36,9 +36,16 @@
     parse/2
 ]).
 
+-export([
+    maybe_format_share/1,
+    get_shared_real_topic/1,
+    make_shared_record/2
+]).
+
 -type topic() :: emqx_types:topic().
 -type word() :: emqx_types:word().
 -type words() :: emqx_types:words().
+-type share() :: emqx_types:share().
 
 %% Guards
 -define(MULTI_LEVEL_WILDCARD_NOT_LAST(C, REST),
@@ -50,7 +57,9 @@
 %%--------------------------------------------------------------------
 
 %% @doc Is wildcard topic?
--spec wildcard(topic() | words()) -> true | false.
+-spec wildcard(topic() | share() | words()) -> true | false.
+wildcard(#share{topic = Topic}) when is_binary(Topic) ->
+    wildcard(Topic);
 wildcard(Topic) when is_binary(Topic) ->
     wildcard(words(Topic));
 wildcard([]) ->
@@ -64,7 +73,7 @@ wildcard([_H | T]) ->
 
 %% @doc Match Topic name with filter.
 -spec match(Name, Filter) -> boolean() when
-    Name :: topic() | words(),
+    Name :: topic() | share() | words(),
     Filter :: topic() | words().
 match(<<$$, _/binary>>, <<$+, _/binary>>) ->
     false;
@@ -72,6 +81,10 @@ match(<<$$, _/binary>>, <<$#, _/binary>>) ->
     false;
 match(Name, Filter) when is_binary(Name), is_binary(Filter) ->
     match(words(Name), words(Filter));
+match(#share{} = Name, Filter) ->
+    match_share(Name, Filter);
+match(Name, #share{} = Filter) ->
+    match_share(Name, Filter);
 match([], []) ->
     true;
 match([H | T1], [H | T2]) ->
@@ -87,12 +100,29 @@ match([_H1 | _], []) ->
 match([], [_H | _T2]) ->
     false.
 
+-spec match_share(Name, Filter) -> boolean() when
+    Name :: share(),
+    Filter :: topic() | share().
+match_share(#share{topic = Name}, Filter) when is_binary(Filter) ->
+    %% only match real topic filter for normal topic filter.
+    match(words(Name), words(Filter));
+match_share(#share{group = Group, topic = Name}, #share{group = Group, topic = Filter}) ->
+    %% Matching real topic filter When subed same share group.
+    match(words(Name), words(Filter));
+match_share(#share{}, _) ->
+    %% Otherwise, non-matched.
+    false;
+match_share(Name, #share{topic = Filter}) when is_binary(Name) ->
+    %% Only match real topic filter for normal topic_filter/topic_name.
+    match(Name, Filter).
+
 -spec match_any(Name, [Filter]) -> boolean() when
     Name :: topic() | words(),
     Filter :: topic() | words().
 match_any(Topic, Filters) ->
     lists:any(fun(Filter) -> match(Topic, Filter) end, Filters).
 
+%% TODO: validate share topic #share{} for emqx_trace.erl
 %% @doc Validate topic name or filter
 -spec validate(topic() | {name | filter, topic()}) -> true.
 validate(Topic) when is_binary(Topic) ->
@@ -107,7 +137,7 @@ validate(_, <<>>) ->
 validate(_, Topic) when is_binary(Topic) andalso (size(Topic) > ?MAX_TOPIC_LEN) ->
     %% MQTT-5.0 [MQTT-4.7.3-3]
     error(topic_too_long);
-validate(filter, SharedFilter = <<"$share/", _Rest/binary>>) ->
+validate(filter, SharedFilter = <<?SHARE, "/", _Rest/binary>>) ->
     validate_share(SharedFilter);
 validate(filter, Filter) when is_binary(Filter) ->
     validate2(words(Filter));
@@ -139,12 +169,12 @@ validate3(<<C/utf8, _Rest/binary>>) when C == $#; C == $+; C == 0 ->
 validate3(<<_/utf8, Rest/binary>>) ->
     validate3(Rest).
 
-validate_share(<<"$share/", Rest/binary>>) when
+validate_share(<<?SHARE, "/", Rest/binary>>) when
     Rest =:= <<>> orelse Rest =:= <<"/">>
 ->
     %% MQTT-5.0 [MQTT-4.8.2-1]
     error(?SHARE_EMPTY_FILTER);
-validate_share(<<"$share/", Rest/binary>>) ->
+validate_share(<<?SHARE, "/", Rest/binary>>) ->
     case binary:split(Rest, <<"/">>) of
         %% MQTT-5.0 [MQTT-4.8.2-1]
         [<<>>, _] ->
@@ -156,7 +186,7 @@ validate_share(<<"$share/", Rest/binary>>) ->
             validate_share(ShareName, Filter)
     end.
 
-validate_share(_, <<"$share/", _Rest/binary>>) ->
+validate_share(_, <<?SHARE, "/", _Rest/binary>>) ->
     error(?SHARE_RECURSIVELY);
 validate_share(ShareName, Filter) ->
     case binary:match(ShareName, [<<"+">>, <<"#">>]) of
@@ -185,7 +215,9 @@ bin('#') -> <<"#">>;
 bin(B) when is_binary(B) -> B;
 bin(L) when is_list(L) -> list_to_binary(L).
 
--spec levels(topic()) -> pos_integer().
+-spec levels(topic() | share()) -> pos_integer().
+levels(#share{topic = Topic}) when is_binary(Topic) ->
+    levels(Topic);
 levels(Topic) when is_binary(Topic) ->
     length(tokens(Topic)).
 
@@ -197,6 +229,8 @@ tokens(Topic) ->
 
 %% @doc Split Topic Path to Words
 -spec words(topic()) -> words().
+words(#share{topic = Topic}) when is_binary(Topic) ->
+    words(Topic);
 words(Topic) when is_binary(Topic) ->
     [word(W) || W <- tokens(Topic)].
 
@@ -237,26 +271,29 @@ do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
 do_join(TopicAcc, [Word | Words]) ->
     do_join(<<TopicAcc/binary, "/", (bin(Word))/binary>>, Words).
 
--spec parse(topic() | {topic(), map()}) -> {topic(), #{share => binary()}}.
+-spec parse(topic() | {topic(), map()}) -> {topic() | share(), map()}.
 parse(TopicFilter) when is_binary(TopicFilter) ->
     parse(TopicFilter, #{});
 parse({TopicFilter, Options}) when is_binary(TopicFilter) ->
     parse(TopicFilter, Options).
 
--spec parse(topic(), map()) -> {topic(), map()}.
+-spec parse(topic() | share(), map()) -> {topic() | share(), map()}.
-parse(TopicFilter = <<"$queue/", _/binary>>, #{share := _Group}) ->
+%% <<"$queue/[real_topic_filter]>">> equivalent to <<"$share/$queue/[real_topic_filter]">>
-    error({invalid_topic_filter, TopicFilter});
+%% So the head of `real_topic_filter` MUST NOT be `<<$queue>>` or `<<$share>>`
-parse(TopicFilter = <<"$share/", _/binary>>, #{share := _Group}) ->
+parse(#share{topic = Topic = <<?QUEUE, "/", _/binary>>}, _Options) ->
-    error({invalid_topic_filter, TopicFilter});
+    error({invalid_topic_filter, Topic});
-parse(<<"$queue/", TopicFilter/binary>>, Options) ->
+parse(#share{topic = Topic = <<?SHARE, "/", _/binary>>}, _Options) ->
-    parse(TopicFilter, Options#{share => <<"$queue">>});
+    error({invalid_topic_filter, Topic});
-parse(TopicFilter = <<"$share/", Rest/binary>>, Options) ->
+parse(<<?QUEUE, "/", Topic/binary>>, Options) ->
+    parse(#share{group = <<?QUEUE>>, topic = Topic}, Options);
+parse(TopicFilter = <<?SHARE, "/", Rest/binary>>, Options) ->
     case binary:split(Rest, <<"/">>) of
         [_Any] ->
             error({invalid_topic_filter, TopicFilter});
-        [ShareName, Filter] ->
+        %% `Group` could be `$share` or `$queue`
-            case binary:match(ShareName, [<<"+">>, <<"#">>]) of
+        [Group, Topic] ->
-                nomatch -> parse(Filter, Options#{share => ShareName});
+            case binary:match(Group, [<<"+">>, <<"#">>]) of
+                nomatch -> parse(#share{group = Group, topic = Topic}, Options);
                 _ -> error({invalid_topic_filter, TopicFilter})
             end
     end;
@@ -267,5 +304,22 @@ parse(TopicFilter = <<"$exclusive/", Topic/binary>>, Options) ->
         _ ->
             {Topic, Options#{is_exclusive => true}}
     end;
-parse(TopicFilter, Options) ->
+parse(TopicFilter, Options) when
+    ?IS_TOPIC(TopicFilter)
+->
     {TopicFilter, Options}.
+
+get_shared_real_topic(#share{topic = TopicFilter}) ->
+    TopicFilter;
+get_shared_real_topic(TopicFilter) when is_binary(TopicFilter) ->
+    TopicFilter.
+
+make_shared_record(Group, Topic) ->
+    #share{group = Group, topic = Topic}.
+
+maybe_format_share(#share{group = <<?QUEUE>>, topic = Topic}) ->
+    join([<<?QUEUE>>, Topic]);
+maybe_format_share(#share{group = Group, topic = Topic}) ->
+    join([<<?SHARE>>, Group, Topic]);
+maybe_format_share(Topic) ->
+    join([Topic]).

apps/emqx/src/emqx_trace/emqx_trace.erl

@@ -105,7 +105,7 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
         ignore ->
             ignore;
         Log ->
-            case logger_config:get(ets:whereis(logger), Id) of
+            case logger_config:get(logger, Id) of
                 {ok, #{module := Module} = HandlerConfig0} ->
                     HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
                     try

apps/emqx/src/emqx_types.erl

@@ -40,6 +40,10 @@
     words/0
 ]).
 
+-export_type([
+    share/0
+]).
+
 -export_type([
     socktype/0,
     sockstate/0,
@@ -136,11 +140,14 @@
 
 -type subid() :: binary() | atom().
 
--type group() :: binary() | undefined.
+%% '_' for match spec
+-type group() :: binary() | '_'.
 -type topic() :: binary().
 -type word() :: '' | '+' | '#' | binary().
 -type words() :: list(word()).
 
+-type share() :: #share{}.
+
 -type socktype() :: tcp | udp | ssl | proxy | atom().
 -type sockstate() :: idle | running | blocked | closed.
 -type conninfo() :: #{
@@ -207,7 +214,6 @@
     rap := 0 | 1,
     nl := 0 | 1,
     qos := qos(),
-    share => binary(),
     atom() => term()
 }.
 -type reason_code() :: 0..16#FF.

apps/emqx/test/emqx_broker_SUITE.erl

@@ -299,14 +299,19 @@ t_nosub_pub(Config) when is_list(Config) ->
     ?assertEqual(1, emqx_metrics:val('messages.dropped')).
 
 t_shared_subscribe({init, Config}) ->
-    emqx_broker:subscribe(<<"topic">>, <<"clientid">>, #{share => <<"group">>}),
+    emqx_broker:subscribe(
+        emqx_topic:make_shared_record(<<"group">>, <<"topic">>), <<"clientid">>, #{}
+    ),
     ct:sleep(100),
     Config;
 t_shared_subscribe(Config) when is_list(Config) ->
     emqx_broker:safe_publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)),
     ?assert(
         receive
-            {deliver, <<"topic">>, #message{payload = <<"hello">>}} ->
+            {deliver, <<"topic">>, #message{
+                headers = #{redispatch_to := ?REDISPATCH_TO(<<"group">>, <<"topic">>)},
+                payload = <<"hello">>
+            }} ->
                 true;
             Msg ->
                 ct:pal("Msg: ~p", [Msg]),
@@ -316,7 +321,7 @@ t_shared_subscribe(Config) when is_list(Config) ->
         end
     );
 t_shared_subscribe({'end', _Config}) ->
-    emqx_broker:unsubscribe(<<"$share/group/topic">>).
+    emqx_broker:unsubscribe(emqx_topic:make_shared_record(<<"group">>, <<"topic">>)).
 
 t_shared_subscribe_2({init, Config}) ->
     Config;
@@ -723,24 +728,6 @@ t_connack_auth_error(Config) when is_list(Config) ->
     ?assertEqual(2, emqx_metrics:val('packets.connack.auth_error')),
     ok.
 
-t_handle_in_empty_client_subscribe_hook({init, Config}) ->
-    Hook = {?MODULE, client_subscribe_delete_all_hook, []},
-    ok = emqx_hooks:put('client.subscribe', Hook, _Priority = 100),
-    Config;
-t_handle_in_empty_client_subscribe_hook({'end', _Config}) ->
-    emqx_hooks:del('client.subscribe', {?MODULE, client_subscribe_delete_all_hook}),
-    ok;
-t_handle_in_empty_client_subscribe_hook(Config) when is_list(Config) ->
-    {ok, C} = emqtt:start_link(),
-    {ok, _} = emqtt:connect(C),
-    try
-        {ok, _, RCs} = emqtt:subscribe(C, <<"t">>),
-        ?assertEqual([?RC_UNSPECIFIED_ERROR], RCs),
-        ok
-    after
-        emqtt:disconnect(C)
-    end.
-
 authenticate_deny(_Credentials, _Default) ->
     {stop, {error, bad_username_or_password}}.
 
@@ -800,7 +787,3 @@ recv_msgs(Count, Msgs) ->
     after 100 ->
         Msgs
     end.
-
-client_subscribe_delete_all_hook(_ClientInfo, _Username, TopicFilter) ->
-    EmptyFilters = [{T, Opts#{deny_subscription => true}} || {T, Opts} <- TopicFilter],
-    {stop, EmptyFilters}.

apps/emqx/test/emqx_channel_SUITE.erl

@@ -456,7 +456,7 @@ t_process_subscribe(_) ->
     ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end),
     TopicFilters = [TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}],
     {[{TopicFilter, ?RC_SUCCESS}], _Channel} =
-        emqx_channel:process_subscribe(TopicFilters, #{}, channel()).
+        emqx_channel:process_subscribe(TopicFilters, channel()).
 
 t_process_unsubscribe(_) ->
     ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end),
@@ -914,7 +914,13 @@ t_check_pub_alias(_) ->
 t_check_sub_authzs(_) ->
     emqx_config:put_zone_conf(default, [authorization, enable], true),
     TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS},
-    [{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()).
+    SubPkt = ?SUBSCRIBE_PACKET(1, #{}, [TopicFilter]),
+    CheckedSubPkt = ?SUBSCRIBE_PACKET(1, #{}, [{TopicFilter, ?RC_SUCCESS}]),
+    Channel = channel(),
+    ?assertEqual(
+        {ok, CheckedSubPkt, Channel},
+        emqx_channel:check_sub_authzs(SubPkt, Channel)
+    ).
 
 t_enrich_connack_caps(_) ->
     ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]),
@@ -1061,6 +1067,7 @@ clientinfo(InitProps) ->
             clientid => <<"clientid">>,
             username => <<"username">>,
             is_superuser => false,
+            is_bridge => false,
             mountpoint => undefined
         },
         InitProps

apps/emqx/test/emqx_common_test_http.erl

@@ -34,6 +34,9 @@
 -define(DEFAULT_APP_KEY, <<"default_app_key">>).
 -define(DEFAULT_APP_SECRET, <<"default_app_secret">>).
 
+%% from emqx_dashboard/include/emqx_dashboard_rbac.hrl
+-define(ROLE_API_SUPERUSER, <<"administrator">>).
+
 request_api(Method, Url, Auth) ->
     request_api(Method, Url, [], Auth, []).
 
@@ -96,7 +99,8 @@ create_default_app() ->
         ?DEFAULT_APP_SECRET,
         true,
         ExpiredAt,
-        <<"default app key for test">>
+        <<"default app key for test">>,
+        ?ROLE_API_SUPERUSER
     ).
 
 delete_default_app() ->

apps/emqx/test/emqx_mountpoint_SUITE.erl

@@ -29,6 +29,7 @@
 ).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
 all() -> emqx_common_test_helpers:all(?MODULE).
@@ -52,6 +53,27 @@ t_mount(_) ->
         mount(<<"device/1/">>, TopicFilters)
     ).
 
+t_mount_share(_) ->
+    T = {TopicFilter, Opts} = emqx_topic:parse(<<"$share/group/topic">>),
+    TopicFilters = [T],
+    ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
+
+    %% should not mount share topic when make message.
+    Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>),
+
+    ?assertEqual(
+        TopicFilter,
+        mount(undefined, TopicFilter)
+    ),
+    ?assertEqual(
+        #share{group = <<"group">>, topic = <<"device/1/topic">>},
+        mount(<<"device/1/">>, TopicFilter)
+    ),
+    ?assertEqual(
+        [{#share{group = <<"group">>, topic = <<"device/1/topic">>}, Opts}],
+        mount(<<"device/1/">>, TopicFilters)
+    ).
+
 t_unmount(_) ->
     Msg = emqx_message:make(<<"clientid">>, <<"device/1/topic">>, <<"payload">>),
     ?assertEqual(<<"topic">>, unmount(undefined, <<"topic">>)),
@@ -61,6 +83,23 @@ t_unmount(_) ->
     ?assertEqual(<<"device/1/topic">>, unmount(<<"device/2/">>, <<"device/1/topic">>)),
     ?assertEqual(Msg#message{topic = <<"device/1/topic">>}, unmount(<<"device/2/">>, Msg)).
 
+t_unmount_share(_) ->
+    {TopicFilter, _Opts} = emqx_topic:parse(<<"$share/group/topic">>),
+    MountedTopicFilter = #share{group = <<"group">>, topic = <<"device/1/topic">>},
+
+    ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
+
+    %% should not unmount share topic when make message.
+    Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>),
+    ?assertEqual(
+        TopicFilter,
+        unmount(undefined, TopicFilter)
+    ),
+    ?assertEqual(
+        #share{group = <<"group">>, topic = <<"topic">>},
+        unmount(<<"device/1/">>, MountedTopicFilter)
+    ).
+
 t_replvar(_) ->
     ?assertEqual(undefined, replvar(undefined, #{})),
     ?assertEqual(

apps/emqx/test/emqx_mqtt_caps_SUITE.erl

@@ -76,6 +76,8 @@ t_check_sub(_) ->
     ),
     ?assertEqual(
         {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED},
-        emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true})
+        emqx_mqtt_caps:check_sub(
+            ClientInfo, #share{group = <<"group">>, topic = <<"topic">>}, SubOpts
+        )
     ),
     emqx_config:put([zones], OldConf).

apps/emqx/test/emqx_proper_types.erl

@@ -511,13 +511,7 @@ peercert() ->
 conn_mod() ->
     oneof([
         emqx_connection,
-        emqx_ws_connection,
+        emqx_ws_connection
-        emqx_coap_mqtt_adapter,
-        emqx_sn_gateway,
-        emqx_lwm2m_protocol,
-        emqx_gbt32960_conn,
-        emqx_jt808_connection,
-        emqx_tcp_connection
     ]).
 
 proto_name() ->

apps/emqx/test/emqx_secret_tests.erl

@@ -0,0 +1,76 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_secret_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+wrap_unwrap_test() ->
+    ?assertEqual(
+        42,
+        emqx_secret:unwrap(emqx_secret:wrap(42))
+    ).
+
+unwrap_immediate_test() ->
+    ?assertEqual(
+        42,
+        emqx_secret:unwrap(42)
+    ).
+
+wrap_unwrap_load_test_() ->
+    Secret = <<"foobaz">>,
+    {
+        setup,
+        fun() -> write_temp_file(Secret) end,
+        fun(Filename) -> file:delete(Filename) end,
+        fun(Filename) ->
+            ?_assertEqual(
+                Secret,
+                emqx_secret:unwrap(emqx_secret:wrap_load({file, Filename}))
+            )
+        end
+    }.
+
+wrap_load_term_test() ->
+    ?assertEqual(
+        {file, "no/such/file/i/swear"},
+        emqx_secret:term(emqx_secret:wrap_load({file, "no/such/file/i/swear"}))
+    ).
+
+wrap_unwrap_missing_file_test() ->
+    ?assertThrow(
+        #{msg := failed_to_read_secret_file, reason := "No such file or directory"},
+        emqx_secret:unwrap(emqx_secret:wrap_load({file, "no/such/file/i/swear"}))
+    ).
+
+wrap_term_test() ->
+    ?assertEqual(
+        42,
+        emqx_secret:term(emqx_secret:wrap(42))
+    ).
+
+external_fun_term_error_test() ->
+    Term = {foo, bar},
+    ?assertError(
+        badarg,
+        emqx_secret:term(fun() -> Term end)
+    ).
+
+write_temp_file(Bytes) ->
+    Ts = erlang:system_time(millisecond),
+    Filename = filename:join("/tmp", ?MODULE_STRING ++ integer_to_list(-Ts)),
+    ok = file:write_file(Filename, Bytes),
+    Filename.

apps/emqx/test/emqx_shared_sub_SUITE.erl

@@ -137,7 +137,8 @@ t_random_basic(Config) when is_list(Config) ->
     ClientId = <<"ClientId">>,
     Topic = <<"foo">>,
     Payload = <<"hello">>,
-    emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}),
+    Group = <<"group1">>,
+    emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}),
     MsgQoS2 = emqx_message:make(ClientId, 2, Topic, Payload),
     %% wait for the subscription to show up
     ct:sleep(200),
@@ -402,7 +403,7 @@ t_hash(Config) when is_list(Config) ->
     ok = ensure_config(hash_clientid, false),
     test_two_messages(hash_clientid).
 
-t_hash_clinetid(Config) when is_list(Config) ->
+t_hash_clientid(Config) when is_list(Config) ->
     ok = ensure_config(hash_clientid, false),
     test_two_messages(hash_clientid).
 
@@ -528,14 +529,15 @@ last_message(ExpectedPayload, Pids, Timeout) ->
 t_dispatch(Config) when is_list(Config) ->
     ok = ensure_config(random),
     Topic = <<"foo">>,
+    Group = <<"group1">>,
     ?assertEqual(
         {error, no_subscribers},
-        emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
+        emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}})
     ),
-    emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}),
+    emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}),
     ?assertEqual(
         {ok, 1},
-        emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
+        emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}})
     ).
 
 t_uncovered_func(Config) when is_list(Config) ->
@@ -991,37 +993,110 @@ t_session_kicked(Config) when is_list(Config) ->
     ?assertEqual([], collect_msgs(0)),
     ok.
 
-%% FIXME: currently doesn't work
+-define(UPDATE_SUB_QOS(ConnPid, Topic, QoS),
-%% t_different_groups_same_topic({init, Config}) ->
+    ?assertMatch({ok, _, [QoS]}, emqtt:subscribe(ConnPid, {Topic, QoS}))
-%%     TestName = atom_to_binary(?FUNCTION_NAME),
+).
-%%     ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
-%%     {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
-%%     {ok, _} = emqtt:connect(C),
-%%     [{client, C}, {clientid, ClientId} | Config];
-%% t_different_groups_same_topic({'end', Config}) ->
-%%     C = ?config(client, Config),
-%%     emqtt:stop(C),
-%%     ok;
-%% t_different_groups_same_topic(Config) when is_list(Config) ->
-%%     C = ?config(client, Config),
-%%     ClientId = ?config(clientid, Config),
-%%     %% Subscribe and unsubscribe to both $queue and $shared topics
-%%     Topic = <<"t/1">>,
-%%     SharedTopic0 = <<"$share/aa/", Topic/binary>>,
-%%     SharedTopic1 = <<"$share/bb/", Topic/binary>>,
-%%     {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic0, 2}),
-%%     {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic1, 2}),
 
-%%     Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
+t_different_groups_same_topic({init, Config}) ->
-%%     emqx:publish(Message0),
+    TestName = atom_to_binary(?FUNCTION_NAME),
-%%     ?assertMatch([ {publish, #{payload := <<"hi">>}}
+    ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
-%%                  , {publish, #{payload := <<"hi">>}}
+    {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
-%%                  ], collect_msgs(5_000), #{routes => ets:tab2list(emqx_route)}),
+    {ok, _} = emqtt:connect(C),
+    [{client, C}, {clientid, ClientId} | Config];
+t_different_groups_same_topic({'end', Config}) ->
+    C = ?config(client, Config),
+    emqtt:stop(C),
+    ok;
+t_different_groups_same_topic(Config) when is_list(Config) ->
+    C = ?config(client, Config),
+    ClientId = ?config(clientid, Config),
+    %% Subscribe and unsubscribe to different group `aa` and `bb` with same topic
+    GroupA = <<"aa">>,
+    GroupB = <<"bb">>,
+    Topic = <<"t/1">>,
 
-%%     {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic0),
+    SharedTopicGroupA = ?SHARE(GroupA, Topic),
-%%     {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic1),
+    ?UPDATE_SUB_QOS(C, SharedTopicGroupA, ?QOS_2),
+    SharedTopicGroupB = ?SHARE(GroupB, Topic),
+    ?UPDATE_SUB_QOS(C, SharedTopicGroupB, ?QOS_2),
 
-%%     ok.
+    ?retry(
+        _Sleep0 = 100,
+        _Attempts0 = 50,
+        begin
+            ?assertEqual(2, length(emqx_router:match_routes(Topic)))
+        end
+    ),
+
+    Message0 = emqx_message:make(ClientId, ?QOS_2, Topic, <<"hi">>),
+    emqx:publish(Message0),
+    ?assertMatch(
+        [
+            {publish, #{payload := <<"hi">>}},
+            {publish, #{payload := <<"hi">>}}
+        ],
+        collect_msgs(5_000),
+        #{routes => ets:tab2list(emqx_route)}
+    ),
+
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA),
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB),
+
+    ok.
+
+t_different_groups_update_subopts({init, Config}) ->
+    TestName = atom_to_binary(?FUNCTION_NAME),
+    ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
+    {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(C),
+    [{client, C}, {clientid, ClientId} | Config];
+t_different_groups_update_subopts({'end', Config}) ->
+    C = ?config(client, Config),
+    emqtt:stop(C),
+    ok;
+t_different_groups_update_subopts(Config) when is_list(Config) ->
+    C = ?config(client, Config),
+    ClientId = ?config(clientid, Config),
+    %% Subscribe and unsubscribe to different group `aa` and `bb` with same topic
+    Topic = <<"t/1">>,
+    GroupA = <<"aa">>,
+    GroupB = <<"bb">>,
+    SharedTopicGroupA = ?SHARE(GroupA, Topic),
+    SharedTopicGroupB = ?SHARE(GroupB, Topic),
+
+    Fun = fun(Group, QoS) ->
+        ?UPDATE_SUB_QOS(C, ?SHARE(Group, Topic), QoS),
+        ?assertMatch(
+            #{qos := QoS},
+            emqx_broker:get_subopts(ClientId, emqx_topic:make_shared_record(Group, Topic))
+        )
+    end,
+
+    [Fun(Group, QoS) || QoS <- [?QOS_0, ?QOS_1, ?QOS_2], Group <- [GroupA, GroupB]],
+
+    ?retry(
+        _Sleep0 = 100,
+        _Attempts0 = 50,
+        begin
+            ?assertEqual(2, length(emqx_router:match_routes(Topic)))
+        end
+    ),
+
+    Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
+    emqx:publish(Message0),
+    ?assertMatch(
+        [
+            {publish, #{payload := <<"hi">>}},
+            {publish, #{payload := <<"hi">>}}
+        ],
+        collect_msgs(5_000),
+        #{routes => ets:tab2list(emqx_route)}
+    ),
+
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA),
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB),
+
+    ok.
 
 t_queue_subscription({init, Config}) ->
     TestName = atom_to_binary(?FUNCTION_NAME),
@@ -1038,23 +1113,19 @@ t_queue_subscription({'end', Config}) ->
 t_queue_subscription(Config) when is_list(Config) ->
     C = ?config(client, Config),
     ClientId = ?config(clientid, Config),
-    %% Subscribe and unsubscribe to both $queue and $shared topics
+    %% Subscribe and unsubscribe to both $queue share and $share/<group> with same topic
     Topic = <<"t/1">>,
     QueueTopic = <<"$queue/", Topic/binary>>,
     SharedTopic = <<"$share/aa/", Topic/binary>>,
-    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {QueueTopic, 2}),
-    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {SharedTopic, 2}),
 
-    %% FIXME: we should actually see 2 routes, one for each group
+    ?UPDATE_SUB_QOS(C, QueueTopic, ?QOS_2),
-    %% ($queue and aa), but currently the latest subscription
+    ?UPDATE_SUB_QOS(C, SharedTopic, ?QOS_2),
-    %% overwrites the existing one.
+
     ?retry(
         _Sleep0 = 100,
         _Attempts0 = 50,
         begin
-            ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
+            ?assertEqual(2, length(emqx_router:match_routes(Topic)))
-            %% FIXME: should ensure we have 2 subscriptions
-            [_] = emqx_router:lookup_routes(Topic)
         end
     ),
 
@@ -1063,37 +1134,29 @@ t_queue_subscription(Config) when is_list(Config) ->
     emqx:publish(Message0),
     ?assertMatch(
         [
+            {publish, #{payload := <<"hi">>}},
             {publish, #{payload := <<"hi">>}}
-            %% FIXME: should receive one message from each group
-            %% , {publish, #{payload := <<"hi">>}}
         ],
-        collect_msgs(5_000)
+        collect_msgs(5_000),
+        #{routes => ets:tab2list(emqx_route)}
     ),
 
     {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, QueueTopic),
-    %% FIXME: return code should be success instead of 17 ("no_subscription_existed")
+    {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopic),
-    {ok, _, [?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(C, SharedTopic),
 
-    %% FIXME: this should eventually be true, but currently we leak
+    ?retry(
-    %% the previous group subscription...
+        _Sleep0 = 100,
-    %% ?retry(
+        _Attempts0 = 50,
-    %%     _Sleep0 = 100,
+        begin
-    %%     _Attempts0 = 50,
+            ?assertEqual(0, length(emqx_router:match_routes(Topic)))
-    %%    begin
+        end
-    %%     ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
+    ),
-    %%     [] = emqx_router:lookup_routes(Topic)
-    %%    end
-    %%   ),
     ct:sleep(500),
 
     Message1 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hello">>),
     emqx:publish(Message1),
-    %% FIXME: we should *not* receive any messages...
+    %% we should *not* receive any messages.
-    %% ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}),
+    ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}),
-    %% This is from the leaked group...
-    ?assertMatch([{publish, #{topic := Topic}}], collect_msgs(1_000), #{
-        routes => ets:tab2list(emqx_route)
-    }),
 
     ok.
 

apps/emqx/test/emqx_topic_SUITE.erl

@@ -238,11 +238,11 @@ long_topic() ->
 t_parse(_) ->
     ?assertError(
         {invalid_topic_filter, <<"$queue/t">>},
-        parse(<<"$queue/t">>, #{share => <<"g">>})
+        parse(#share{group = <<"$queue">>, topic = <<"$queue/t">>}, #{})
     ),
     ?assertError(
         {invalid_topic_filter, <<"$share/g/t">>},
-        parse(<<"$share/g/t">>, #{share => <<"g">>})
+        parse(#share{group = <<"g">>, topic = <<"$share/g/t">>}, #{})
     ),
     ?assertError(
         {invalid_topic_filter, <<"$share/t">>},
@@ -254,8 +254,12 @@ t_parse(_) ->
     ),
     ?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)),
     ?assertEqual({<<"a/b/+/#">>, #{qos => 1}}, parse({<<"a/b/+/#">>, #{qos => 1}})),
-    ?assertEqual({<<"topic">>, #{share => <<"$queue">>}}, parse(<<"$queue/topic">>)),
+    ?assertEqual(
-    ?assertEqual({<<"topic">>, #{share => <<"group">>}}, parse(<<"$share/group/topic">>)),
+        {#share{group = <<"$queue">>, topic = <<"topic">>}, #{}}, parse(<<"$queue/topic">>)
+    ),
+    ?assertEqual(
+        {#share{group = <<"group">>, topic = <<"topic">>}, #{}}, parse(<<"$share/group/topic">>)
+    ),
     %% The '$local' and '$fastlane' topics have been deprecated.
     ?assertEqual({<<"$local/topic">>, #{}}, parse(<<"$local/topic">>)),
     ?assertEqual({<<"$local/$queue/topic">>, #{}}, parse(<<"$local/$queue/topic">>)),

apps/emqx/test/emqx_topic_index_SUITE.erl

@@ -209,9 +209,6 @@ t_match_fast_forward(Config) ->
     M:insert(<<"a/b/1/2/3/4/5/6/7/8/9/#">>, id1, <<>>, Tab),
     M:insert(<<"z/y/x/+/+">>, id2, <<>>, Tab),
     M:insert(<<"a/b/c/+">>, id3, <<>>, Tab),
-    % dbg:tracer(),
-    % dbg:p(all, c),
-    % dbg:tpl({ets, next, '_'}, x),
     ?assertEqual(id1, id(match(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab))),
     ?assertEqual([id1], [id(X) || X <- matches(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab)]).
 

apps/emqx_audit/BSL.txt

@@ -0,0 +1,94 @@
+Business Source License 1.1
+
+Licensor:             Hangzhou EMQ Technologies Co., Ltd.
+Licensed Work:        EMQX Enterprise Edition
+                      The Licensed Work is (c) 2023
+                      Hangzhou EMQ Technologies Co., Ltd.
+Additional Use Grant: Students and educators are granted right to copy,
+                      modify, and create derivative work for research
+                      or education.
+Change Date:          2027-02-01
+Change License:       Apache License, Version 2.0
+
+For information about alternative licensing arrangements for the Software,
+please contact Licensor: https://www.emqx.com/en/contact
+
+Notice
+
+The Business Source License (this document, or the “License”) is not an Open
+Source license. However, the Licensed Work will eventually be made available
+under an Open Source License, as stated in this License.
+
+License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
+“Business Source License” is a trademark of MariaDB Corporation Ab.
+
+-----------------------------------------------------------------------------
+
+Business Source License 1.1
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited
+production use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.
+
+MariaDB hereby grants you permission to use this License’s text to license
+your works, and to refer to it using the trademark “Business Source License”,
+as long as you comply with the Covenants of Licensor below.
+
+Covenants of Licensor
+
+In consideration of the right to use this License’s text and the “Business
+Source License” name and trademark, Licensor covenants to MariaDB, and to all
+other recipients of the licensed work to be provided by Licensor:
+
+1. To specify as the Change License the GPL Version 2.0 or any later version,
+   or a license that is compatible with GPL Version 2.0 or a later version,
+   where “compatible” means that software provided under the Change License can
+   be included in a program with software provided under GPL Version 2.0 or a
+   later version. Licensor may specify additional Change Licenses without
+   limitation.
+
+2. To either: (a) specify an additional grant of rights to use that does not
+   impose any additional restriction on the right granted in this License, as
+   the Additional Use Grant; or (b) insert the text “None”.
+
+3. To specify a Change Date.
+
+4. Not to modify this License in any other way.

apps/emqx_audit/README.md

@@ -0,0 +1,5 @@
+emqx_audit
+=====
+
+Audit log for EMQX, empowers users to efficiently access the desired audit trail data
+and facilitates auditing, compliance, troubleshooting, and security analysis.

apps/emqx_audit/include/emqx_audit.hrl

@@ -0,0 +1,26 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-define(AUDIT, emqx_audit).
+
+-record(?AUDIT, {
+    %% basic info
+    created_at,
+    node,
+    from,
+    source,
+    source_ip,
+    %% operation info
+    operation_id,
+    operation_type,
+    args,
+    operation_result,
+    failure,
+    %% request detail
+    http_method,
+    http_request,
+    http_status_code,
+    duration_ms,
+    extra
+}).

apps/emqx_audit/rebar.config

@@ -0,0 +1,5 @@
+{erl_opts, [debug_info]}.
+{deps, [
+  {emqx, {path, "../emqx"}},
+  {emqx_utils, {path, "../emqx_utils"}}
+]}.

apps/emqx_audit/src/emqx_audit.app.src

@@ -0,0 +1,10 @@
+{application, emqx_audit, [
+    {description, "Audit log for EMQX"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {mod, {emqx_audit_app, []}},
+    {applications, [kernel, stdlib, emqx]},
+    {env, []},
+    {modules, []},
+    {links, []}
+]}.

apps/emqx_audit/src/emqx_audit.erl

@@ -0,0 +1,245 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_audit).
+
+-behaviour(gen_server).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include("emqx_audit.hrl").
+
+%% API
+-export([start_link/0]).
+-export([log/3]).
+
+-export([trans_clean_expired/2]).
+
+%% gen_server callbacks
+-export([
+    init/1,
+    handle_continue/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
+
+-define(FILTER_REQ, [cert, host_info, has_sent_resp, pid, path_info, peer, ref, sock, streamid]).
+
+-ifdef(TEST).
+-define(INTERVAL, 100).
+-else.
+-define(INTERVAL, 10000).
+-endif.
+
+to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) ->
+    #?AUDIT{
+        operation_id = <<"">>,
+        operation_type = atom_to_binary(Cmd),
+        args = Args,
+        operation_result = <<"">>,
+        failure = <<"">>,
+        duration_ms = DurationMs,
+        from = cli,
+        source = <<"">>,
+        source_ip = <<"">>,
+        http_status_code = <<"">>,
+        http_method = <<"">>,
+        http_request = <<"">>
+    };
+to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api ->
+    #{
+        source := Source,
+        source_ip := SourceIp,
+        %% operation info
+        operation_id := OperationId,
+        operation_type := OperationType,
+        operation_result := OperationResult,
+        %% request detail
+        http_status_code := StatusCode,
+        http_method := Method,
+        http_request := Request,
+        duration_ms := DurationMs
+    } = Log,
+    #?AUDIT{
+        from = From,
+        source = Source,
+        source_ip = SourceIp,
+        %% operation info
+        operation_id = OperationId,
+        operation_type = OperationType,
+        operation_result = OperationResult,
+        failure = maps:get(failure, Log, <<"">>),
+        %% request detail
+        http_status_code = StatusCode,
+        http_method = Method,
+        http_request = Request,
+        duration_ms = DurationMs,
+        args = <<"">>
+    };
+to_audit(#{from := erlang_console, function := F, args := Args}) ->
+    #?AUDIT{
+        from = erlang_console,
+        source = <<"">>,
+        source_ip = <<"">>,
+        %% operation info
+        operation_id = <<"">>,
+        operation_type = <<"">>,
+        operation_result = <<"">>,
+        failure = <<"">>,
+        %% request detail
+        http_status_code = <<"">>,
+        http_method = <<"">>,
+        http_request = <<"">>,
+        duration_ms = 0,
+        args = iolist_to_binary(io_lib:format("~p: ~p~n", [F, Args]))
+    }.
+
+log(_Level, undefined, _Handler) ->
+    ok;
+log(Level, Meta1, Handler) ->
+    Meta2 = Meta1#{time => logger:timestamp(), level => Level},
+    log_to_file(Level, Meta2, Handler),
+    log_to_db(Meta2),
+    remove_handler_when_disabled().
+
+remove_handler_when_disabled() ->
+    case emqx_config:get([log, audit, enable], false) of
+        true ->
+            ok;
+        false ->
+            _ = logger:remove_handler(?AUDIT_HANDLER),
+            ok
+    end.
+
+log_to_db(Log) ->
+    Audit0 = to_audit(Log),
+    Audit = Audit0#?AUDIT{
+        node = node(),
+        created_at = erlang:system_time(microsecond)
+    },
+    mria:dirty_write(?AUDIT, Audit).
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init([]) ->
+    ok = mria:create_table(?AUDIT, [
+        {type, ordered_set},
+        {rlog_shard, ?COMMON_SHARD},
+        {storage, disc_copies},
+        {record_name, ?AUDIT},
+        {attributes, record_info(fields, ?AUDIT)}
+    ]),
+    {ok, #{}, {continue, setup}}.
+
+handle_continue(setup, State) ->
+    ok = mria:wait_for_tables([?AUDIT]),
+    NewState = State#{role => mria_rlog:role()},
+    ?AUDIT(alert, #{
+        cmd => emqx,
+        args => ["start"],
+        version => emqx_release:version(),
+        from => cli,
+        duration_ms => 0
+    }),
+    {noreply, NewState, interval(NewState)}.
+
+handle_call(_Request, _From, State) ->
+    {reply, ignore, State, interval(State)}.
+
+handle_cast(_Request, State) ->
+    {noreply, State, interval(State)}.
+
+handle_info(timeout, State) ->
+    ExtraWait = clean_expired_logs(),
+    {noreply, State, interval(State) + ExtraWait};
+handle_info(_Info, State) ->
+    {noreply, State, interval(State)}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+
+%% if clean_expired transaction aborted, it will be scheduled with extra 60 seconds.
+clean_expired_logs() ->
+    MaxSize = max_size(),
+    Oldest = mnesia:dirty_first(?AUDIT),
+    CurSize = mnesia:table_info(?AUDIT, size),
+    case CurSize - MaxSize of
+        DelSize when DelSize > 0 ->
+            case
+                mria:transaction(
+                    ?COMMON_SHARD,
+                    fun ?MODULE:trans_clean_expired/2,
+                    [Oldest, DelSize]
+                )
+            of
+                {atomic, ok} ->
+                    0;
+                {aborted, Reason} ->
+                    ?SLOG(error, #{
+                        msg => "clean_expired_audit_aborted",
+                        reason => Reason,
+                        delete_size => DelSize,
+                        current_size => CurSize,
+                        max_count => MaxSize
+                    }),
+                    60000
+            end;
+        _ ->
+            0
+    end.
+
+trans_clean_expired(Oldest, DelCount) ->
+    First = mnesia:first(?AUDIT),
+    %% Other node already clean from the oldest record.
+    %% ensure not delete twice, otherwise records that should not be deleted will be deleted.
+    case First =:= Oldest of
+        true -> do_clean_expired(First, DelCount);
+        false -> ok
+    end.
+
+do_clean_expired(_, DelSize) when DelSize =< 0 -> ok;
+do_clean_expired('$end_of_table', _DelSize) ->
+    ok;
+do_clean_expired(CurKey, DeleteSize) ->
+    mnesia:delete(?AUDIT, CurKey, sticky_write),
+    do_clean_expired(mnesia:next(?AUDIT, CurKey), DeleteSize - 1).
+
+max_size() ->
+    emqx_conf:get([log, audit, max_filter_size], 5000).
+
+interval(#{role := replicant}) -> hibernate;
+interval(#{role := core}) -> ?INTERVAL + rand:uniform(?INTERVAL).
+
+log_to_file(Level, Meta, #{module := Module} = Handler) ->
+    Log = #{level => Level, meta => Meta, msg => undefined},
+    Handler1 = maps:without(?OWN_KEYS, Handler),
+    try
+        erlang:apply(Module, log, [Log, Handler1])
+    catch
+        C:R:S ->
+            case logger:remove_handler(?AUDIT_HANDLER) of
+                ok ->
+                    logger:internal_log(
+                        error, {removed_failing_handler, ?AUDIT_HANDLER, C, R, S}
+                    );
+                {error, {not_found, _}} ->
+                    ok;
+                {error, Reason} ->
+                    logger:internal_log(
+                        error,
+                        {removed_handler_failed, ?AUDIT_HANDLER, Reason, C, R, S}
+                    )
+            end
+    end.

apps/emqx_audit/src/emqx_audit_api.erl

@@ -0,0 +1,398 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_audit_api).
+
+-behaviour(minirest_api).
+
+%% API
+-export([api_spec/0, paths/0, schema/1, namespace/0, fields/1]).
+-export([audit/2]).
+-export([qs2ms/2, format/1]).
+
+-include_lib("emqx/include/logger.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+-include_lib("typerefl/include/types.hrl").
+-include("emqx_audit.hrl").
+
+-import(hoconsc, [mk/2, ref/2, array/1]).
+
+-define(TAGS, ["Audit"]).
+
+-define(AUDIT_QS_SCHEMA, [
+    {<<"node">>, atom},
+    {<<"from">>, atom},
+    {<<"source">>, binary},
+    {<<"source_ip">>, binary},
+    {<<"operation_id">>, binary},
+    {<<"operation_type">>, binary},
+    {<<"operation_result">>, atom},
+    {<<"http_status_code">>, integer},
+    {<<"http_method">>, atom},
+    {<<"gte_created_at">>, timestamp},
+    {<<"lte_created_at">>, timestamp},
+    {<<"gte_duration_ms">>, timestamp},
+    {<<"lte_duration_ms">>, timestamp}
+]).
+-define(DISABLE_MSG, <<"Audit is disabled">>).
+
+namespace() -> "audit".
+
+api_spec() ->
+    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
+
+paths() ->
+    ["/audit"].
+
+schema("/audit") ->
+    #{
+        'operationId' => audit,
+        get => #{
+            tags => ?TAGS,
+            description => ?DESC(audit_get),
+            parameters => [
+                {node,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        required => false,
+                        example => <<"emqx@127.0.0.1">>,
+                        desc => ?DESC(filter_node)
+                    })},
+                {from,
+                    ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{
+                        in => query,
+                        required => false,
+                        example => <<"dashboard">>,
+                        desc => ?DESC(filter_from)
+                    })},
+                {source,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        required => false,
+                        example => <<"admin">>,
+                        desc => ?DESC(filter_source)
+                    })},
+                {source_ip,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        required => false,
+                        example => <<"127.0.0.1">>,
+                        desc => ?DESC(filter_source_ip)
+                    })},
+                {operation_id,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        required => false,
+                        example => <<"/rules/{id}">>,
+                        desc => ?DESC(filter_operation_id)
+                    })},
+                {operation_type,
+                    ?HOCON(binary(), #{
+                        in => query,
+                        example => <<"rules">>,
+                        required => false,
+                        desc => ?DESC(filter_operation_type)
+                    })},
+                {operation_result,
+                    ?HOCON(?ENUM([success, failure]), #{
+                        in => query,
+                        example => failure,
+                        required => false,
+                        desc => ?DESC(filter_operation_result)
+                    })},
+                {http_status_code,
+                    ?HOCON(integer(), #{
+                        in => query,
+                        example => 200,
+                        required => false,
+                        desc => ?DESC(filter_http_status_code)
+                    })},
+                {http_method,
+                    ?HOCON(?ENUM([post, put, delete]), #{
+                        in => query,
+                        example => post,
+                        required => false,
+                        desc => ?DESC(filter_http_method)
+                    })},
+                {gte_duration_ms,
+                    ?HOCON(integer(), #{
+                        in => query,
+                        example => 0,
+                        required => false,
+                        desc => ?DESC(filter_gte_duration_ms)
+                    })},
+                {lte_duration_ms,
+                    ?HOCON(integer(), #{
+                        in => query,
+                        example => 1000,
+                        required => false,
+                        desc => ?DESC(filter_lte_duration_ms)
+                    })},
+                {gte_created_at,
+                    ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
+                        in => query,
+                        required => false,
+                        example => <<"2023-10-15T00:00:00.820384+08:00">>,
+                        desc => ?DESC(filter_gte_created_at)
+                    })},
+                {lte_created_at,
+                    ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
+                        in => query,
+                        example => <<"2023-10-16T00:00:00.820384+08:00">>,
+                        required => false,
+                        desc => ?DESC(filter_lte_created_at)
+                    })},
+                ref(emqx_dashboard_swagger, page),
+                ref(emqx_dashboard_swagger, limit)
+            ],
+            summary => <<"List audit logs">>,
+            responses => #{
+                200 =>
+                    emqx_dashboard_swagger:schema_with_example(
+                        array(?REF(audit_list)),
+                        audit_log_list_example()
+                    ),
+                400 => emqx_dashboard_swagger:error_codes(
+                    ['BAD_REQUEST'],
+                    ?DISABLE_MSG
+                )
+            }
+        }
+    }.
+
+fields(audit_list) ->
+    [
+        {data, mk(array(?REF(audit)), #{desc => ?DESC("audit_resp")})},
+        {meta, mk(ref(emqx_dashboard_swagger, meta), #{})}
+    ];
+fields(audit) ->
+    [
+        {created_at,
+            ?HOCON(
+                emqx_utils_calendar:epoch_millisecond(),
+                #{
+                    desc => "The time when the log is created"
+                }
+            )},
+        {node,
+            ?HOCON(binary(), #{
+                desc => "The node name to which the log is created"
+            })},
+        {from,
+            ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{
+                desc => "The source type of the log"
+            })},
+        {source,
+            ?HOCON(binary(), #{
+                desc => "The source of the log"
+            })},
+        {source_ip,
+            ?HOCON(binary(), #{
+                desc => "The source ip of the log"
+            })},
+        {operation_id,
+            ?HOCON(binary(), #{
+                desc => "The operation id of the log"
+            })},
+        {operation_type,
+            ?HOCON(binary(), #{
+                desc => "The operation type of the log"
+            })},
+        {operation_result,
+            ?HOCON(?ENUM([success, failure]), #{
+                desc => "The operation result of the log"
+            })},
+        {http_status_code,
+            ?HOCON(integer(), #{
+                desc => "The http status code of the log"
+            })},
+        {http_method,
+            ?HOCON(?ENUM([post, put, delete]), #{
+                desc => "The http method of the log"
+            })},
+        {duration_ms,
+            ?HOCON(integer(), #{
+                desc => "The duration of the log"
+            })},
+        {args,
+            ?HOCON(?ARRAY(binary()), #{
+                desc => "The args of the log"
+            })},
+        {failure,
+            ?HOCON(?ARRAY(binary()), #{
+                desc => "The failure of the log"
+            })},
+        {http_request,
+            ?HOCON(?REF(http_request), #{
+                desc => "The http request of the log"
+            })}
+    ];
+fields(http_request) ->
+    [
+        {bindings, ?HOCON(map(), #{})},
+        {body, ?HOCON(map(), #{})},
+        {headers, ?HOCON(map(), #{})},
+        {method, ?HOCON(?ENUM([post, put, delete]), #{})}
+    ].
+
+audit(get, #{query_string := QueryString}) ->
+    case emqx_config:get([log, audit, enable], false) of
+        false ->
+            {400, #{code => 'BAD_REQUEST', message => ?DISABLE_MSG}};
+        true ->
+            case
+                emqx_mgmt_api:node_query(
+                    node(),
+                    ?AUDIT,
+                    QueryString,
+                    ?AUDIT_QS_SCHEMA,
+                    fun ?MODULE:qs2ms/2,
+                    fun ?MODULE:format/1
+                )
+            of
+                {error, page_limit_invalid} ->
+                    {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}};
+                {error, Node, Error} ->
+                    Message = list_to_binary(
+                        io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])
+                    ),
+                    {500, #{code => <<"NODE_DOWN">>, message => Message}};
+                Result ->
+                    {200, Result}
+            end
+    end.
+
+qs2ms(_Tab, {Qs, _}) ->
+    #{
+        match_spec => gen_match_spec(Qs, #?AUDIT{_ = '_'}, []),
+        fuzzy_fun => undefined
+    }.
+
+gen_match_spec([], Audit, Conn) ->
+    [{Audit, Conn, ['$_']}];
+gen_match_spec([{node, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{node = T}, Conn);
+gen_match_spec([{from, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{from = T}, Conn);
+gen_match_spec([{source, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{source = T}, Conn);
+gen_match_spec([{source_ip, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{source_ip = T}, Conn);
+gen_match_spec([{operation_id, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{operation_id = T}, Conn);
+gen_match_spec([{operation_type, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{operation_type = T}, Conn);
+gen_match_spec([{operation_result, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{operation_result = T}, Conn);
+gen_match_spec([{http_status_code, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{http_status_code = T}, Conn);
+gen_match_spec([{http_method, '=:=', T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{http_method = T}, Conn);
+gen_match_spec([{created_at, Hold, T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [{'$1', Hold, T} | Conn]);
+gen_match_spec([{created_at, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [
+        {'$1', Hold1, T1}, {'$1', Hold2, T2} | Conn
+    ]);
+gen_match_spec([{duration_ms, Hold, T} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [{'$2', Hold, T} | Conn]);
+gen_match_spec([{duration_ms, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) ->
+    gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [
+        {'$2', Hold1, T1}, {'$2', Hold2, T2} | Conn
+    ]).
+
+format(Audit) ->
+    #?AUDIT{
+        created_at = CreatedAt,
+        node = Node,
+        from = From,
+        source = Source,
+        source_ip = SourceIp,
+        operation_id = OperationId,
+        operation_type = OperationType,
+        operation_result = OperationResult,
+        http_status_code = HttpStatusCode,
+        http_method = HttpMethod,
+        duration_ms = DurationMs,
+        args = Args,
+        failure = Failure,
+        http_request = HttpRequest
+    } = Audit,
+    #{
+        created_at => emqx_utils_calendar:epoch_to_rfc3339(CreatedAt, microsecond),
+        node => Node,
+        from => From,
+        source => Source,
+        source_ip => SourceIp,
+        operation_id => OperationId,
+        operation_type => OperationType,
+        operation_result => OperationResult,
+        http_status_code => HttpStatusCode,
+        http_method => HttpMethod,
+        duration_ms => DurationMs,
+        args => Args,
+        failure => Failure,
+        http_request => HttpRequest
+    }.
+
+audit_log_list_example() ->
+    #{
+        data => [api_example(), cli_example()],
+        meta => #{
+            <<"count">> => 2,
+            <<"hasnext">> => false,
+            <<"limit">> => 50,
+            <<"page">> => 1
+        }
+    }.
+
+api_example() ->
+    #{
+        <<"args">> => "",
+        <<"created_at">> => "2023-10-17T10:41:20.383993+08:00",
+        <<"duration_ms">> => 0,
+        <<"failure">> => "",
+        <<"from">> => "dashboard",
+        <<"http_method">> => "post",
+        <<"http_request">> => #{
+            <<"bindings">> => #{},
+            <<"body">> => #{
+                <<"password">> => "******",
+                <<"username">> => "admin"
+            },
+            <<"headers">> => #{
+                <<"accept">> => "*/*",
+                <<"authorization">> => "******",
+                <<"connection">> => "keep-alive",
+                <<"content-length">> => "45",
+                <<"content-type">> => "application/json"
+            },
+            <<"method">> => "post"
+        },
+        <<"http_status_code">> => 200,
+        <<"node">> => "emqx@127.0.0.1",
+        <<"operation_id">> => "/login",
+        <<"operation_result">> => "success",
+        <<"operation_type">> => "login",
+        <<"source">> => "admin",
+        <<"source_ip">> => "127.0.0.1"
+    }.
+
+cli_example() ->
+    #{
+        <<"args">> => [<<"show">>, <<"log">>],
+        <<"created_at">> => "2023-10-17T10:45:13.100426+08:00",
+        <<"duration_ms">> => 7,
+        <<"failure">> => "",
+        <<"from">> => "cli",
+        <<"http_method">> => "",
+        <<"http_request">> => "",
+        <<"http_status_code">> => "",
+        <<"node">> => "emqx@127.0.0.1",
+        <<"operation_id">> => "",
+        <<"operation_result">> => "",
+        <<"operation_type">> => "conf",
+        <<"source">> => "",
+        <<"source_ip">> => ""
+    }.

apps/emqx_audit/src/emqx_audit_app.erl

@@ -0,0 +1,15 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_audit_app).
+
+-behaviour(application).
+
+-export([start/2, stop/1]).
+
+start(_StartType, _StartArgs) ->
+    emqx_audit_sup:start_link().
+
+stop(_State) ->
+    ok.

apps/emqx_audit/src/emqx_audit_sup.erl

@@ -0,0 +1,33 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_audit_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/0]).
+
+-export([init/1]).
+
+-define(SERVER, ?MODULE).
+
+start_link() ->
+    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
+
+init([]) ->
+    SupFlags = #{
+        strategy => one_for_all,
+        intensity => 10,
+        period => 10
+    },
+    ChildSpecs = [
+        #{
+            id => emqx_audit,
+            start => {emqx_audit, start_link, []},
+            type => worker,
+            restart => transient,
+            shutdown => 1000
+        }
+    ],
+    {ok, {SupFlags, ChildSpecs}}.

apps/emqx_audit/test/emqx_audit_api_SUITE.erl

@@ -0,0 +1,248 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_audit_api_SUITE).
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+all() ->
+    [
+        {group, audit, [sequence]}
+    ].
+
+groups() ->
+    [
+        {audit, [sequence], common_tests()}
+    ].
+
+common_tests() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+-define(CONF_DEFAULT, #{
+    node =>
+        #{
+            name => "emqx1@127.0.0.1",
+            cookie => "emqxsecretcookie",
+            data_dir => "data"
+        },
+    log => #{
+        audit =>
+            #{
+                enable => true,
+                ignore_high_frequency_request => true,
+                level => info,
+                max_filter_size => 15,
+                rotation_count => 2,
+                rotation_size => "10MB",
+                time_offset => "system"
+            }
+    }
+}).
+
+init_per_suite(Config) ->
+    _ = application:load(emqx_conf),
+    emqx_config:erase_all(),
+    emqx_mgmt_api_test_util:init_suite([emqx_ctl, emqx_conf, emqx_audit]),
+    ok = emqx_common_test_helpers:load_config(emqx_enterprise_schema, ?CONF_DEFAULT),
+    emqx_config:save_schema_mod_and_names(emqx_enterprise_schema),
+    ok = emqx_config_logger:refresh_config(),
+    application:set_env(emqx, boot_modules, []),
+    emqx_conf_cli:load(),
+    Config.
+
+end_per_suite(_) ->
+    emqx_mgmt_api_test_util:end_suite([emqx_audit, emqx_conf, emqx_ctl]).
+
+t_http_api(_) ->
+    process_flag(trap_exit, true),
+    AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    {ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(),
+    NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1),
+    {ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones),
+    ?assertMatch(#{<<"max_qos_allowed">> := 1}, Res),
+    {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
+    ?assertMatch(
+        #{
+            <<"data">> := [
+                #{
+                    <<"from">> := <<"rest_api">>,
+                    <<"operation_id">> := <<"/configs/global_zone">>,
+                    <<"source_ip">> := <<"127.0.0.1">>,
+                    <<"source">> := _,
+                    <<"http_request">> := #{
+                        <<"method">> := <<"put">>,
+                        <<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}},
+                        <<"bindings">> := _,
+                        <<"headers">> := #{<<"authorization">> := <<"******">>}
+                    },
+                    <<"http_status_code">> := 200,
+                    <<"operation_result">> := <<"success">>,
+                    <<"operation_type">> := <<"configs">>
+                }
+            ]
+        },
+        emqx_utils_json:decode(Res1, [return_maps])
+    ),
+    ok.
+
+t_disabled(_) ->
+    Enable = [log, audit, enable],
+    ?assertEqual(true, emqx:get_config(Enable)),
+    AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    {ok, _} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
+    Size1 = mnesia:table_info(emqx_audit, size),
+
+    {ok, Logs} = emqx_mgmt_api_configs_SUITE:get_config("log"),
+    Logs1 = emqx_utils_maps:deep_put([<<"audit">>, <<"max_filter_size">>], Logs, 100),
+    NewLogs = emqx_utils_maps:deep_put([<<"audit">>, <<"enable">>], Logs1, false),
+    {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", NewLogs),
+    {ok, GetLog1} = emqx_mgmt_api_configs_SUITE:get_config("log"),
+    ?assertEqual(NewLogs, GetLog1),
+    ?assertMatch(
+        {error, _},
+        emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader)
+    ),
+
+    Size2 = mnesia:table_info(emqx_audit, size),
+    %% Record the audit disable action, so the size + 1
+    ?assertEqual(Size1 + 1, Size2),
+
+    {ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(),
+    NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_topic_levels">>], Zones, 111),
+    {ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones),
+    ?assertMatch(#{<<"max_topic_levels">> := 111}, Res),
+    Size3 = mnesia:table_info(emqx_audit, size),
+    %% Don't record mqtt update request.
+    ?assertEqual(Size2, Size3),
+    %% enabled again
+    {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", Logs1),
+    {ok, GetLog2} = emqx_mgmt_api_configs_SUITE:get_config("log"),
+    ?assertEqual(Logs1, GetLog2),
+    Size4 = mnesia:table_info(emqx_audit, size),
+    ?assertEqual(Size3 + 1, Size4),
+    ok.
+
+t_cli(_Config) ->
+    ok = emqx_ctl:run_command(["conf", "show", "log"]),
+    AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    {ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
+    #{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]),
+    ?assertMatch(
+        [
+            #{
+                <<"from">> := <<"cli">>,
+                <<"operation_id">> := <<"">>,
+                <<"source_ip">> := <<"">>,
+                <<"operation_type">> := <<"conf">>,
+                <<"args">> := [<<"show">>, <<"log">>],
+                <<"node">> := _,
+                <<"source">> := <<"">>,
+                <<"http_request">> := <<"">>
+            }
+        ],
+        Data
+    ),
+
+    %% check filter
+    {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader),
+    #{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]),
+    ?assertEqual(Data, Data1),
+    {ok, Res2} = emqx_mgmt_api_test_util:request_api(
+        get, AuditPath, "from=erlang_console", AuthHeader
+    ),
+    ?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])),
+    ok.
+
+t_max_size(_Config) ->
+    {ok, _} = emqx:update_config([log, audit, max_filter_size], 1000),
+    SizeFun =
+        fun() ->
+            AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+            AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+            Limit = "limit=1000",
+            {ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, Limit, AuthHeader),
+            #{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]),
+            erlang:length(Data)
+        end,
+    InitSize = SizeFun(),
+    lists:foreach(
+        fun(_) ->
+            ok = emqx_ctl:run_command(["conf", "show", "log"])
+        end,
+        lists:duplicate(100, 1)
+    ),
+    timer:sleep(110),
+    Size1 = SizeFun(),
+    ?assert(Size1 - InitSize >= 100, {Size1, InitSize}),
+    {ok, _} = emqx:update_config([log, audit, max_filter_size], 10),
+    %% wait for clean_expired
+    timer:sleep(250),
+    ExpectSize = emqx:get_config([log, audit, max_filter_size]),
+    Size2 = SizeFun(),
+    ?assertEqual(ExpectSize, Size2, {sys:get_state(emqx_audit)}),
+    ok.
+
+t_kickout_clients_without_log(_) ->
+    process_flag(trap_exit, true),
+    AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
+    {ok, AuditLogs1} = emqx_mgmt_api_test_util:request_api(get, AuditPath),
+    kickout_clients(),
+    {ok, AuditLogs2} = emqx_mgmt_api_test_util:request_api(get, AuditPath),
+    ?assertEqual(AuditLogs1, AuditLogs2),
+    ok.
+
+kickout_clients() ->
+    ClientId1 = <<"client1">>,
+    ClientId2 = <<"client2">>,
+    ClientId3 = <<"client3">>,
+
+    {ok, C1} = emqtt:start_link(#{
+        clientid => ClientId1,
+        proto_ver => v5,
+        properties => #{'Session-Expiry-Interval' => 120}
+    }),
+    {ok, _} = emqtt:connect(C1),
+    {ok, C2} = emqtt:start_link(#{clientid => ClientId2}),
+    {ok, _} = emqtt:connect(C2),
+    {ok, C3} = emqtt:start_link(#{clientid => ClientId3}),
+    {ok, _} = emqtt:connect(C3),
+
+    timer:sleep(300),
+
+    %% get /clients
+    ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]),
+    {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
+    ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]),
+    ClientsMeta = maps:get(<<"meta">>, ClientsResponse),
+    ClientsPage = maps:get(<<"page">>, ClientsMeta),
+    ClientsLimit = maps:get(<<"limit">>, ClientsMeta),
+    ClientsCount = maps:get(<<"count">>, ClientsMeta),
+    ?assertEqual(ClientsPage, 1),
+    ?assertEqual(ClientsLimit, emqx_mgmt:default_row_limit()),
+    ?assertEqual(ClientsCount, 3),
+
+    %% kickout clients
+    KickoutPath = emqx_mgmt_api_test_util:api_path(["clients", "kickout", "bulk"]),
+    KickoutBody = [ClientId1, ClientId2, ClientId3],
+    {ok, 204, _} = emqx_mgmt_api_test_util:request_api_with_body(post, KickoutPath, KickoutBody),
+
+    {ok, Clients2} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
+    ClientsResponse2 = emqx_utils_json:decode(Clients2, [return_maps]),
+    ?assertMatch(#{<<"data">> := []}, ClientsResponse2).

apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl

@@ -26,10 +26,6 @@
 -define(AUTHN_BACKEND, ldap).
 -define(AUTHN_BACKEND_BIN, <<"ldap">>).
 
--define(AUTHN_BACKEND_BIND, ldap_bind).
--define(AUTHN_BACKEND_BIND_BIN, <<"ldap_bind">>).
-
 -define(AUTHN_TYPE, {?AUTHN_MECHANISM, ?AUTHN_BACKEND}).
--define(AUTHN_TYPE_BIND, {?AUTHN_MECHANISM, ?AUTHN_BACKEND_BIND}).
 
 -endif.

apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl

@@ -25,12 +25,10 @@
 start(_StartType, _StartArgs) ->
     ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_ldap),
     ok = emqx_authn:register_provider(?AUTHN_TYPE, emqx_authn_ldap),
-    ok = emqx_authn:register_provider(?AUTHN_TYPE_BIND, emqx_authn_ldap_bind),
     {ok, Sup} = emqx_auth_ldap_sup:start_link(),
     {ok, Sup}.
 
 stop(_State) ->
     ok = emqx_authn:deregister_provider(?AUTHN_TYPE),
-    ok = emqx_authn:deregister_provider(?AUTHN_TYPE_BIND),
     ok = emqx_authz:unregister_source(?AUTHZ_TYPE),
     ok.

apps/emqx_auth_ldap/src/emqx_authn_ldap.erl

@@ -16,19 +16,10 @@
 
 -module(emqx_authn_ldap).
 
--include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("emqx/include/logger.hrl").
--include_lib("eldap/include/eldap.hrl").
 
 -behaviour(emqx_authn_provider).
 
-%% a compatible attribute for version 4.x
--define(ISENABLED_ATTR, "isEnabled").
--define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]).
-%% TODO
-%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512
-%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112
-
 -export([
     create/2,
     update/2,
@@ -69,163 +60,25 @@ authenticate(#{auth_method := _}, _) ->
     ignore;
 authenticate(#{password := undefined}, _) ->
     {error, bad_username_or_password};
-authenticate(
+authenticate(Credential, #{method := #{type := Type}} = State) ->
-    #{password := Password} = Credential,
+    case Type of
-    #{
+        hash ->
-        password_attribute := PasswordAttr,
+            emqx_authn_ldap_hash:authenticate(Credential, State);
-        is_superuser_attribute := IsSuperuserAttr,
+        bind ->
-        query_timeout := Timeout,
+            emqx_authn_ldap_bind:authenticate(Credential, State)
-        resource_id := ResourceId
-    } = State
-) ->
-    case
-        emqx_resource:simple_sync_query(
-            ResourceId,
-            {query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout}
-        )
-    of
-        {ok, []} ->
-            ignore;
-        {ok, [Entry]} ->
-            is_enabled(Password, Entry, State);
-        {error, Reason} ->
-            ?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{
-                resource => ResourceId,
-                timeout => Timeout,
-                reason => Reason
-            }),
-            ignore
     end.
 
+%% it used the deprecated config form
+parse_config(
+    #{password_attribute := PasswordAttr, is_superuser_attribute := IsSuperuserAttr} = Config0
+) ->
+    Config = maps:without([password_attribute, is_superuser_attribute], Config0),
+    parse_config(Config#{
+        method => #{
+            type => hash,
+            password_attribute => PasswordAttr,
+            is_superuser_attribute => IsSuperuserAttr
+        }
+    });
 parse_config(Config) ->
-    maps:with([query_timeout, password_attribute, is_superuser_attribute], Config).
+    maps:with([query_timeout, method], Config).
-
-%% To compatible v4.x
-is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) ->
-    IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"),
-    case emqx_authn_utils:to_bool(IsEnabled) of
-        true ->
-            ensure_password(Password, Entry, State);
-        _ ->
-            {error, user_disabled}
-    end.
-
-ensure_password(
-    Password,
-    #eldap_entry{attributes = Attributes} = Entry,
-    #{password_attribute := PasswordAttr} = State
-) ->
-    case get_value(PasswordAttr, Attributes) of
-        undefined ->
-            {error, no_password};
-        [LDAPPassword | _] ->
-            extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State)
-    end.
-
-%% RFC 2307 format password
-%% https://datatracker.ietf.org/doc/html/rfc2307
-extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) ->
-    case
-        re:run(
-            LDAPPassword,
-            "{([^{}]+)}(.+)",
-            [{capture, all_but_first, list}, global]
-        )
-    of
-        {match, [[HashTypeStr, PasswordHashStr]]} ->
-            case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of
-                {ok, HashType} ->
-                    PasswordHash = to_binary(PasswordHashStr),
-                    is_valid_algorithm(HashType, PasswordHash, Password, Entry, State);
-                _Error ->
-                    {error, invalid_hash_type}
-            end;
-        _ ->
-            OnFail(LDAPPassword, Password, Entry, State)
-    end.
-
-is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) ->
-    case lists:member(HashType, ?VALID_ALGORITHMS) of
-        true ->
-            verify_password(HashType, PasswordHash, Password, Entry, State);
-        _ ->
-            {error, {invalid_hash_type, HashType}}
-    end.
-
-%% this password is in LDIF format which is base64 encoding
-try_decode_password(LDAPPassword, Password, Entry, State) ->
-    case safe_base64_decode(LDAPPassword) of
-        {ok, Decode} ->
-            extract_hash_algorithm(
-                Decode,
-                Password,
-                fun(_, _, _, _) ->
-                    {error, invalid_password}
-                end,
-                Entry,
-                State
-            );
-        {error, Reason} ->
-            {error, {invalid_password, Reason}}
-    end.
-
-%% sha with salt
-%% https://www.openldap.org/faq/data/cache/347.html
-verify_password(ssha, PasswordData, Password, Entry, State) ->
-    case safe_base64_decode(PasswordData) of
-        {ok, <<PasswordHash:20/binary, Salt/binary>>} ->
-            verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State);
-        {ok, _} ->
-            {error, invalid_ssha_password};
-        {error, Reason} ->
-            {error, {invalid_password, Reason}}
-    end;
-verify_password(
-    Algorithm,
-    Base64HashData,
-    Password,
-    Entry,
-    State
-) ->
-    verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State).
-
-verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) ->
-    PasswordHash = hash_password(Algorithm, Salt, Position, Password),
-    case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of
-        true ->
-            {ok, is_superuser(Entry, State)};
-        _ ->
-            {error, bad_username_or_password}
-    end.
-
-is_superuser(Entry, #{is_superuser_attribute := Attr} = _State) ->
-    Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"),
-    #{is_superuser => emqx_authn_utils:to_bool(Value)}.
-
-safe_base64_decode(Data) ->
-    try
-        {ok, base64:decode(Data)}
-    catch
-        _:Reason ->
-            {error, {invalid_base64_data, Reason}}
-    end.
-
-get_lower_bin_value(Key, Proplists, Default) ->
-    [Value | _] = get_value(Key, Proplists, [Default]),
-    to_binary(string:to_lower(Value)).
-
-to_binary(Value) ->
-    erlang:list_to_binary(Value).
-
-hash_password(Algorithm, _Salt, disable, Password) ->
-    hash_password(Algorithm, Password);
-hash_password(Algorithm, Salt, suffix, Password) ->
-    hash_password(Algorithm, <<Password/binary, Salt/binary>>).
-
-hash_password(Algorithm, Data) ->
-    crypto:hash(Algorithm, Data).
-
-compare_password(hash, LDAPPasswordHash, PasswordHash) ->
-    emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash);
-compare_password(base64, Base64HashData, PasswordHash) ->
-    emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)).

apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl

@@ -20,32 +20,13 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("eldap/include/eldap.hrl").
 
--behaviour(emqx_authn_provider).
-
 -export([
-    create/2,
+    authenticate/2
-    update/2,
-    authenticate/2,
-    destroy/1
 ]).
 
 %%------------------------------------------------------------------------------
 %% APIs
 %%------------------------------------------------------------------------------
-
-create(_AuthenticatorID, Config) ->
-    emqx_authn_ldap:do_create(?MODULE, Config).
-
-update(Config, State) ->
-    emqx_authn_ldap:update(Config, State).
-
-destroy(State) ->
-    emqx_authn_ldap:destroy(State).
-
-authenticate(#{auth_method := _}, _) ->
-    ignore;
-authenticate(#{password := undefined}, _) ->
-    {error, bad_username_or_password};
 authenticate(
     #{password := _Password} = Credential,
     #{

apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl

@@ -1,66 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_authn_ldap_bind_schema).
-
--behaviour(emqx_authn_schema).
-
--export([
-    fields/1,
-    desc/1,
-    refs/0,
-    select_union_member/1,
-    namespace/0
-]).
-
--include("emqx_auth_ldap.hrl").
--include_lib("hocon/include/hoconsc.hrl").
-
-namespace() -> "authn".
-
-refs() ->
-    [?R_REF(ldap_bind)].
-
-select_union_member(#{
-    <<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIND_BIN
-}) ->
-    refs();
-select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIND_BIN}) ->
-    throw(#{
-        reason => "unknown_mechanism",
-        expected => ?AUTHN_MECHANISM
-    });
-select_union_member(_) ->
-    undefined.
-
-fields(ldap_bind) ->
-    [
-        {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)},
-        {backend, emqx_authn_schema:backend(?AUTHN_BACKEND_BIND)},
-        {query_timeout, fun query_timeout/1}
-    ] ++
-        emqx_authn_schema:common_fields() ++
-        emqx_ldap:fields(config) ++ emqx_ldap:fields(bind_opts).
-
-desc(ldap_bind) ->
-    ?DESC(ldap_bind);
-desc(_) ->
-    undefined.
-
-query_timeout(type) -> emqx_schema:timeout_duration_ms();
-query_timeout(desc) -> ?DESC(?FUNCTION_NAME);
-query_timeout(default) -> <<"5s">>;
-query_timeout(_) -> undefined.

apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl

@@ -0,0 +1,197 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authn_ldap_hash).
+
+-include_lib("emqx_auth/include/emqx_authn.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("eldap/include/eldap.hrl").
+
+%% a compatible attribute for version 4.x
+-define(ISENABLED_ATTR, "isEnabled").
+-define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]).
+%% TODO
+%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512
+%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112
+
+-export([
+    authenticate/2
+]).
+
+-import(proplists, [get_value/2, get_value/3]).
+
+%%------------------------------------------------------------------------------
+%% APIs
+%%------------------------------------------------------------------------------
+authenticate(
+    #{password := Password} = Credential,
+    #{
+        method := #{
+            password_attribute := PasswordAttr,
+            is_superuser_attribute := IsSuperuserAttr
+        },
+        query_timeout := Timeout,
+        resource_id := ResourceId
+    } = State
+) ->
+    case
+        emqx_resource:simple_sync_query(
+            ResourceId,
+            {query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout}
+        )
+    of
+        {ok, []} ->
+            ignore;
+        {ok, [Entry]} ->
+            is_enabled(Password, Entry, State);
+        {error, Reason} ->
+            ?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{
+                resource => ResourceId,
+                timeout => Timeout,
+                reason => Reason
+            }),
+            ignore
+    end.
+
+%% To compatible v4.x
+is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) ->
+    IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"),
+    case emqx_authn_utils:to_bool(IsEnabled) of
+        true ->
+            ensure_password(Password, Entry, State);
+        _ ->
+            {error, user_disabled}
+    end.
+
+ensure_password(
+    Password,
+    #eldap_entry{attributes = Attributes} = Entry,
+    #{method := #{password_attribute := PasswordAttr}} = State
+) ->
+    case get_value(PasswordAttr, Attributes) of
+        undefined ->
+            {error, no_password};
+        [LDAPPassword | _] ->
+            extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State)
+    end.
+
+%% RFC 2307 format password
+%% https://datatracker.ietf.org/doc/html/rfc2307
+extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) ->
+    case
+        re:run(
+            LDAPPassword,
+            "{([^{}]+)}(.+)",
+            [{capture, all_but_first, list}, global]
+        )
+    of
+        {match, [[HashTypeStr, PasswordHashStr]]} ->
+            case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of
+                {ok, HashType} ->
+                    PasswordHash = to_binary(PasswordHashStr),
+                    is_valid_algorithm(HashType, PasswordHash, Password, Entry, State);
+                _Error ->
+                    {error, invalid_hash_type}
+            end;
+        _ ->
+            OnFail(LDAPPassword, Password, Entry, State)
+    end.
+
+is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) ->
+    case lists:member(HashType, ?VALID_ALGORITHMS) of
+        true ->
+            verify_password(HashType, PasswordHash, Password, Entry, State);
+        _ ->
+            {error, {invalid_hash_type, HashType}}
+    end.
+
+%% this password is in LDIF format which is base64 encoding
+try_decode_password(LDAPPassword, Password, Entry, State) ->
+    case safe_base64_decode(LDAPPassword) of
+        {ok, Decode} ->
+            extract_hash_algorithm(
+                Decode,
+                Password,
+                fun(_, _, _, _) ->
+                    {error, invalid_password}
+                end,
+                Entry,
+                State
+            );
+        {error, Reason} ->
+            {error, {invalid_password, Reason}}
+    end.
+
+%% sha with salt
+%% https://www.openldap.org/faq/data/cache/347.html
+verify_password(ssha, PasswordData, Password, Entry, State) ->
+    case safe_base64_decode(PasswordData) of
+        {ok, <<PasswordHash:20/binary, Salt/binary>>} ->
+            verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State);
+        {ok, _} ->
+            {error, invalid_ssha_password};
+        {error, Reason} ->
+            {error, {invalid_password, Reason}}
+    end;
+verify_password(
+    Algorithm,
+    Base64HashData,
+    Password,
+    Entry,
+    State
+) ->
+    verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State).
+
+verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) ->
+    PasswordHash = hash_password(Algorithm, Salt, Position, Password),
+    case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of
+        true ->
+            {ok, is_superuser(Entry, State)};
+        _ ->
+            {error, bad_username_or_password}
+    end.
+
+is_superuser(Entry, #{method := #{is_superuser_attribute := Attr}} = _State) ->
+    Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"),
+    #{is_superuser => emqx_authn_utils:to_bool(Value)}.
+
+safe_base64_decode(Data) ->
+    try
+        {ok, base64:decode(Data)}
+    catch
+        _:Reason ->
+            {error, {invalid_base64_data, Reason}}
+    end.
+
+get_lower_bin_value(Key, Proplists, Default) ->
+    [Value | _] = get_value(Key, Proplists, [Default]),
+    to_binary(string:to_lower(Value)).
+
+to_binary(Value) ->
+    erlang:list_to_binary(Value).
+
+hash_password(Algorithm, _Salt, disable, Password) ->
+    hash_password(Algorithm, Password);
+hash_password(Algorithm, Salt, suffix, Password) ->
+    hash_password(Algorithm, <<Password/binary, Salt/binary>>).
+
+hash_password(Algorithm, Data) ->
+    crypto:hash(Algorithm, Data).
+
+compare_password(hash, LDAPPasswordHash, PasswordHash) ->
+    emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash);
+compare_password(base64, Base64HashData, PasswordHash) ->
+    emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)).

apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl

@@ -32,7 +32,7 @@
 namespace() -> "authn".
 
 refs() ->
-    [?R_REF(ldap)].
+    [?R_REF(ldap), ?R_REF(ldap_deprecated)].
 
 select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN}) ->
     refs();
@@ -44,12 +44,34 @@ select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIN}) ->
 select_union_member(_) ->
     undefined.
 
+fields(ldap_deprecated) ->
+    common_fields() ++
+        [
+            {password_attribute, password_attribute()},
+            {is_superuser_attribute, is_superuser_attribute()}
+        ];
 fields(ldap) ->
+    common_fields() ++
+        [
+            {method,
+                ?HOCON(
+                    ?UNION([?R_REF(hash_method), ?R_REF(bind_method)]),
+                    #{desc => ?DESC(method)}
+                )}
+        ];
+fields(hash_method) ->
+    [
+        {type, method_type(hash)},
+        {password_attribute, password_attribute()},
+        {is_superuser_attribute, is_superuser_attribute()}
+    ];
+fields(bind_method) ->
+    [{type, method_type(bind)}] ++ emqx_ldap:fields(bind_opts).
+
+common_fields() ->
     [
         {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)},
         {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)},
-        {password_attribute, fun password_attribute/1},
-        {is_superuser_attribute, fun is_superuser_attribute/1},
         {query_timeout, fun query_timeout/1}
     ] ++
         emqx_authn_schema:common_fields() ++
@@ -57,18 +79,35 @@ fields(ldap) ->
 
 desc(ldap) ->
     ?DESC(ldap);
+desc(ldap_deprecated) ->
+    ?DESC(ldap_deprecated);
+desc(hash_method) ->
+    ?DESC(hash_method);
+desc(bind_method) ->
+    ?DESC(bind_method);
 desc(_) ->
     undefined.
 
-password_attribute(type) -> string();
+method_type(Type) ->
-password_attribute(desc) -> ?DESC(?FUNCTION_NAME);
+    ?HOCON(?ENUM([Type]), #{desc => ?DESC(?FUNCTION_NAME), default => Type}).
-password_attribute(default) -> <<"userPassword">>;
-password_attribute(_) -> undefined.
 
-is_superuser_attribute(type) -> string();
+password_attribute() ->
-is_superuser_attribute(desc) -> ?DESC(?FUNCTION_NAME);
+    ?HOCON(
-is_superuser_attribute(default) -> <<"isSuperuser">>;
+        string(),
-is_superuser_attribute(_) -> undefined.
+        #{
+            desc => ?DESC(?FUNCTION_NAME),
+            default => <<"userPassword">>
+        }
+    ).
+
+is_superuser_attribute() ->
+    ?HOCON(
+        string(),
+        #{
+            desc => ?DESC(?FUNCTION_NAME),
+            default => <<"isSuperuser">>
+        }
+    ).
 
 query_timeout(type) -> emqx_schema:timeout_duration_ms();
 query_timeout(desc) -> ?DESC(?FUNCTION_NAME);

apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl

@@ -70,6 +70,29 @@ end_per_suite(Config) ->
 %% Tests
 %%------------------------------------------------------------------------------
 
+t_create_with_deprecated_cfg(_Config) ->
+    AuthConfig = deprecated_raw_ldap_auth_config(),
+
+    {ok, _} = emqx:update_config(
+        ?PATH,
+        {create_authenticator, ?GLOBAL, AuthConfig}
+    ),
+
+    {ok, [#{provider := emqx_authn_ldap, state := State}]} = emqx_authn_chains:list_authenticators(
+        ?GLOBAL
+    ),
+    ?assertMatch(
+        #{
+            method := #{
+                type := hash,
+                is_superuser_attribute := _,
+                password_attribute := "not_the_default_value"
+            }
+        },
+        State
+    ),
+    emqx_authn_test_lib:delete_config(?ResourceID).
+
 t_create(_Config) ->
     AuthConfig = raw_ldap_auth_config(),
 
@@ -225,6 +248,19 @@ raw_ldap_auth_config() ->
         <<"pool_size">> => 8
     }.
 
+deprecated_raw_ldap_auth_config() ->
+    #{
+        <<"mechanism">> => <<"password_based">>,
+        <<"backend">> => <<"ldap">>,
+        <<"server">> => ldap_server(),
+        <<"is_superuser_attribute">> => <<"isSuperuser">>,
+        <<"password_attribute">> => <<"not_the_default_value">>,
+        <<"base_dn">> => <<"uid=${username},ou=testdevice,dc=emqx,dc=io">>,
+        <<"username">> => <<"cn=root,dc=emqx,dc=io">>,
+        <<"password">> => <<"public">>,
+        <<"pool_size">> => 8
+    }.
+
 user_seeds() ->
     New = fun(Username, Password, Result) ->
         #{

apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl

@@ -27,7 +27,7 @@
 -define(LDAP_RESOURCE, <<"emqx_authn_ldap_bind_SUITE">>).
 
 -define(PATH, [authentication]).
--define(ResourceID, <<"password_based:ldap_bind">>).
+-define(ResourceID, <<"password_based:ldap">>).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -78,7 +78,7 @@ t_create(_Config) ->
         {create_authenticator, ?GLOBAL, AuthConfig}
     ),
 
-    {ok, [#{provider := emqx_authn_ldap_bind}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
+    {ok, [#{provider := emqx_authn_ldap}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
     emqx_authn_test_lib:delete_config(?ResourceID).
 
 t_create_invalid(_Config) ->
@@ -146,10 +146,10 @@ t_destroy(_Config) ->
         {create_authenticator, ?GLOBAL, AuthConfig}
     ),
 
-    {ok, [#{provider := emqx_authn_ldap_bind, state := State}]} =
+    {ok, [#{provider := emqx_authn_ldap, state := State}]} =
         emqx_authn_chains:list_authenticators(?GLOBAL),
 
-    {ok, _} = emqx_authn_ldap_bind:authenticate(
+    {ok, _} = emqx_authn_ldap:authenticate(
         #{
             username => <<"mqttuser0001">>,
             password => <<"mqttuser0001">>
@@ -165,7 +165,7 @@ t_destroy(_Config) ->
     % Authenticator should not be usable anymore
     ?assertMatch(
         ignore,
-        emqx_authn_ldap_bind:authenticate(
+        emqx_authn_ldap:authenticate(
             #{
                 username => <<"mqttuser0001">>,
                 password => <<"mqttuser0001">>
@@ -199,7 +199,7 @@ t_update(_Config) ->
     % We update with config with correct query, provider should update and work properly
     {ok, _} = emqx:update_config(
         ?PATH,
-        {update_authenticator, ?GLOBAL, <<"password_based:ldap_bind">>, CorrectConfig}
+        {update_authenticator, ?GLOBAL, <<"password_based:ldap">>, CorrectConfig}
     ),
 
     {ok, _} = emqx_access_control:authenticate(
@@ -218,14 +218,17 @@ t_update(_Config) ->
 raw_ldap_auth_config() ->
     #{
         <<"mechanism">> => <<"password_based">>,
-        <<"backend">> => <<"ldap_bind">>,
+        <<"backend">> => <<"ldap">>,
         <<"server">> => ldap_server(),
         <<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>,
         <<"filter">> => <<"(uid=${username})">>,
         <<"username">> => <<"cn=root,dc=emqx,dc=io">>,
         <<"password">> => <<"public">>,
         <<"pool_size">> => 8,
+        <<"method">> => #{
+            <<"type">> => <<"bind">>,
             <<"bind_password">> => <<"${password}">>
+        }
     }.
 
 user_seeds() ->

apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl

@@ -278,6 +278,10 @@ raw_mongo_auth_config() ->
         <<"server">> => mongo_server(),
         <<"w_mode">> => <<"unsafe">>,
 
+        <<"auth_source">> => mongo_authsource(),
+        <<"username">> => mongo_username(),
+        <<"password">> => mongo_password(),
+
         <<"filter">> => #{<<"username">> => <<"${username}">>},
         <<"password_hash_field">> => <<"password_hash">>,
         <<"salt_field">> => <<"salt">>,
@@ -464,9 +468,21 @@ mongo_config() ->
         {database, <<"mqtt">>},
         {host, ?MONGO_HOST},
         {port, ?MONGO_DEFAULT_PORT},
+        {auth_source, mongo_authsource()},
+        {login, mongo_username()},
+        {password, mongo_password()},
         {register, ?MONGO_CLIENT}
     ].
 
+mongo_authsource() ->
+    iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")).
+
+mongo_username() ->
+    iolist_to_binary(os:getenv("MONGO_USERNAME", "")).
+
+mongo_password() ->
+    iolist_to_binary(os:getenv("MONGO_PASSWORD", "")).
+
 start_apps(Apps) ->
     lists:foreach(fun application:ensure_all_started/1, Apps).
 

apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl

@@ -397,6 +397,10 @@ raw_mongo_authz_config() ->
         <<"collection">> => <<"acl">>,
         <<"server">> => mongo_server(),
 
+        <<"auth_source">> => mongo_authsource(),
+        <<"username">> => mongo_username(),
+        <<"password">> => mongo_password(),
+
         <<"filter">> => #{<<"username">> => <<"${username}">>}
     }.
 
@@ -408,9 +412,21 @@ mongo_config() ->
         {database, <<"mqtt">>},
         {host, ?MONGO_HOST},
         {port, ?MONGO_DEFAULT_PORT},
+        {auth_source, mongo_authsource()},
+        {login, mongo_username()},
+        {password, mongo_password()},
         {register, ?MONGO_CLIENT}
     ].
 
+mongo_authsource() ->
+    iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")).
+
+mongo_username() ->
+    iolist_to_binary(os:getenv("MONGO_USERNAME", "")).
+
+mongo_password() ->
+    iolist_to_binary(os:getenv("MONGO_PASSWORD", "")).
+
 start_apps(Apps) ->
     lists:foreach(fun application:ensure_all_started/1, Apps).
 

apps/emqx_bridge/src/emqx_bridge.erl

@@ -93,7 +93,8 @@
     T == iotdb;
     T == kinesis_producer;
     T == greptimedb;
-    T == azure_event_hub_producer
+    T == azure_event_hub_producer;
+    T == syskeeper_forwarder
 ).
 
 -define(ROOT_KEY, bridges).

apps/emqx_bridge/src/emqx_bridge_resource.erl

@@ -356,9 +356,10 @@ parse_confs(<<"iotdb">>, Name, Conf) ->
         authentication :=
             #{
                 username := Username,
-                password := Password
+                password := Secret
             }
     } = Conf,
+    Password = emqx_secret:unwrap(Secret),
     BasicToken = base64:encode(<<Username/binary, ":", Password/binary>>),
     %% This version atom correspond to the macro ?VSN_1_1_X in
     %% emqx_bridge_iotdb.hrl. It would be better to use the macro directly, but

apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_cassandra, [
     {description, "EMQX Enterprise Cassandra Bridge"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl

@@ -70,7 +70,7 @@ cassandra_db_fields() ->
         {keyspace, fun keyspace/1},
         {pool_size, fun emqx_connector_schema_lib:pool_size/1},
         {username, fun emqx_connector_schema_lib:username/1},
-        {password, fun emqx_connector_schema_lib:password/1},
+        {password, emqx_connector_schema_lib:password_field()},
         {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1}
     ].
 
@@ -111,14 +111,14 @@ on_start(
             emqx_schema:parse_servers(Servers0, ?DEFAULT_SERVER_OPTION)
         ),
 
-    Options = [
+    Options =
+        maps:to_list(maps:with([username, password], Config)) ++
+            [
                 {nodes, Servers},
                 {keyspace, Keyspace},
                 {auto_reconnect, ?AUTO_RECONNECT_INTERVAL},
                 {pool_size, PoolSize}
             ],
-    Options1 = maybe_add_opt(username, Config, Options),
-    Options2 = maybe_add_opt(password, Config, Options1, _IsSensitive = true),
 
     SslOpts =
         case maps:get(enable, SSL) of
@@ -131,7 +131,7 @@ on_start(
                 []
         end,
     State = parse_prepare_cql(Config),
-    case emqx_resource_pool:start(InstId, ?MODULE, Options2 ++ SslOpts) of
+    case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of
         ok ->
             {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})};
         {error, Reason} ->
@@ -387,6 +387,7 @@ conn_opts(Opts) ->
 conn_opts([], Acc) ->
     Acc;
 conn_opts([{password, Password} | Opts], Acc) ->
+    %% TODO: teach `ecql` to accept 0-arity closures as passwords.
     conn_opts(Opts, [{password, emqx_secret:unwrap(Password)} | Acc]);
 conn_opts([Opt | Opts], Acc) ->
     conn_opts(Opts, [Opt | Acc]).
@@ -512,19 +513,3 @@ maybe_assign_type(V) when is_integer(V) ->
 maybe_assign_type(V) when is_float(V) -> {double, V};
 maybe_assign_type(V) ->
     V.
-
-maybe_add_opt(Key, Conf, Opts) ->
-    maybe_add_opt(Key, Conf, Opts, _IsSensitive = false).
-
-maybe_add_opt(Key, Conf, Opts, IsSensitive) ->
-    case Conf of
-        #{Key := Val} ->
-            [{Key, maybe_wrap(IsSensitive, Val)} | Opts];
-        _ ->
-            Opts
-    end.
-
-maybe_wrap(false = _IsSensitive, Val) ->
-    Val;
-maybe_wrap(true, Val) ->
-    emqx_secret:wrap(Val).

apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl

@@ -40,10 +40,9 @@ all() ->
     ].
 
 groups() ->
-    TCs = emqx_common_test_helpers:all(?MODULE),
     [
-        {auth, TCs},
+        {auth, [t_lifecycle, t_start_passfile]},
-        {noauth, TCs}
+        {noauth, [t_lifecycle]}
     ].
 
 cassandra_servers(CassandraHost) ->
@@ -115,32 +114,37 @@ end_per_testcase(_, _Config) ->
 
 t_lifecycle(Config) ->
     perform_lifecycle_check(
-        <<"emqx_connector_cassandra_SUITE">>,
+        <<?MODULE_STRING>>,
         cassandra_config(Config)
     ).
 
-show(X) ->
+t_start_passfile(Config) ->
-    erlang:display(X),
+    ResourceID = atom_to_binary(?FUNCTION_NAME),
-    X.
+    PasswordFilename = filename:join(?config(priv_dir, Config), "passfile"),
-
+    ok = file:write_file(PasswordFilename, ?CASSA_PASSWORD),
-show(Label, What) ->
+    InitialConfig = emqx_utils_maps:deep_merge(
-    erlang:display({Label, What}),
+        cassandra_config(Config),
-    What.
+        #{
+            <<"config">> => #{
+                password => iolist_to_binary(["file://", PasswordFilename])
+            }
+        }
+    ),
+    ?assertMatch(
+        #{status := connected},
+        create_local_resource(ResourceID, check_config(InitialConfig))
+    ),
+    ?assertEqual(
+        ok,
+        emqx_resource:remove_local(ResourceID)
+    ).
 
 perform_lifecycle_check(ResourceId, InitialConfig) ->
-    {ok, #{config := CheckedConfig}} =
+    CheckedConfig = check_config(InitialConfig),
-        emqx_resource:check_config(?CASSANDRA_RESOURCE_MOD, InitialConfig),
+    #{
-    {ok, #{
         state := #{pool_name := PoolName} = State,
         status := InitialStatus
-    }} =
+    } = create_local_resource(ResourceId, CheckedConfig),
-        emqx_resource:create_local(
-            ResourceId,
-            ?CONNECTOR_RESOURCE_GROUP,
-            ?CASSANDRA_RESOURCE_MOD,
-            CheckedConfig,
-            #{}
-        ),
     ?assertEqual(InitialStatus, connected),
     % Instance should match the state and status of the just started resource
     {ok, ?CONNECTOR_RESOURCE_GROUP, #{
@@ -191,6 +195,21 @@ perform_lifecycle_check(ResourceId, InitialConfig) ->
 %% utils
 %%--------------------------------------------------------------------
 
+check_config(Config) ->
+    {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?CASSANDRA_RESOURCE_MOD, Config),
+    CheckedConfig.
+
+create_local_resource(ResourceId, CheckedConfig) ->
+    {ok, Bridge} =
+        emqx_resource:create_local(
+            ResourceId,
+            ?CONNECTOR_RESOURCE_GROUP,
+            ?CASSANDRA_RESOURCE_MOD,
+            CheckedConfig,
+            #{}
+        ),
+    Bridge.
+
 cassandra_config(Config) ->
     Host = ?config(cassa_host, Config),
     AuthOpts = maps:from_list(?config(cassa_auth_opts, Config)),

apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_clickhouse, [
     {description, "EMQX Enterprise ClickHouse Bridge"},
-    {vsn, "0.2.3"},
+    {vsn, "0.2.4"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl

@@ -145,7 +145,7 @@ on_start(
     Options = [
         {url, URL},
         {user, maps:get(username, Config, "default")},
-        {key, emqx_secret:wrap(maps:get(password, Config, "public"))},
+        {key, maps:get(password, Config, emqx_secret:wrap("public"))},
         {database, DB},
         {auto_reconnect, ?AUTO_RECONNECT_INTERVAL},
         {pool_size, PoolSize},
@@ -243,6 +243,7 @@ connect(Options) ->
     URL = iolist_to_binary(emqx_http_lib:normalize(proplists:get_value(url, Options))),
     User = proplists:get_value(user, Options),
     Database = proplists:get_value(database, Options),
+    %% TODO: teach `clickhouse` to accept 0-arity closures as passwords.
     Key = emqx_secret:unwrap(proplists:get_value(key, Options)),
     Pool = proplists:get_value(pool, Options),
     PoolSize = proplists:get_value(pool_size, Options),

apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl

@@ -10,10 +10,12 @@
 -include("emqx_connector.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("stdlib/include/assert.hrl").
+-include_lib("common_test/include/ct.hrl").
 
 -define(APP, emqx_bridge_clickhouse).
 -define(CLICKHOUSE_HOST, "clickhouse").
 -define(CLICKHOUSE_RESOURCE_MOD, emqx_bridge_clickhouse_connector).
+-define(CLICKHOUSE_PASSWORD, "public").
 
 %% This test SUITE requires a running clickhouse instance. If you don't want to
 %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script
@@ -57,7 +59,7 @@ init_per_suite(Config) ->
                 clickhouse:start_link([
                     {url, clickhouse_url()},
                     {user, <<"default">>},
-                    {key, "public"},
+                    {key, ?CLICKHOUSE_PASSWORD},
                     {pool, tmp_pool}
                 ]),
             {ok, _, _} = clickhouse:query(Conn, <<"CREATE DATABASE IF NOT EXISTS mqtt">>, #{}),
@@ -92,6 +94,31 @@ t_lifecycle(_Config) ->
         clickhouse_config()
     ).
 
+t_start_passfile(Config) ->
+    ResourceID = atom_to_binary(?FUNCTION_NAME),
+    PasswordFilename = filename:join(?config(priv_dir, Config), "passfile"),
+    ok = file:write_file(PasswordFilename, <<?CLICKHOUSE_PASSWORD>>),
+    InitialConfig = clickhouse_config(#{
+        password => iolist_to_binary(["file://", PasswordFilename])
+    }),
+    {ok, #{config := ResourceConfig}} =
+        emqx_resource:check_config(?CLICKHOUSE_RESOURCE_MOD, InitialConfig),
+    ?assertMatch(
+        {ok, #{status := connected}},
+        emqx_resource:create_local(
+            ResourceID,
+            ?CONNECTOR_RESOURCE_GROUP,
+            ?CLICKHOUSE_RESOURCE_MOD,
+            ResourceConfig,
+            #{}
+        )
+    ),
+    ?assertEqual(
+        ok,
+        emqx_resource:remove_local(ResourceID)
+    ),
+    ok.
+
 show(X) ->
     erlang:display(X),
     X.
@@ -168,12 +195,15 @@ perform_lifecycle_check(ResourceID, InitialConfig) ->
 % %%------------------------------------------------------------------------------
 
 clickhouse_config() ->
+    clickhouse_config(#{}).
+
+clickhouse_config(Overrides) ->
     Config =
         #{
             auto_reconnect => true,
             database => <<"mqtt">>,
             username => <<"default">>,
-            password => <<"public">>,
+            password => <<?CLICKHOUSE_PASSWORD>>,
             pool_size => 8,
             url => iolist_to_binary(
                 io_lib:format(
@@ -186,7 +216,7 @@ clickhouse_config() ->
             ),
             connect_timeout => <<"10s">>
         },
-    #{<<"config">> => Config}.
+    #{<<"config">> => maps:merge(Config, Overrides)}.
 
 test_query_no_params() ->
     {query, <<"SELECT 1">>}.

apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_dynamo, [
     {description, "EMQX Enterprise Dynamo Bridge"},
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl

@@ -45,12 +45,10 @@ fields(config) ->
                 #{required => true, desc => ?DESC("aws_access_key_id")}
             )},
         {aws_secret_access_key,
-            mk(
+            emqx_schema_secret:mk(
-                binary(),
                 #{
                     required => true,
-                    desc => ?DESC("aws_secret_access_key"),
+                    desc => ?DESC("aws_secret_access_key")
-                    sensitive => true
                 }
             )},
         {pool_size, fun emqx_connector_schema_lib:pool_size/1},
@@ -89,7 +87,7 @@ on_start(
             host => Host,
             port => Port,
             aws_access_key_id => to_str(AccessKeyID),
-            aws_secret_access_key => to_str(SecretAccessKey),
+            aws_secret_access_key => SecretAccessKey,
             schema => Schema
         }},
         {pool_size, PoolSize}
@@ -182,9 +180,8 @@ do_query(
     end.
 
 connect(Opts) ->
-    Options = proplists:get_value(config, Opts),
+    Config = proplists:get_value(config, Opts),
-    {ok, _Pid} = Result = emqx_bridge_dynamo_connector_client:start_link(Options),
+    {ok, _Pid} = emqx_bridge_dynamo_connector_client:start_link(Config).
-    Result.
 
 parse_template(Config) ->
     Templates =

apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl

@@ -20,8 +20,7 @@
     handle_cast/2,
     handle_info/2,
     terminate/2,
-    code_change/3,
+    code_change/3
-    format_status/2
 ]).
 
 -ifdef(TEST).
@@ -62,11 +61,13 @@ start_link(Options) ->
 %% Initialize dynamodb data bridge
 init(#{
     aws_access_key_id := AccessKeyID,
-    aws_secret_access_key := SecretAccessKey,
+    aws_secret_access_key := Secret,
     host := Host,
     port := Port,
     schema := Schema
 }) ->
+    %% TODO: teach `erlcloud` to to accept 0-arity closures as passwords.
+    SecretAccessKey = to_str(emqx_secret:unwrap(Secret)),
     erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema),
     {ok, #{}}.
 
@@ -101,13 +102,6 @@ terminate(_Reason, _State) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
--spec format_status(
-    Opt :: normal | terminate,
-    Status :: list()
-) -> Status :: term().
-format_status(_Opt, Status) ->
-    Status.
-
 %%%===================================================================
 %%% Internal functions
 %%%===================================================================
@@ -184,3 +178,8 @@ convert2binary(Value) when is_list(Value) ->
     unicode:characters_to_binary(Value);
 convert2binary(Value) when is_map(Value) ->
     emqx_utils_json:encode(Value).
+
+to_str(List) when is_list(List) ->
+    List;
+to_str(Bin) when is_binary(Bin) ->
+    erlang:binary_to_list(Bin).

apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl

@@ -22,8 +22,6 @@
 -define(BATCH_SIZE, 10).
 -define(PAYLOAD, <<"HELLO">>).
 
--define(GET_CONFIG(KEY__, CFG__), proplists:get_value(KEY__, CFG__)).
-
 %% How to run it locally (all commands are run in $PROJ_ROOT dir):
 %% run ct in docker container
 %% run script:
@@ -84,7 +82,9 @@ end_per_group(_Group, _Config) ->
     ok.
 
 init_per_suite(Config) ->
-    Config.
+    SecretFile = filename:join(?config(priv_dir, Config), "secret"),
+    ok = file:write_file(SecretFile, <<?SECRET_ACCESS_KEY>>),
+    [{dynamo_secretfile, SecretFile} | Config].
 
 end_per_suite(_Config) ->
     emqx_mgmt_api_test_util:end_suite(),
@@ -158,32 +158,35 @@ common_init(ConfigT) ->
     end.
 
 dynamo_config(BridgeType, Config) ->
-    Port = integer_to_list(?GET_CONFIG(port, Config)),
+    Host = ?config(host, Config),
-    Url = "http://" ++ ?GET_CONFIG(host, Config) ++ ":" ++ Port,
+    Port = ?config(port, Config),
     Name = atom_to_binary(?MODULE),
-    BatchSize = ?GET_CONFIG(batch_size, Config),
+    BatchSize = ?config(batch_size, Config),
-    QueryMode = ?GET_CONFIG(query_mode, Config),
+    QueryMode = ?config(query_mode, Config),
+    SecretFile = ?config(dynamo_secretfile, Config),
     ConfigString =
         io_lib:format(
-            "bridges.~s.~s {\n"
+            "bridges.~s.~s {"
-            "  enable = true\n"
+            "\n   enable = true"
-            "  url = ~p\n"
+            "\n   url = \"http://~s:~p\""
-            "  table = ~p\n"
+            "\n   table = ~p"
-            "  aws_access_key_id = ~p\n"
+            "\n   aws_access_key_id = ~p"
-            "  aws_secret_access_key = ~p\n"
+            "\n   aws_secret_access_key = ~p"
-            "  resource_opts = {\n"
+            "\n   resource_opts = {"
-            "    request_ttl = 500ms\n"
+            "\n     request_ttl = 500ms"
-            "    batch_size = ~b\n"
+            "\n     batch_size = ~b"
-            "    query_mode = ~s\n"
+            "\n     query_mode = ~s"
-            "  }\n"
+            "\n   }"
-            "}",
+            "\n }",
             [
                 BridgeType,
                 Name,
-                Url,
+                Host,
+                Port,
                 ?TABLE,
                 ?ACCESS_KEY_ID,
-                ?SECRET_ACCESS_KEY,
+                %% NOTE: using file-based secrets with HOCON configs
+                "file://" ++ SecretFile,
                 BatchSize,
                 QueryMode
             ]
@@ -252,8 +255,8 @@ delete_table(_Config) ->
     erlcloud_ddb2:delete_table(?TABLE_BIN).
 
 setup_dynamo(Config) ->
-    Host = ?GET_CONFIG(host, Config),
+    Host = ?config(host, Config),
-    Port = ?GET_CONFIG(port, Config),
+    Port = ?config(port, Config),
     erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, Host, Port, ?SCHEMA).
 
 directly_setup_dynamo() ->
@@ -313,7 +316,9 @@ t_setup_via_http_api_and_publish(Config) ->
     PgsqlConfig0 = ?config(dynamo_config, Config),
     PgsqlConfig = PgsqlConfig0#{
         <<"name">> => Name,
-        <<"type">> => BridgeType
+        <<"type">> => BridgeType,
+        %% NOTE: using literal secret with HTTP API requests.
+        <<"aws_secret_access_key">> => <<?SECRET_ACCESS_KEY>>
     },
     ?assertMatch(
         {ok, _},
@@ -400,7 +405,7 @@ t_simple_query(Config) ->
     ),
     Request = {get_item, {<<"id">>, <<"not_exists">>}},
     Result = query_resource(Config, Request),
-    case ?GET_CONFIG(batch_size, Config) of
+    case ?config(batch_size, Config) of
         ?BATCH_SIZE ->
             ?assertMatch({error, {unrecoverable_error, {invalid_request, _}}}, Result);
         1 ->

apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl

@@ -147,13 +147,7 @@ fields(greptimedb) ->
         [
             {dbname, mk(binary(), #{required => true, desc => ?DESC("dbname")})},
             {username, mk(binary(), #{desc => ?DESC("username")})},
-            {password,
+            {password, emqx_schema_secret:mk(#{desc => ?DESC("password")})}
-                mk(binary(), #{
-                    desc => ?DESC("password"),
-                    format => <<"password">>,
-                    sensitive => true,
-                    converter => fun emqx_schema:password_converter/2
-                })}
         ] ++ emqx_connector_schema_lib:ssl_fields().
 
 server() ->
@@ -302,7 +296,8 @@ ssl_config(SSL = #{enable := true}) ->
 
 auth(#{username := Username, password := Password}) ->
     [
-        {auth, {basic, #{username => str(Username), password => str(Password)}}}
+        %% TODO: teach `greptimedb` to accept 0-arity closures as passwords.
+        {auth, {basic, #{username => str(Username), password => emqx_secret:unwrap(Password)}}}
     ];
 auth(_) ->
     [].

apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl

@@ -192,20 +192,14 @@ fields(influxdb_api_v1) ->
         [
             {database, mk(binary(), #{required => true, desc => ?DESC("database")})},
             {username, mk(binary(), #{desc => ?DESC("username")})},
-            {password,
+            {password, emqx_schema_secret:mk(#{desc => ?DESC("password")})}
-                mk(binary(), #{
-                    desc => ?DESC("password"),
-                    format => <<"password">>,
-                    sensitive => true,
-                    converter => fun emqx_schema:password_converter/2
-                })}
         ] ++ emqx_connector_schema_lib:ssl_fields();
 fields(influxdb_api_v2) ->
     fields(common) ++
         [
             {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})},
             {org, mk(binary(), #{required => true, desc => ?DESC("org")})},
-            {token, mk(binary(), #{required => true, desc => ?DESC("token")})}
+            {token, emqx_schema_secret:mk(#{required => true, desc => ?DESC("token")})}
         ] ++ emqx_connector_schema_lib:ssl_fields().
 
 server() ->
@@ -363,7 +357,8 @@ protocol_config(#{
         {version, v2},
         {bucket, str(Bucket)},
         {org, str(Org)},
-        {token, Token}
+        %% TODO: teach `influxdb` to accept 0-arity closures as passwords.
+        {token, emqx_secret:unwrap(Token)}
     ] ++ ssl_config(SSL).
 
 ssl_config(#{enable := false}) ->
@@ -383,7 +378,8 @@ username(_) ->
     [].
 
 password(#{password := Password}) ->
-    [{password, str(Password)}];
+    %% TODO: teach `influxdb` to accept 0-arity closures as passwords.
+    [{password, str(emqx_secret:unwrap(Password))}];
 password(_) ->
     [].
 

apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge_iotdb, [
     {description, "EMQX Enterprise Apache IoTDB Bridge"},
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {modules, [
         emqx_bridge_iotdb,
         emqx_bridge_iotdb_impl

apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl

@@ -51,12 +51,9 @@ fields(auth_basic) ->
     [
         {username, mk(binary(), #{required => true, desc => ?DESC("config_auth_basic_username")})},
         {password,
-            mk(binary(), #{
+            emqx_schema_secret:mk(#{
                 required => true,
-                desc => ?DESC("config_auth_basic_password"),
+                desc => ?DESC("config_auth_basic_password")
-                format => <<"password">>,
-                sensitive => true,
-                converter => fun emqx_schema:password_converter/2
             })}
     ].
 

apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl

@@ -283,11 +283,9 @@ fields(auth_username_password) ->
             })},
         {username, mk(binary(), #{required => true, desc => ?DESC(auth_sasl_username)})},
         {password,
-            mk(binary(), #{
+            emqx_connector_schema_lib:password_field(#{
                 required => true,
-                sensitive => true,
+                desc => ?DESC(auth_sasl_password)
-                desc => ?DESC(auth_sasl_password),
-                converter => fun emqx_schema:password_converter/2
             })}
     ];
 fields(auth_gssapi_kerberos) ->

apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl

@@ -31,8 +31,8 @@ make_client_id(BridgeType0, BridgeName0) ->
 
 sasl(none) ->
     undefined;
-sasl(#{mechanism := Mechanism, username := Username, password := Password}) ->
+sasl(#{mechanism := Mechanism, username := Username, password := Secret}) ->
-    {Mechanism, Username, emqx_secret:wrap(Password)};
+    {Mechanism, Username, Secret};
 sasl(#{
     kerberos_principal := Principal,
     kerberos_keytab_file := KeyTabFile

apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl

@@ -30,29 +30,41 @@ all() ->
     ].
 
 groups() ->
-    AllTCs = emqx_common_test_helpers:all(?MODULE),
+    SASLGroups = [
-    SASLAuths = [
+        {sasl_auth_plain, testcases(sasl)},
-        sasl_auth_plain,
+        {sasl_auth_scram256, testcases(sasl)},
-        sasl_auth_scram256,
+        {sasl_auth_scram512, testcases(sasl)},
-        sasl_auth_scram512,
+        {sasl_auth_kerberos, testcases(sasl_auth_kerberos)}
-        sasl_auth_kerberos
     ],
-    SASLAuthGroups = [{group, Type} || Type <- SASLAuths],
+    SASLAuthGroups = [{group, Group} || {Group, _} <- SASLGroups],
-    OnlyOnceTCs = only_once_tests(),
-    MatrixTCs = AllTCs -- OnlyOnceTCs,
-    SASLTests = [{Group, MatrixTCs} || Group <- SASLAuths],
     [
-        {plain, MatrixTCs ++ OnlyOnceTCs},
+        {plain, testcases(plain)},
-        {ssl, MatrixTCs},
+        {ssl, testcases(common)},
         {sasl_plain, SASLAuthGroups},
         {sasl_ssl, SASLAuthGroups}
-    ] ++ SASLTests.
+        | SASLGroups
-
+    ].
-sasl_only_tests() ->
-    [t_failed_creation_then_fixed].
 
+testcases(all) ->
+    emqx_common_test_helpers:all(?MODULE);
+testcases(plain) ->
+    %% NOTE: relevant only for a subset of SASL testcases
+    Exclude = [t_failed_creation_then_fixed],
+    testcases(all) -- Exclude;
+testcases(common) ->
+    testcases(plain) -- testcases(once);
+testcases(sasl) ->
+    testcases(all) -- testcases(once);
+testcases(sasl_auth_kerberos) ->
+    %% NOTE: need a proxy to run these tests
+    Exclude = [
+        t_failed_creation_then_fixed,
+        t_on_get_status,
+        t_receive_after_recovery
+    ],
+    testcases(sasl) -- Exclude;
+testcases(once) ->
     %% tests that do not need to be run on all groups
-only_once_tests() ->
     [
         t_begin_offset_earliest,
         t_bridge_rule_action_source,
@@ -220,7 +232,7 @@ init_per_group(sasl_auth_kerberos, Config0) ->
                 (KV) ->
                     KV
             end,
-            [{has_proxy, false}, {sasl_auth_mechanism, kerberos} | Config0]
+            [{sasl_auth_mechanism, kerberos} | Config0]
         ),
     Config;
 init_per_group(_Group, Config) ->
@@ -264,43 +276,6 @@ end_per_group(Group, Config) when
 end_per_group(_Group, _Config) ->
     ok.
 
-init_per_testcase(TestCase, Config) when
-    TestCase =:= t_failed_creation_then_fixed
-->
-    KafkaType = ?config(kafka_type, Config),
-    AuthMechanism = ?config(sasl_auth_mechanism, Config),
-    IsSASL = lists:member(KafkaType, [sasl_plain, sasl_ssl]),
-    case {IsSASL, AuthMechanism} of
-        {true, kerberos} ->
-            [{skip_does_not_apply, true}];
-        {true, _} ->
-            common_init_per_testcase(TestCase, Config);
-        {false, _} ->
-            [{skip_does_not_apply, true}]
-    end;
-init_per_testcase(TestCase, Config) when
-    TestCase =:= t_failed_creation_then_fixed
-->
-    %% test with one partiton only for this case because
-    %% the wait probe may not be always sent to the same partition
-    HasProxy = proplists:get_value(has_proxy, Config, true),
-    case HasProxy of
-        false ->
-            [{skip_does_not_apply, true}];
-        true ->
-            common_init_per_testcase(TestCase, [{num_partitions, 1} | Config])
-    end;
-init_per_testcase(TestCase, Config) when
-    TestCase =:= t_on_get_status;
-    TestCase =:= t_receive_after_recovery
-->
-    HasProxy = proplists:get_value(has_proxy, Config, true),
-    case HasProxy of
-        false ->
-            [{skip_does_not_apply, true}];
-        true ->
-            common_init_per_testcase(TestCase, Config)
-    end;
 init_per_testcase(t_cluster_group = TestCase, Config0) ->
     Config = emqx_utils:merge_opts(Config0, [{num_partitions, 6}]),
     common_init_per_testcase(TestCase, Config);
@@ -393,10 +368,6 @@ common_init_per_testcase(TestCase, Config0) ->
     ].
 
 end_per_testcase(_Testcase, Config) ->
-    case proplists:get_bool(skip_does_not_apply, Config) of
-        true ->
-            ok;
-        false ->
     ProxyHost = ?config(proxy_host, Config),
     ProxyPort = ?config(proxy_port, Config),
     ProducersConfigs = ?config(kafka_producers, Config),
@@ -414,9 +385,7 @@ end_per_testcase(_Testcase, Config) ->
     %% in CI, apparently this needs more time since the
     %% machines struggle with all the containers running...
     emqx_common_test_helpers:call_janitor(60_000),
-            ok = snabbkaffe:stop(),
+    ok = snabbkaffe:stop().
-            ok
-    end.
 
 %%------------------------------------------------------------------------------
 %% Helper fns
@@ -1391,14 +1360,6 @@ t_multiple_topic_mappings(Config) ->
     ok.
 
 t_on_get_status(Config) ->
-    case proplists:get_bool(skip_does_not_apply, Config) of
-        true ->
-            ok;
-        false ->
-            do_t_on_get_status(Config)
-    end.
-
-do_t_on_get_status(Config) ->
     ProxyPort = ?config(proxy_port, Config),
     ProxyHost = ?config(proxy_host, Config),
     ProxyName = ?config(proxy_name, Config),
@@ -1421,14 +1382,6 @@ do_t_on_get_status(Config) ->
 %% ensure that we can create and use the bridge successfully after
 %% creating it with bad config.
 t_failed_creation_then_fixed(Config) ->
-    case proplists:get_bool(skip_does_not_apply, Config) of
-        true ->
-            ok;
-        false ->
-            ?check_trace(do_t_failed_creation_then_fixed(Config), [])
-    end.
-
-do_t_failed_creation_then_fixed(Config) ->
     ct:timetrap({seconds, 180}),
     MQTTTopic = ?config(mqtt_topic, Config),
     MQTTQoS = ?config(mqtt_qos, Config),
@@ -1516,14 +1469,6 @@ do_t_failed_creation_then_fixed(Config) ->
 %% recovering from a network partition will make the subscribers
 %% consume the messages produced during the down time.
 t_receive_after_recovery(Config) ->
-    case proplists:get_bool(skip_does_not_apply, Config) of
-        true ->
-            ok;
-        false ->
-            do_t_receive_after_recovery(Config)
-    end.
-
-do_t_receive_after_recovery(Config) ->
     ct:timetrap(120_000),
     ProxyPort = ?config(proxy_port, Config),
     ProxyHost = ?config(proxy_host, Config),

apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl

@@ -28,13 +28,8 @@
 ).
 
 -include_lib("eunit/include/eunit.hrl").
--include_lib("emqx/include/emqx.hrl").
--include_lib("emqx_dashboard/include/emqx_dashboard.hrl").
 
 -define(HOST, "http://127.0.0.1:18083").
-
-%% -define(API_VERSION, "v5").
-
 -define(BASE_PATH, "/api/v5").
 
 %% NOTE: it's "kafka", but not "kafka_producer"
@@ -48,13 +43,6 @@
 %%------------------------------------------------------------------------------
 
 all() ->
-    case code:get_object_code(cthr) of
-        {Module, Code, Filename} ->
-            {module, Module} = code:load_binary(Module, Filename, Code),
-            ok;
-        error ->
-            error
-    end,
     All0 = emqx_common_test_helpers:all(?MODULE),
     All = All0 -- matrix_cases(),
     Groups = lists:map(fun({G, _, _}) -> {group, G} end, groups()),
@@ -105,23 +93,12 @@ init_per_suite(Config0) ->
             emqx_connector,
             emqx_bridge_kafka,
             emqx_bridge,
-            emqx_rule_engine
+            emqx_rule_engine,
+            {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}
         ],
         #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
-    emqx_mgmt_api_test_util:init_suite(),
     wait_until_kafka_is_up(),
-    %% Wait until bridges API is up
-    (fun WaitUntilRestApiUp() ->
-        case http_get(["bridges"]) of
-            {ok, 200, _Res} ->
-                ok;
-            Val ->
-                ct:pal("REST API for bridges not up. Wait and try again. Response: ~p", [Val]),
-                timer:sleep(1000),
-                WaitUntilRestApiUp()
-        end
-    end)(),
     [{apps, Apps} | Config].
 
 end_per_suite(Config) ->
@@ -183,6 +160,7 @@ t_query_mode_async(CtConfig) ->
 t_publish(matrix) ->
     {publish, [
         [tcp, none, key_dispatch, sync],
+        [ssl, plain_passfile, random, sync],
         [ssl, scram_sha512, random, async],
         [ssl, kerberos, random, sync]
     ]};
@@ -200,9 +178,15 @@ t_publish(Config) ->
         end,
     Auth1 =
         case Auth of
-            none -> "none";
+            none ->
-            scram_sha512 -> valid_sasl_scram512_settings();
+                "none";
-            kerberos -> valid_sasl_kerberos_settings()
+            plain_passfile ->
+                Passfile = filename:join(?config(priv_dir, Config), "passfile"),
+                valid_sasl_plain_passfile_settings(Passfile);
+            scram_sha512 ->
+                valid_sasl_scram512_settings();
+            kerberos ->
+                valid_sasl_kerberos_settings()
         end,
     ConnCfg = #{
         "bootstrap_hosts" => Hosts,
@@ -1018,112 +1002,89 @@ hocon_config(Args, ConfigTemplateFun) ->
     ),
     Hocon.
 
-%% erlfmt-ignore
 hocon_config_template() ->
-"""
+    "bridges.kafka.{{ bridge_name }} {"
-bridges.kafka.{{ bridge_name }} {
+    "\n   bootstrap_hosts = \"{{ kafka_hosts_string }}\""
-  bootstrap_hosts = \"{{ kafka_hosts_string }}\"
+    "\n   enable = true"
-  enable = true
+    "\n   authentication = {{{ authentication }}}"
-  authentication = {{{ authentication }}}
+    "\n   ssl = {{{ ssl }}}"
-  ssl = {{{ ssl }}}
+    "\n   local_topic = \"{{ local_topic }}\""
-  local_topic = \"{{ local_topic }}\"
+    "\n   kafka = {"
-  kafka = {
+    "\n     message = {"
-    message = {
+    "\n       key = \"${clientid}\""
-      key = \"${clientid}\"
+    "\n       value = \"${.payload}\""
-      value = \"${.payload}\"
+    "\n       timestamp = \"${timestamp}\""
-      timestamp = \"${timestamp}\"
+    "\n     }"
-    }
+    "\n     buffer = {"
-    buffer = {
+    "\n       memory_overload_protection = false"
-      memory_overload_protection = false
+    "\n     }"
-    }
+    "\n     partition_strategy = {{ partition_strategy }}"
-    partition_strategy = {{ partition_strategy }}
+    "\n     topic = \"{{ kafka_topic }}\""
-    topic = \"{{ kafka_topic }}\"
+    "\n     query_mode = {{ query_mode }}"
-    query_mode = {{ query_mode }}
+    "\n   }"
-  }
+    "\n   metadata_request_timeout = 5s"
-  metadata_request_timeout = 5s
+    "\n   min_metadata_refresh_interval = 3s"
-  min_metadata_refresh_interval = 3s
+    "\n   socket_opts {"
-  socket_opts {
+    "\n     nodelay = true"
-    nodelay = true
+    "\n   }"
-  }
+    "\n   connect_timeout = 5s"
-  connect_timeout = 5s
+    "\n }".
-}
-""".
 
-%% erlfmt-ignore
 hocon_config_template_with_headers() ->
-"""
+    "bridges.kafka.{{ bridge_name }} {"
-bridges.kafka.{{ bridge_name }} {
+    "\n   bootstrap_hosts = \"{{ kafka_hosts_string }}\""
-  bootstrap_hosts = \"{{ kafka_hosts_string }}\"
+    "\n   enable = true"
-  enable = true
+    "\n   authentication = {{{ authentication }}}"
-  authentication = {{{ authentication }}}
+    "\n   ssl = {{{ ssl }}}"
-  ssl = {{{ ssl }}}
+    "\n   local_topic = \"{{ local_topic }}\""
-  local_topic = \"{{ local_topic }}\"
+    "\n   kafka = {"
-  kafka = {
+    "\n     message = {"
-    message = {
+    "\n       key = \"${clientid}\""
-      key = \"${clientid}\"
+    "\n       value = \"${.payload}\""
-      value = \"${.payload}\"
+    "\n       timestamp = \"${timestamp}\""
-      timestamp = \"${timestamp}\"
+    "\n     }"
-    }
+    "\n     buffer = {"
-    buffer = {
+    "\n       memory_overload_protection = false"
-      memory_overload_protection = false
+    "\n     }"
-    }
+    "\n     kafka_headers = \"{{ kafka_headers }}\""
-    kafka_headers = \"{{ kafka_headers }}\"
+    "\n     kafka_header_value_encode_mode: json"
-    kafka_header_value_encode_mode: json
+    "\n     kafka_ext_headers: {{{ kafka_ext_headers }}}"
-    kafka_ext_headers: {{{ kafka_ext_headers }}}
+    "\n     partition_strategy = {{ partition_strategy }}"
-    partition_strategy = {{ partition_strategy }}
+    "\n     topic = \"{{ kafka_topic }}\""
-    topic = \"{{ kafka_topic }}\"
+    "\n     query_mode = {{ query_mode }}"
-    query_mode = {{ query_mode }}
+    "\n   }"
-  }
+    "\n   metadata_request_timeout = 5s"
-  metadata_request_timeout = 5s
+    "\n   min_metadata_refresh_interval = 3s"
-  min_metadata_refresh_interval = 3s
+    "\n   socket_opts {"
-  socket_opts {
+    "\n     nodelay = true"
-    nodelay = true
+    "\n   }"
-  }
+    "\n   connect_timeout = 5s"
-  connect_timeout = 5s
+    "\n }".
-}
-""".
 
-%% erlfmt-ignore
 hocon_config_template_authentication("none") ->
     "none";
 hocon_config_template_authentication(#{"mechanism" := _}) ->
-"""
+    "{"
-{
+    "\n     mechanism = {{ mechanism }}"
-    mechanism = {{ mechanism }}
+    "\n     password = \"{{ password }}\""
-    password = {{ password }}
+    "\n     username = \"{{ username }}\""
-    username = {{ username }}
+    "\n }";
-}
-""";
 hocon_config_template_authentication(#{"kerberos_principal" := _}) ->
-"""
+    "{"
-{
+    "\n     kerberos_principal = \"{{ kerberos_principal }}\""
-    kerberos_principal = \"{{ kerberos_principal }}\"
+    "\n     kerberos_keytab_file = \"{{ kerberos_keytab_file }}\""
-    kerberos_keytab_file = \"{{ kerberos_keytab_file }}\"
+    "\n }".
-}
-""".
 
-%% erlfmt-ignore
 hocon_config_template_ssl(Map) when map_size(Map) =:= 0 ->
-"""
+    "{ enable = false }";
-{
-    enable = false
-}
-""";
 hocon_config_template_ssl(#{"enable" := "false"}) ->
-"""
+    "{ enable = false }";
-{
-    enable = false
-}
-""";
 hocon_config_template_ssl(#{"enable" := "true"}) ->
-"""
+    "{      enable = true"
-{
+    "\n     cacertfile = \"{{{cacertfile}}}\""
-    enable = true
+    "\n     certfile = \"{{{certfile}}}\""
-    cacertfile = \"{{{cacertfile}}}\"
+    "\n     keyfile = \"{{{keyfile}}}\""
-    certfile = \"{{{certfile}}}\"
+    "\n }".
-    keyfile = \"{{{keyfile}}}\"
-}
-""".
 
 kafka_hosts_string(tcp, none) ->
     kafka_hosts_string();
@@ -1197,6 +1158,13 @@ valid_sasl_kerberos_settings() ->
         "kerberos_keytab_file" => shared_secret(rig_keytab)
     }.
 
+valid_sasl_plain_passfile_settings(Passfile) ->
+    Auth = valid_sasl_plain_settings(),
+    ok = file:write_file(Passfile, maps:get("password", Auth)),
+    Auth#{
+        "password" := "file://" ++ Passfile
+    }.
+
 kafka_hosts() ->
     kpro:parse_endpoints(kafka_hosts_string()).
 

apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl

@@ -223,144 +223,136 @@ check_atom_key(Conf) when is_map(Conf) ->
 %% Data section
 %%===========================================================================
 
-%% erlfmt-ignore
 kafka_producer_old_hocon(_WithLocalTopic = true) ->
     kafka_producer_old_hocon("mqtt {topic = \"mqtt/local\"}\n");
 kafka_producer_old_hocon(_WithLocalTopic = false) ->
     kafka_producer_old_hocon("mqtt {}\n");
 kafka_producer_old_hocon(MQTTConfig) when is_list(MQTTConfig) ->
-"""
+    [
-bridges.kafka {
+        "bridges.kafka {"
-  myproducer {
+        "\n  myproducer {"
-    authentication = \"none\"
+        "\n    authentication = \"none\""
-    bootstrap_hosts = \"toxiproxy:9292\"
+        "\n    bootstrap_hosts = \"toxiproxy:9292\""
-    connect_timeout = \"5s\"
+        "\n    connect_timeout = \"5s\""
-    metadata_request_timeout = \"5s\"
+        "\n    metadata_request_timeout = \"5s\""
-    min_metadata_refresh_interval = \"3s\"
+        "\n    min_metadata_refresh_interval = \"3s\""
-    producer {
+        "\n    producer {"
-      kafka {
+        "\n      kafka {"
-        buffer {
+        "\n        buffer {"
-          memory_overload_protection = false
+        "\n          memory_overload_protection = false"
-          mode = \"memory\"
+        "\n          mode = \"memory\""
-          per_partition_limit = \"2GB\"
+        "\n          per_partition_limit = \"2GB\""
-          segment_bytes = \"100MB\"
+        "\n          segment_bytes = \"100MB\""
-        }
+        "\n        }"
-        compression = \"no_compression\"
+        "\n        compression = \"no_compression\""
-        max_batch_bytes = \"896KB\"
+        "\n        max_batch_bytes = \"896KB\""
-        max_inflight = 10
+        "\n        max_inflight = 10"
-        message {
+        "\n        message {"
-          key = \"${.clientid}\"
+        "\n          key = \"${.clientid}\""
-          timestamp = \"${.timestamp}\"
+        "\n          timestamp = \"${.timestamp}\""
-          value = \"${.}\"
+        "\n          value = \"${.}\""
-        }
+        "\n        }"
-        partition_count_refresh_interval = \"60s\"
+        "\n        partition_count_refresh_interval = \"60s\""
-        partition_strategy = \"random\"
+        "\n        partition_strategy = \"random\""
-        required_acks = \"all_isr\"
+        "\n        required_acks = \"all_isr\""
-        topic = \"test-topic-two-partitions\"
+        "\n        topic = \"test-topic-two-partitions\""
-      }
+        "\n      }",
-""" ++ MQTTConfig ++
+        MQTTConfig,
-"""
+        "\n    }"
-    }
+        "\n    socket_opts {"
-    socket_opts {
+        "\n      nodelay = true"
-      nodelay = true
+        "\n      recbuf = \"1024KB\""
-      recbuf = \"1024KB\"
+        "\n      sndbuf = \"1024KB\""
-      sndbuf = \"1024KB\"
+        "\n    }"
-    }
+        "\n    ssl {enable = false, verify = \"verify_peer\"}"
-    ssl {enable = false, verify = \"verify_peer\"}
+        "\n  }"
-  }
+        "\n}"
-}
+    ].
-""".
 
 kafka_producer_new_hocon() ->
-    ""
+    "bridges.kafka {"
-    "\n"
+    "\n  myproducer {"
-    "bridges.kafka {\n"
+    "\n    authentication = \"none\""
-    "  myproducer {\n"
+    "\n    bootstrap_hosts = \"toxiproxy:9292\""
-    "    authentication = \"none\"\n"
+    "\n    connect_timeout = \"5s\""
-    "    bootstrap_hosts = \"toxiproxy:9292\"\n"
+    "\n    metadata_request_timeout = \"5s\""
-    "    connect_timeout = \"5s\"\n"
+    "\n    min_metadata_refresh_interval = \"3s\""
-    "    metadata_request_timeout = \"5s\"\n"
+    "\n    kafka {"
-    "    min_metadata_refresh_interval = \"3s\"\n"
+    "\n      buffer {"
-    "    kafka {\n"
+    "\n        memory_overload_protection = false"
-    "      buffer {\n"
+    "\n        mode = \"memory\""
-    "        memory_overload_protection = false\n"
+    "\n        per_partition_limit = \"2GB\""
-    "        mode = \"memory\"\n"
+    "\n        segment_bytes = \"100MB\""
-    "        per_partition_limit = \"2GB\"\n"
+    "\n      }"
-    "        segment_bytes = \"100MB\"\n"
+    "\n      compression = \"no_compression\""
-    "      }\n"
+    "\n      max_batch_bytes = \"896KB\""
-    "      compression = \"no_compression\"\n"
+    "\n      max_inflight = 10"
-    "      max_batch_bytes = \"896KB\"\n"
+    "\n      message {"
-    "      max_inflight = 10\n"
+    "\n        key = \"${.clientid}\""
-    "      message {\n"
+    "\n        timestamp = \"${.timestamp}\""
-    "        key = \"${.clientid}\"\n"
+    "\n        value = \"${.}\""
-    "        timestamp = \"${.timestamp}\"\n"
+    "\n      }"
-    "        value = \"${.}\"\n"
+    "\n      partition_count_refresh_interval = \"60s\""
-    "      }\n"
+    "\n      partition_strategy = \"random\""
-    "      partition_count_refresh_interval = \"60s\"\n"
+    "\n      required_acks = \"all_isr\""
-    "      partition_strategy = \"random\"\n"
+    "\n      topic = \"test-topic-two-partitions\""
-    "      required_acks = \"all_isr\"\n"
+    "\n    }"
-    "      topic = \"test-topic-two-partitions\"\n"
+    "\n    local_topic = \"mqtt/local\""
-    "    }\n"
+    "\n    socket_opts {"
-    "    local_topic = \"mqtt/local\"\n"
+    "\n      nodelay = true"
-    "    socket_opts {\n"
+    "\n      recbuf = \"1024KB\""
-    "      nodelay = true\n"
+    "\n      sndbuf = \"1024KB\""
-    "      recbuf = \"1024KB\"\n"
+    "\n    }"
-    "      sndbuf = \"1024KB\"\n"
+    "\n    ssl {enable = false, verify = \"verify_peer\"}"
-    "    }\n"
+    "\n    resource_opts {"
-    "    ssl {enable = false, verify = \"verify_peer\"}\n"
+    "\n      health_check_interval = 10s"
-    "    resource_opts {\n"
+    "\n    }"
-    "      health_check_interval = 10s\n"
+    "\n  }"
-    "    }\n"
+    "\n}".
-    "  }\n"
-    "}\n"
-    "".
 
-%% erlfmt-ignore
 kafka_consumer_hocon() ->
-"""
+    "bridges.kafka_consumer.my_consumer {"
-bridges.kafka_consumer.my_consumer {
+    "\n   enable = true"
-  enable = true
+    "\n   bootstrap_hosts = \"kafka-1.emqx.net:9292\""
-  bootstrap_hosts = \"kafka-1.emqx.net:9292\"
+    "\n   connect_timeout = 5s"
-  connect_timeout = 5s
+    "\n   min_metadata_refresh_interval = 3s"
-  min_metadata_refresh_interval = 3s
+    "\n   metadata_request_timeout = 5s"
-  metadata_request_timeout = 5s
+    "\n   authentication = {"
-  authentication = {
+    "\n     mechanism = plain"
-    mechanism = plain
+    "\n     username = emqxuser"
-    username = emqxuser
+    "\n     password = password"
-    password = password
+    "\n   }"
-  }
+    "\n   kafka {"
-  kafka {
+    "\n     max_batch_bytes = 896KB"
-    max_batch_bytes = 896KB
+    "\n     max_rejoin_attempts = 5"
-    max_rejoin_attempts = 5
+    "\n     offset_commit_interval_seconds = 3s"
-    offset_commit_interval_seconds = 3s
+    "\n     offset_reset_policy = latest"
-    offset_reset_policy = latest
+    "\n   }"
-  }
+    "\n   topic_mapping = ["
-  topic_mapping = [
+    "\n     {"
-    {
+    "\n       kafka_topic = \"kafka-topic-1\""
-      kafka_topic = \"kafka-topic-1\"
+    "\n       mqtt_topic = \"mqtt/topic/1\""
-      mqtt_topic = \"mqtt/topic/1\"
+    "\n       qos = 1"
-      qos = 1
+    "\n       payload_template = \"${.}\""
-      payload_template = \"${.}\"
+    "\n     },"
-    },
+    "\n     {"
-    {
+    "\n       kafka_topic = \"kafka-topic-2\""
-      kafka_topic = \"kafka-topic-2\"
+    "\n       mqtt_topic = \"mqtt/topic/2\""
-      mqtt_topic = \"mqtt/topic/2\"
+    "\n       qos = 2"
-      qos = 2
+    "\n       payload_template = \"v = ${.value}\""
-      payload_template = \"v = ${.value}\"
+    "\n     }"
-    }
+    "\n   ]"
-  ]
+    "\n   key_encoding_mode = none"
-  key_encoding_mode = none
+    "\n   value_encoding_mode = none"
-  value_encoding_mode = none
+    "\n   ssl {"
-  ssl {
+    "\n     enable = false"
-    enable = false
+    "\n     verify = verify_none"
-    verify = verify_none
+    "\n     server_name_indication = \"auto\""
-    server_name_indication = \"auto\"
+    "\n   }"
-  }
+    "\n   resource_opts {"
-  resource_opts {
+    "\n     health_check_interval = 10s"
-    health_check_interval = 10s
+    "\n   }"
-  }
+    "\n }".
-}
-""".
 
 %% assert compatibility
 bridge_schema_json_test() ->

apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_kinesis, [
     {description, "EMQX Enterprise Amazon Kinesis Bridge"},
-    {vsn, "0.1.2"},
+    {vsn, "0.1.3"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl

@@ -62,12 +62,10 @@ fields(connector_config) ->
                 }
             )},
         {aws_secret_access_key,
-            mk(
+            emqx_schema_secret:mk(
-                binary(),
                 #{
                     required => true,
-                    desc => ?DESC("aws_secret_access_key"),
+                    desc => ?DESC("aws_secret_access_key")
-                    sensitive => true
                 }
             )},
         {endpoint,

apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl

@@ -97,7 +97,13 @@ init(#{
         partition_key => PartitionKey,
         stream_name => StreamName
     },
-    New =
+    %% TODO: teach `erlcloud` to to accept 0-arity closures as passwords.
+    ok = erlcloud_config:configure(
+        to_str(AwsAccessKey),
+        to_str(emqx_secret:unwrap(AwsSecretAccessKey)),
+        Host,
+        Port,
+        Scheme,
         fun(AccessKeyID, SecretAccessKey, HostAddr, HostPort, ConnectionScheme) ->
             Config0 = erlcloud_kinesis:new(
                 AccessKeyID,
@@ -107,9 +113,7 @@ init(#{
                 ConnectionScheme ++ "://"
             ),
             Config0#aws_config{retry_num = MaxRetries}
-        end,
+        end
-    erlcloud_config:configure(
-        to_str(AwsAccessKey), to_str(AwsSecretAccessKey), Host, Port, Scheme, New
     ),
     % check the connection
     case erlcloud_kinesis:list_streams() of

apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl

@@ -15,7 +15,7 @@
 
 -type config() :: #{
     aws_access_key_id := binary(),
-    aws_secret_access_key := binary(),
+    aws_secret_access_key := emqx_secret:t(binary()),
     endpoint := binary(),
     stream_name := binary(),
     partition_key := binary(),

apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl

@@ -11,10 +11,11 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--define(PRODUCER, emqx_bridge_kinesis_impl_producer).
 -define(BRIDGE_TYPE, kinesis_producer).
 -define(BRIDGE_TYPE_BIN, <<"kinesis_producer">>).
 -define(KINESIS_PORT, 4566).
+-define(KINESIS_ACCESS_KEY, "aws_access_key_id").
+-define(KINESIS_SECRET_KEY, "aws_secret_access_key").
 -define(TOPIC, <<"t/topic">>).
 
 %%------------------------------------------------------------------------------
@@ -38,6 +39,8 @@ init_per_suite(Config) ->
     ProxyHost = os:getenv("PROXY_HOST", "toxiproxy.emqx.net"),
     ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
     ProxyName = "kinesis",
+    SecretFile = filename:join(?config(priv_dir, Config), "secret"),
+    ok = file:write_file(SecretFile, <<?KINESIS_SECRET_KEY>>),
     ok = emqx_common_test_helpers:start_apps([emqx_conf]),
     ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]),
     {ok, _} = application:ensure_all_started(emqx_connector),
@@ -46,6 +49,7 @@ init_per_suite(Config) ->
         {proxy_host, ProxyHost},
         {proxy_port, ProxyPort},
         {kinesis_port, ?KINESIS_PORT},
+        {kinesis_secretfile, SecretFile},
         {proxy_name, ProxyName}
         | Config
     ].
@@ -130,6 +134,7 @@ kinesis_config(Config) ->
     Scheme = proplists:get_value(connection_scheme, Config, "http"),
     ProxyHost = proplists:get_value(proxy_host, Config),
     KinesisPort = proplists:get_value(kinesis_port, Config),
+    SecretFile = proplists:get_value(kinesis_secretfile, Config),
     BatchSize = proplists:get_value(batch_size, Config, 100),
     BatchTime = proplists:get_value(batch_time, Config, <<"500ms">>),
     PayloadTemplate = proplists:get_value(payload_template, Config, "${payload}"),
@@ -140,29 +145,32 @@ kinesis_config(Config) ->
     Name = <<(atom_to_binary(?MODULE))/binary, (GUID)/binary>>,
     ConfigString =
         io_lib:format(
-            "bridges.kinesis_producer.~s {\n"
+            "bridges.kinesis_producer.~s {"
-            "  enable = true\n"
+            "\n   enable = true"
-            "  aws_access_key_id = \"aws_access_key_id\"\n"
+            "\n   aws_access_key_id = ~p"
-            "  aws_secret_access_key = \"aws_secret_access_key\"\n"
+            "\n   aws_secret_access_key = ~p"
-            "  endpoint = \"~s://~s:~b\"\n"
+            "\n   endpoint = \"~s://~s:~b\""
-            "  stream_name = \"~s\"\n"
+            "\n   stream_name = \"~s\""
-            "  partition_key = \"~s\"\n"
+            "\n   partition_key = \"~s\""
-            "  payload_template = \"~s\"\n"
+            "\n   payload_template = \"~s\""
-            "  max_retries = ~b\n"
+            "\n   max_retries = ~b"
-            "  pool_size = 1\n"
+            "\n   pool_size = 1"
-            "  resource_opts = {\n"
+            "\n   resource_opts = {"
-            "    health_check_interval = \"3s\"\n"
+            "\n     health_check_interval = \"3s\""
-            "    request_ttl = 30s\n"
+            "\n     request_ttl = 30s"
-            "    resume_interval = 1s\n"
+            "\n     resume_interval = 1s"
-            "    metrics_flush_interval = \"700ms\"\n"
+            "\n     metrics_flush_interval = \"700ms\""
-            "    worker_pool_size = 1\n"
+            "\n     worker_pool_size = 1"
-            "    query_mode = ~s\n"
+            "\n     query_mode = ~s"
-            "    batch_size = ~b\n"
+            "\n     batch_size = ~b"
-            "    batch_time = \"~s\"\n"
+            "\n     batch_time = \"~s\""
-            "  }\n"
+            "\n   }"
-            "}\n",
+            "\n }",
             [
                 Name,
+                ?KINESIS_ACCESS_KEY,
+                %% NOTE: using file-based secrets with HOCON configs.
+                "file://" ++ SecretFile,
                 Scheme,
                 ProxyHost,
                 KinesisPort,
@@ -203,9 +211,6 @@ delete_bridge(Config) ->
     ct:pal("deleting bridge ~p", [{Type, Name}]),
     emqx_bridge:remove(Type, Name).
 
-create_bridge_http(Config) ->
-    create_bridge_http(Config, _KinesisConfigOverrides = #{}).
-
 create_bridge_http(Config, KinesisConfigOverrides) ->
     TypeBin = ?BRIDGE_TYPE_BIN,
     Name = ?config(kinesis_name, Config),
@@ -489,7 +494,11 @@ to_bin(Str) when is_list(Str) ->
 %%------------------------------------------------------------------------------
 
 t_create_via_http(Config) ->
-    ?assertMatch({ok, _}, create_bridge_http(Config)),
+    Overrides = #{
+        %% NOTE: using literal secret with HTTP API requests.
+        <<"aws_secret_access_key">> => <<?KINESIS_SECRET_KEY>>
+    },
+    ?assertMatch({ok, _}, create_bridge_http(Config, Overrides)),
     ok.
 
 t_start_failed_then_fix(Config) ->

apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_mongodb, [
     {description, "EMQX Enterprise MongoDB Bridge"},
-    {vsn, "0.2.1"},
+    {vsn, "0.2.2"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl

@@ -6,9 +6,6 @@
 
 -behaviour(emqx_resource).
 
--include_lib("emqx_connector/include/emqx_connector_tables.hrl").
--include_lib("emqx_resource/include/emqx_resource.hrl").
--include_lib("typerefl/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 

apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl

@@ -11,6 +11,8 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
+-import(emqx_utils_conv, [bin/1]).
+
 %%------------------------------------------------------------------------------
 %% CT boilerplate
 %%------------------------------------------------------------------------------
@@ -96,14 +98,27 @@ init_per_group(Type = single, Config) ->
         true ->
             ok = start_apps(),
             emqx_mgmt_api_test_util:init_suite(),
-            {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, Config),
+            %% NOTE: `mongo-single` has auth enabled, see `credentials.env`.
+            AuthSource = bin(os:getenv("MONGO_AUTHSOURCE", "admin")),
+            Username = bin(os:getenv("MONGO_USERNAME", "")),
+            Password = bin(os:getenv("MONGO_PASSWORD", "")),
+            Passfile = filename:join(?config(priv_dir, Config), "passfile"),
+            ok = file:write_file(Passfile, Password),
+            NConfig = [
+                {mongo_authsource, AuthSource},
+                {mongo_username, Username},
+                {mongo_password, Password},
+                {mongo_passfile, Passfile}
+                | Config
+            ],
+            {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, NConfig),
             [
                 {mongo_host, MongoHost},
                 {mongo_port, MongoPort},
                 {mongo_config, MongoConfig},
                 {mongo_type, Type},
                 {mongo_name, Name}
-                | Config
+                | NConfig
             ];
         false ->
             {skip, no_mongo}
@@ -121,13 +136,13 @@ end_per_suite(_Config) ->
     ok.
 
 init_per_testcase(_Testcase, Config) ->
-    catch clear_db(Config),
+    clear_db(Config),
     delete_bridge(Config),
     snabbkaffe:start_trace(),
     Config.
 
 end_per_testcase(_Testcase, Config) ->
-    catch clear_db(Config),
+    clear_db(Config),
     delete_bridge(Config),
     snabbkaffe:stop(),
     ok.
@@ -175,19 +190,19 @@ mongo_config(MongoHost, MongoPort0, rs = Type, Config) ->
     Name = atom_to_binary(?MODULE),
     ConfigString =
         io_lib:format(
-            "bridges.mongodb_rs.~s {\n"
+            "bridges.mongodb_rs.~s {"
-            "  enable = true\n"
+            "\n   enable = true"
-            "  collection = mycol\n"
+            "\n   collection = mycol"
-            "  replica_set_name = rs0\n"
+            "\n   replica_set_name = rs0"
-            "  servers = [~p]\n"
+            "\n   servers = [~p]"
-            "  w_mode = safe\n"
+            "\n   w_mode = safe"
-            "  use_legacy_protocol = auto\n"
+            "\n   use_legacy_protocol = auto"
-            "  database = mqtt\n"
+            "\n   database = mqtt"
-            "  resource_opts = {\n"
+            "\n   resource_opts = {"
-            "    query_mode = ~s\n"
+            "\n     query_mode = ~s"
-            "    worker_pool_size = 1\n"
+            "\n     worker_pool_size = 1"
-            "  }\n"
+            "\n   }"
-            "}",
+            "\n }",
             [
                 Name,
                 Servers,
@@ -202,18 +217,18 @@ mongo_config(MongoHost, MongoPort0, sharded = Type, Config) ->
     Name = atom_to_binary(?MODULE),
     ConfigString =
         io_lib:format(
-            "bridges.mongodb_sharded.~s {\n"
+            "bridges.mongodb_sharded.~s {"
-            "  enable = true\n"
+            "\n   enable = true"
-            "  collection = mycol\n"
+            "\n   collection = mycol"
-            "  servers = [~p]\n"
+            "\n   servers = [~p]"
-            "  w_mode = safe\n"
+            "\n   w_mode = safe"
-            "  use_legacy_protocol = auto\n"
+            "\n   use_legacy_protocol = auto"
-            "  database = mqtt\n"
+            "\n   database = mqtt"
-            "  resource_opts = {\n"
+            "\n   resource_opts = {"
-            "    query_mode = ~s\n"
+            "\n     query_mode = ~s"
-            "    worker_pool_size = 1\n"
+            "\n     worker_pool_size = 1"
-            "  }\n"
+            "\n   }"
-            "}",
+            "\n }",
             [
                 Name,
                 Servers,
@@ -228,21 +243,27 @@ mongo_config(MongoHost, MongoPort0, single = Type, Config) ->
     Name = atom_to_binary(?MODULE),
     ConfigString =
         io_lib:format(
-            "bridges.mongodb_single.~s {\n"
+            "bridges.mongodb_single.~s {"
-            "  enable = true\n"
+            "\n   enable = true"
-            "  collection = mycol\n"
+            "\n   collection = mycol"
-            "  server = ~p\n"
+            "\n   server = ~p"
-            "  w_mode = safe\n"
+            "\n   w_mode = safe"
-            "  use_legacy_protocol = auto\n"
+            "\n   use_legacy_protocol = auto"
-            "  database = mqtt\n"
+            "\n   database = mqtt"
-            "  resource_opts = {\n"
+            "\n   auth_source = ~s"
-            "    query_mode = ~s\n"
+            "\n   username = ~s"
-            "    worker_pool_size = 1\n"
+            "\n   password = \"file://~s\""
-            "  }\n"
+            "\n   resource_opts = {"
-            "}",
+            "\n     query_mode = ~s"
+            "\n     worker_pool_size = 1"
+            "\n   }"
+            "\n }",
             [
                 Name,
                 Server,
+                ?config(mongo_authsource, Config),
+                ?config(mongo_username, Config),
+                ?config(mongo_passfile, Config),
                 QueryMode
             ]
         ),
@@ -284,8 +305,24 @@ clear_db(Config) ->
     Host = ?config(mongo_host, Config),
     Port = ?config(mongo_port, Config),
     Server = Host ++ ":" ++ integer_to_list(Port),
-    #{<<"database">> := Db, <<"collection">> := Collection} = ?config(mongo_config, Config),
+    #{
-    {ok, Client} = mongo_api:connect(Type, [Server], [], [{database, Db}, {w_mode, unsafe}]),
+        <<"database">> := Db,
+        <<"collection">> := Collection
+    } = ?config(mongo_config, Config),
+    WorkerOpts = [
+        {database, Db},
+        {w_mode, unsafe}
+        | lists:flatmap(
+            fun
+                ({mongo_authsource, AS}) -> [{auth_source, AS}];
+                ({mongo_username, User}) -> [{login, User}];
+                ({mongo_password, Pass}) -> [{password, Pass}];
+                (_) -> []
+            end,
+            Config
+        )
+    ],
+    {ok, Client} = mongo_api:connect(Type, [Server], [], WorkerOpts),
     {true, _} = mongo_api:delete(Client, Collection, _Selector = #{}),
     mongo_api:disconnect(Client).
 
@@ -386,13 +423,21 @@ t_setup_via_config_and_publish(Config) ->
     ok.
 
 t_setup_via_http_api_and_publish(Config) ->
-    Type = mongo_type_bin(?config(mongo_type, Config)),
+    Type = ?config(mongo_type, Config),
     Name = ?config(mongo_name, Config),
     MongoConfig0 = ?config(mongo_config, Config),
-    MongoConfig = MongoConfig0#{
+    MongoConfig1 = MongoConfig0#{
         <<"name">> => Name,
-        <<"type">> => Type
+        <<"type">> => mongo_type_bin(Type)
     },
+    MongoConfig =
+        case Type of
+            single ->
+                %% NOTE: using literal password with HTTP API requests.
+                MongoConfig1#{<<"password">> => ?config(mongo_password, Config)};
+            _ ->
+                MongoConfig1
+        end,
     ?assertMatch(
         {ok, _},
         create_bridge_http(MongoConfig)

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge_mqtt, [
     {description, "EMQX MQTT Broker Bridge"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl

@@ -96,7 +96,7 @@ choose_ingress_pool_size(
     #{remote := #{topic := RemoteTopic}, pool_size := PoolSize}
 ) ->
     case emqx_topic:parse(RemoteTopic) of
-        {_Filter, #{share := _Name}} ->
+        {#share{} = _Filter, _SubOpts} ->
             % NOTE: this is shared subscription, many workers may subscribe
             PoolSize;
         {_Filter, #{}} when PoolSize > 1 ->
@@ -326,7 +326,7 @@ mk_client_opts(
         ],
         Config
     ),
-    Options#{
+    mk_client_opt_password(Options#{
         hosts => [HostPort],
         clientid => clientid(ResourceId, ClientScope, Config),
         connect_timeout => 30,
@@ -334,7 +334,13 @@ mk_client_opts(
         force_ping => true,
         ssl => EnableSsl,
         ssl_opts => maps:to_list(maps:remove(enable, Ssl))
-    }.
+    }).
+
+mk_client_opt_password(Options = #{password := Secret}) ->
+    %% TODO: Teach `emqtt` to accept 0-arity closures as passwords.
+    Options#{password := emqx_secret:unwrap(Secret)};
+mk_client_opt_password(Options) ->
+    Options.
 
 ms_to_s(Ms) ->
     erlang:ceil(Ms / 1000).

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl

@@ -99,13 +99,9 @@ fields("server_configs") ->
                 }
             )},
         {password,
-            mk(
+            emqx_schema_secret:mk(
-                binary(),
                 #{
-                    format => <<"password">>,
+                    desc => ?DESC("password")
-                    sensitive => true,
-                    desc => ?DESC("password"),
-                    converter => fun emqx_schema:password_converter/2
                 }
             )},
         {clean_start,

apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl

@@ -21,13 +21,15 @@
 -import(emqx_dashboard_api_test_helpers, [request/4, uri/1]).
 
 -include("emqx/include/emqx.hrl").
+-include("emqx/include/emqx_hooks.hrl").
+-include("emqx/include/asserts.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 %% output functions
 -export([inspect/3]).
 
--define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
 -define(TYPE_MQTT, <<"mqtt">>).
 -define(BRIDGE_NAME_INGRESS, <<"ingress_mqtt_bridge">>).
 -define(BRIDGE_NAME_EGRESS, <<"egress_mqtt_bridge">>).
@@ -38,14 +40,18 @@
 -define(EGRESS_REMOTE_TOPIC, "egress_remote_topic").
 -define(EGRESS_LOCAL_TOPIC, "egress_local_topic").
 
--define(SERVER_CONF(Username), #{
+-define(SERVER_CONF, #{
+    <<"type">> => ?TYPE_MQTT,
     <<"server">> => <<"127.0.0.1:1883">>,
-    <<"username">> => Username,
-    <<"password">> => <<"">>,
     <<"proto_ver">> => <<"v4">>,
     <<"ssl">> => #{<<"enable">> => false}
 }).
 
+-define(SERVER_CONF(Username, Password), (?SERVER_CONF)#{
+    <<"username">> => Username,
+    <<"password">> => Password
+}).
+
 -define(INGRESS_CONF, #{
     <<"remote">> => #{
         <<"topic">> => <<?INGRESS_REMOTE_TOPIC, "/#">>,
@@ -129,43 +135,32 @@ suite() ->
     [{timetrap, {seconds, 30}}].
 
 init_per_suite(Config) ->
-    _ = application:load(emqx_conf),
+    Apps = emqx_cth_suite:start(
-    ok = emqx_common_test_helpers:start_apps(
         [
+            emqx_conf,
+            emqx_bridge,
             emqx_rule_engine,
-            emqx_bridge,
             emqx_bridge_mqtt,
-            emqx_dashboard
+            {emqx_dashboard,
+                "dashboard {"
+                "\n listeners.http { bind = 18083 }"
+                "\n default_username = connector_admin"
+                "\n default_password = public"
+                "\n }"}
         ],
-        fun set_special_configs/1
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
-    ok = emqx_common_test_helpers:load_config(
+    [{suite_apps, Apps} | Config].
-        emqx_rule_engine_schema,
-        <<"rule_engine {rules {}}">>
-    ),
-    ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
-    Config.
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    emqx_common_test_helpers:stop_apps([
+    emqx_cth_suite:stop(?config(suite_apps, Config)).
-        emqx_dashboard,
-        emqx_bridge_mqtt,
-        emqx_bridge,
-        emqx_rule_engine
-    ]),
-    ok.
-
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config(<<"connector_admin">>);
-set_special_configs(_) ->
-    ok.
 
 init_per_testcase(_, Config) ->
-    {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
     ok = snabbkaffe:start_trace(),
     Config.
 
 end_per_testcase(_, _Config) ->
+    ok = unhook_authenticate(),
     clear_resources(),
     snabbkaffe:stop(),
     ok.
@@ -187,14 +182,86 @@ clear_resources() ->
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
+
+t_conf_bridge_authn_anonymous(_) ->
+    ok = hook_authenticate(),
+    {ok, 201, _Bridge} = request(
+        post,
+        uri(["bridges"]),
+        ?SERVER_CONF#{
+            <<"name">> => <<"t_conf_bridge_anonymous">>,
+            <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1}
+        }
+    ),
+    ?assertReceive(
+        {authenticate, #{username := undefined, password := undefined}}
+    ).
+
+t_conf_bridge_authn_password(_) ->
+    Username1 = <<"user1">>,
+    Password1 = <<"from-here">>,
+    ok = hook_authenticate(),
+    {ok, 201, _Bridge1} = request(
+        post,
+        uri(["bridges"]),
+        ?SERVER_CONF(Username1, Password1)#{
+            <<"name">> => <<"t_conf_bridge_authn_password">>,
+            <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1}
+        }
+    ),
+    ?assertReceive(
+        {authenticate, #{username := Username1, password := Password1}}
+    ).
+
+t_conf_bridge_authn_passfile(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Username2 = <<"user2">>,
+    PasswordFilename = filename:join(DataDir, "password"),
+    Password2 = <<"from-there">>,
+    ok = hook_authenticate(),
+    {ok, 201, _Bridge2} = request(
+        post,
+        uri(["bridges"]),
+        ?SERVER_CONF(Username2, iolist_to_binary(["file://", PasswordFilename]))#{
+            <<"name">> => <<"t_conf_bridge_authn_passfile">>,
+            <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1}
+        }
+    ),
+    ?assertReceive(
+        {authenticate, #{username := Username2, password := Password2}}
+    ),
+    ?assertMatch(
+        {ok, 201, #{
+            <<"status">> := <<"disconnected">>,
+            <<"status_reason">> := <<"#{msg => failed_to_read_secret_file", _/bytes>>
+        }},
+        request_json(
+            post,
+            uri(["bridges"]),
+            ?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{
+                <<"name">> => <<"t_conf_bridge_authn_no_passfile">>
+            }
+        )
+    ).
+
+hook_authenticate() ->
+    emqx_hooks:add('client.authenticate', {?MODULE, authenticate, [self()]}, ?HP_HIGHEST).
+
+unhook_authenticate() ->
+    emqx_hooks:del('client.authenticate', {?MODULE, authenticate}).
+
+authenticate(Credential, _, TestRunnerPid) ->
+    _ = TestRunnerPid ! {authenticate, Credential},
+    ignore.
+
+%%------------------------------------------------------------------------------
+
 t_mqtt_conn_bridge_ingress(_) ->
-    User1 = <<"user1">>,
     %% create an MQTT bridge, using POST
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ServerConf = ?SERVER_CONF(User1)#{
+        ServerConf = ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_INGRESS,
             <<"ingress">> => ?INGRESS_CONF
         }
@@ -249,7 +316,6 @@ t_mqtt_conn_bridge_ingress(_) ->
     ok.
 
 t_mqtt_conn_bridge_ingress_full_context(_Config) ->
-    User1 = <<"user1">>,
     IngressConf =
         emqx_utils_maps:deep_merge(
             ?INGRESS_CONF,
@@ -258,8 +324,7 @@ t_mqtt_conn_bridge_ingress_full_context(_Config) ->
     {ok, 201, _Bridge} = request(
         post,
         uri(["bridges"]),
-        ?SERVER_CONF(User1)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_INGRESS,
             <<"ingress">> => IngressConf
         }
@@ -297,8 +362,7 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) ->
     Ns = lists:seq(1, 10),
     BridgeName = atom_to_binary(?FUNCTION_NAME),
     BridgeID = create_bridge(
-        ?SERVER_CONF(<<>>)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => BridgeName,
             <<"ingress">> => #{
                 <<"pool_size">> => PoolSize,
@@ -337,8 +401,7 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) ->
 t_mqtt_egress_bridge_ignores_clean_start(_) ->
     BridgeName = atom_to_binary(?FUNCTION_NAME),
     BridgeID = create_bridge(
-        ?SERVER_CONF(<<"user1">>)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => BridgeName,
             <<"egress">> => ?EGRESS_CONF,
             <<"clean_start">> => false
@@ -366,8 +429,7 @@ t_mqtt_egress_bridge_ignores_clean_start(_) ->
 t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) ->
     BridgeName = atom_to_binary(?FUNCTION_NAME),
     BridgeID = create_bridge(
-        ?SERVER_CONF(<<"user1">>)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => BridgeName,
             <<"ingress">> => emqx_utils_maps:deep_merge(
                 ?INGRESS_CONF,
@@ -392,9 +454,8 @@ t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) ->
     ok.
 
 t_mqtt_conn_bridge_ingress_no_payload_template(_) ->
-    User1 = <<"user1">>,
     BridgeIDIngress = create_bridge(
-        ?SERVER_CONF(User1)#{
+        ?SERVER_CONF#{
             <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_INGRESS,
             <<"ingress">> => ?INGRESS_CONF_NO_PAYLOAD_TEMPLATE
@@ -428,10 +489,8 @@ t_mqtt_conn_bridge_ingress_no_payload_template(_) ->
 
 t_mqtt_conn_bridge_egress(_) ->
     %% then we add a mqtt connector, using POST
-    User1 = <<"user1">>,
     BridgeIDEgress = create_bridge(
-        ?SERVER_CONF(User1)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_EGRESS,
             <<"egress">> => ?EGRESS_CONF
         }
@@ -473,11 +532,8 @@ t_mqtt_conn_bridge_egress(_) ->
 
 t_mqtt_conn_bridge_egress_no_payload_template(_) ->
     %% then we add a mqtt connector, using POST
-    User1 = <<"user1">>,
-
     BridgeIDEgress = create_bridge(
-        ?SERVER_CONF(User1)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_EGRESS,
             <<"egress">> => ?EGRESS_CONF_NO_PAYLOAD_TEMPLATE
         }
@@ -520,11 +576,9 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) ->
     ok.
 
 t_egress_custom_clientid_prefix(_Config) ->
-    User1 = <<"user1">>,
     BridgeIDEgress = create_bridge(
-        ?SERVER_CONF(User1)#{
+        ?SERVER_CONF#{
             <<"clientid_prefix">> => <<"my-custom-prefix">>,
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_EGRESS,
             <<"egress">> => ?EGRESS_CONF
         }
@@ -545,17 +599,14 @@ t_egress_custom_clientid_prefix(_Config) ->
     ok.
 
 t_mqtt_conn_bridge_ingress_and_egress(_) ->
-    User1 = <<"user1">>,
     BridgeIDIngress = create_bridge(
-        ?SERVER_CONF(User1)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_INGRESS,
             <<"ingress">> => ?INGRESS_CONF
         }
     ),
     BridgeIDEgress = create_bridge(
-        ?SERVER_CONF(User1)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_EGRESS,
             <<"egress">> => ?EGRESS_CONF
         }
@@ -627,8 +678,7 @@ t_mqtt_conn_bridge_ingress_and_egress(_) ->
 
 t_ingress_mqtt_bridge_with_rules(_) ->
     BridgeIDIngress = create_bridge(
-        ?SERVER_CONF(<<"user1">>)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_INGRESS,
             <<"ingress">> => ?INGRESS_CONF
         }
@@ -712,8 +762,7 @@ t_ingress_mqtt_bridge_with_rules(_) ->
 
 t_egress_mqtt_bridge_with_rules(_) ->
     BridgeIDEgress = create_bridge(
-        ?SERVER_CONF(<<"user1">>)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_EGRESS,
             <<"egress">> => ?EGRESS_CONF
         }
@@ -789,10 +838,8 @@ t_egress_mqtt_bridge_with_rules(_) ->
 
 t_mqtt_conn_bridge_egress_reconnect(_) ->
     %% then we add a mqtt connector, using POST
-    User1 = <<"user1">>,
     BridgeIDEgress = create_bridge(
-        ?SERVER_CONF(User1)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_EGRESS,
             <<"egress">> => ?EGRESS_CONF,
             <<"resource_opts">> => #{
@@ -897,10 +944,8 @@ t_mqtt_conn_bridge_egress_reconnect(_) ->
     ok.
 
 t_mqtt_conn_bridge_egress_async_reconnect(_) ->
-    User1 = <<"user1">>,
     BridgeIDEgress = create_bridge(
-        ?SERVER_CONF(User1)#{
+        ?SERVER_CONF#{
-            <<"type">> => ?TYPE_MQTT,
             <<"name">> => ?BRIDGE_NAME_EGRESS,
             <<"egress">> => ?EGRESS_CONF,
             <<"resource_opts">> => #{
@@ -1018,5 +1063,9 @@ request_bridge_metrics(BridgeID) ->
     {ok, 200, BridgeMetrics} = request(get, uri(["bridges", BridgeID, "metrics"]), []),
     emqx_utils_json:decode(BridgeMetrics).
 
+request_json(Method, Url, Body) ->
+    {ok, Code, Response} = request(Method, Url, Body),
+    {ok, Code, emqx_utils_json:decode(Response)}.
+
 request(Method, Url, Body) ->
     request(<<"connector_admin">>, Method, Url, Body).

apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE_data/password

@@ -0,0 +1 @@
+from-there

apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl

@@ -21,7 +21,6 @@
     "DEFAULT CHARSET=utf8MB4;"
 ).
 -define(SQL_DROP_TABLE, "DROP TABLE mqtt_test").
--define(SQL_DELETE, "DELETE from mqtt_test").
 -define(SQL_SELECT, "SELECT payload FROM mqtt_test").
 
 % DB defaults
@@ -112,8 +111,8 @@ end_per_suite(_Config) ->
     ok.
 
 init_per_testcase(_Testcase, Config) ->
+    connect_and_drop_table(Config),
     connect_and_create_table(Config),
-    connect_and_clear_table(Config),
     delete_bridge(Config),
     snabbkaffe:start_trace(),
     Config.
@@ -122,9 +121,7 @@ end_per_testcase(_Testcase, Config) ->
     ProxyHost = ?config(proxy_host, Config),
     ProxyPort = ?config(proxy_port, Config),
     emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
-    connect_and_clear_table(Config),
     ok = snabbkaffe:stop(),
-    delete_bridge(Config),
     emqx_common_test_helpers:call_janitor(),
     ok.
 
@@ -323,9 +320,6 @@ connect_and_create_table(Config) ->
 connect_and_drop_table(Config) ->
     query_direct_mysql(Config, ?SQL_DROP_TABLE).
 
-connect_and_clear_table(Config) ->
-    query_direct_mysql(Config, ?SQL_DELETE).
-
 connect_and_get_payload(Config) ->
     query_direct_mysql(Config, ?SQL_SELECT).
 
@@ -777,8 +771,6 @@ t_table_removed(Config) ->
     Name = ?config(mysql_name, Config),
     BridgeType = ?config(mysql_bridge_type, Config),
     ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
-    ?check_trace(
-        begin
     connect_and_create_table(Config),
     ?assertMatch({ok, _}, create_bridge(Config)),
     ?retry(
@@ -792,14 +784,9 @@ t_table_removed(Config) ->
     Timeout = 1000,
     ?assertMatch(
         {error,
-                    {unrecoverable_error,
+            {unrecoverable_error, {1146, <<"42S02">>, <<"Table 'mqtt.mqtt_test' doesn't exist">>}}},
-                        {1146, <<"42S02">>, <<"Table 'mqtt.mqtt_test' doesn't exist">>}}},
         sync_query_resource(Config, {send_message, SentData, [], Timeout})
     ),
-            ok
-        end,
-        []
-    ),
     ok.
 
 t_nested_payload_template(Config) ->
@@ -807,9 +794,6 @@ t_nested_payload_template(Config) ->
     BridgeType = ?config(mysql_bridge_type, Config),
     ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
     Value = integer_to_binary(erlang:unique_integer()),
-    ?check_trace(
-        begin
-            connect_and_create_table(Config),
     {ok, _} = create_bridge(
         Config,
         #{
@@ -837,8 +821,4 @@ t_nested_payload_template(Config) ->
         {ok, [<<"payload">>], [[Value]]},
         connect_and_get_payload(Config)
     ),
-            ok
-        end,
-        []
-    ),
     ok.

apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl

@@ -16,7 +16,6 @@
 -define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_oracle, emqx_bridge_oracle]).
 -define(SID, "XE").
 -define(RULE_TOPIC, "mqtt/rule").
-% -define(RULE_TOPIC_BIN, <<?RULE_TOPIC>>).
 
 %%------------------------------------------------------------------------------
 %% CT boilerplate
@@ -33,9 +32,6 @@ groups() ->
         {plain, AllTCs}
     ].
 
-only_once_tests() ->
-    [t_create_via_http].
-
 init_per_suite(Config) ->
     Config.
 

apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl

@@ -183,31 +183,33 @@ pgsql_config(BridgeType, Config) ->
         end,
     QueryMode = ?config(query_mode, Config),
     TlsEnabled = ?config(enable_tls, Config),
+    %% NOTE: supplying password through a file here, to verify that it works.
+    Password = create_passfile(BridgeType, Config),
     ConfigString =
         io_lib:format(
-            "bridges.~s.~s {\n"
+            "bridges.~s.~s {"
-            "  enable = true\n"
+            "\n   enable = true"
-            "  server = ~p\n"
+            "\n   server = ~p"
-            "  database = ~p\n"
+            "\n   database = ~p"
-            "  username = ~p\n"
+            "\n   username = ~p"
-            "  password = ~p\n"
+            "\n   password = ~p"
-            "  sql = ~p\n"
+            "\n   sql = ~p"
-            "  resource_opts = {\n"
+            "\n   resource_opts = {"
-            "    request_ttl = 500ms\n"
+            "\n     request_ttl = 500ms"
-            "    batch_size = ~b\n"
+            "\n     batch_size = ~b"
-            "    query_mode = ~s\n"
+            "\n     query_mode = ~s"
-            "  }\n"
+            "\n   }"
-            "  ssl = {\n"
+            "\n   ssl = {"
-            "    enable = ~w\n"
+            "\n     enable = ~w"
-            "  }\n"
+            "\n   }"
-            "}",
+            "\n }",
             [
                 BridgeType,
                 Name,
                 Server,
                 ?PGSQL_DATABASE,
                 ?PGSQL_USERNAME,
-                ?PGSQL_PASSWORD,
+                Password,
                 ?SQL_BRIDGE,
                 BatchSize,
                 QueryMode,
@@ -216,6 +218,12 @@ pgsql_config(BridgeType, Config) ->
         ),
     {Name, parse_and_check(ConfigString, BridgeType, Name)}.
 
+create_passfile(BridgeType, Config) ->
+    Filename = binary_to_list(BridgeType) ++ ".passfile",
+    Filepath = filename:join(?config(priv_dir, Config), Filename),
+    ok = file:write_file(Filepath, ?PGSQL_PASSWORD),
+    "file://" ++ Filepath.
+
 parse_and_check(ConfigString, BridgeType, Name) ->
     {ok, RawConf} = hocon:binary(ConfigString, #{format => map}),
     hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}),
@@ -379,7 +387,9 @@ t_setup_via_http_api_and_publish(Config) ->
     QueryMode = ?config(query_mode, Config),
     PgsqlConfig = PgsqlConfig0#{
         <<"name">> => Name,
-        <<"type">> => BridgeType
+        <<"type">> => BridgeType,
+        %% NOTE: using literal passwords with HTTP API requests.
+        <<"password">> => <<?PGSQL_PASSWORD>>
     },
     ?assertMatch(
         {ok, _},

apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_pulsar, [
     {description, "EMQX Pulsar Bridge"},
-    {vsn, "0.1.7"},
+    {vsn, "0.1.8"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl

@@ -170,21 +170,17 @@ fields(auth_basic) ->
     [
         {username, mk(binary(), #{required => true, desc => ?DESC("auth_basic_username")})},
         {password,
-            mk(binary(), #{
+            emqx_schema_secret:mk(#{
                 required => true,
-                desc => ?DESC("auth_basic_password"),
+                desc => ?DESC("auth_basic_password")
-                sensitive => true,
-                converter => fun emqx_schema:password_converter/2
             })}
     ];
 fields(auth_token) ->
     [
         {jwt,
-            mk(binary(), #{
+            emqx_schema_secret:mk(#{
                 required => true,
-                desc => ?DESC("auth_token_jwt"),
+                desc => ?DESC("auth_token_jwt")
-                sensitive => true,
-                converter => fun emqx_schema:password_converter/2
             })}
     ];
 fields("get_" ++ Type) ->

apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl

@@ -78,7 +78,6 @@ query_mode(_Config) ->
 -spec on_start(resource_id(), config()) -> {ok, state()}.
 on_start(InstanceId, Config) ->
     #{
-        authentication := _Auth,
         bridge_name := BridgeName,
         servers := Servers0,
         ssl := SSL
@@ -263,12 +262,14 @@ conn_opts(#{authentication := none}) ->
     #{};
 conn_opts(#{authentication := #{username := Username, password := Password}}) ->
     #{
-        auth_data => iolist_to_binary([Username, <<":">>, Password]),
+        %% TODO: teach `pulsar` to accept 0-arity closures as passwords.
+        auth_data => iolist_to_binary([Username, <<":">>, emqx_secret:unwrap(Password)]),
         auth_method_name => <<"basic">>
     };
 conn_opts(#{authentication := #{jwt := JWT}}) ->
     #{
-        auth_data => JWT,
+        %% TODO: teach `pulsar` to accept 0-arity closures as passwords.
+        auth_data => emqx_secret:unwrap(JWT),
         auth_method_name => <<"token">>
     }.
 

apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl

@@ -74,7 +74,7 @@ fields(config) ->
                     desc => ?DESC("username")
                 }
             )},
-        {password, fun emqx_connector_schema_lib:password_required/1},
+        {password, emqx_connector_schema_lib:password_field(#{required => true})},
         {pool_size,
             hoconsc:mk(
                 typerefl:pos_integer(),
@@ -196,7 +196,6 @@ on_start(
     #{
         pool_size := PoolSize,
         payload_template := PayloadTemplate,
-        password := Password,
         delivery_mode := InitialDeliveryMode
     } = InitialConfig
 ) ->
@@ -206,7 +205,6 @@ on_start(
             persistent -> 2
         end,
     Config = InitialConfig#{
-        password => emqx_secret:wrap(Password),
         delivery_mode => DeliveryMode
     },
     ?SLOG(info, #{
@@ -242,13 +240,11 @@ on_start(
         ok ->
             {ok, State};
         {error, Reason} ->
-            LogMessage =
+            ?SLOG(info, #{
-                #{
                 msg => "rabbitmq_connector_start_failed",
                 error_reason => Reason,
                 config => emqx_utils:redact(Config)
-                },
+            }),
-            ?SLOG(info, LogMessage),
             {error, Reason}
     end.
 
@@ -321,6 +317,7 @@ create_rabbitmq_connection_and_channel(Config) ->
         heartbeat := Heartbeat,
         wait_for_publish_confirmations := WaitForPublishConfirmations
     } = Config,
+    %% TODO: teach `amqp` to accept 0-arity closures as passwords.
     Password = emqx_secret:unwrap(WrappedPassword),
     SSLOptions =
         case maps:get(ssl, Config, #{}) of

apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl

@@ -10,6 +10,7 @@
 -include("emqx_connector.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("stdlib/include/assert.hrl").
+-include_lib("common_test/include/ct.hrl").
 -include_lib("amqp_client/include/amqp_client.hrl").
 
 %% This test SUITE requires a running RabbitMQ instance. If you don't want to
@@ -26,6 +27,9 @@ rabbit_mq_host() ->
 rabbit_mq_port() ->
     5672.
 
+rabbit_mq_password() ->
+    <<"guest">>.
+
 rabbit_mq_exchange() ->
     <<"test_exchange">>.
 
@@ -45,12 +49,12 @@ init_per_suite(Config) ->
         )
     of
         true ->
-            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+            Apps = emqx_cth_suite:start(
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
+                [emqx_conf, emqx_connector, emqx_bridge_rabbitmq],
-            {ok, _} = application:ensure_all_started(emqx_connector),
+                #{work_dir => emqx_cth_suite:work_dir(Config)}
-            {ok, _} = application:ensure_all_started(amqp_client),
+            ),
             ChannelConnection = setup_rabbit_mq_exchange_and_queue(),
-            [{channel_connection, ChannelConnection} | Config];
+            [{channel_connection, ChannelConnection}, {suite_apps, Apps} | Config];
         false ->
             case os:getenv("IS_CI") of
                 "yes" ->
@@ -106,13 +110,11 @@ end_per_suite(Config) ->
         connection := Connection,
         channel := Channel
     } = get_channel_connection(Config),
-    ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
-    ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
-    _ = application:stop(emqx_connector),
     %% Close the channel
     ok = amqp_channel:close(Channel),
     %% Close the connection
-    ok = amqp_connection:close(Connection).
+    ok = amqp_connection:close(Connection),
+    ok = emqx_cth_suite:stop(?config(suite_apps, Config)).
 
 % %%------------------------------------------------------------------------------
 % %% Testcases
@@ -125,23 +127,31 @@ t_lifecycle(Config) ->
         Config
     ).
 
+t_start_passfile(Config) ->
+    ResourceID = atom_to_binary(?FUNCTION_NAME),
+    PasswordFilename = filename:join(?config(priv_dir, Config), "passfile"),
+    ok = file:write_file(PasswordFilename, rabbit_mq_password()),
+    InitialConfig = rabbitmq_config(#{
+        password => iolist_to_binary(["file://", PasswordFilename])
+    }),
+    ?assertMatch(
+        #{status := connected},
+        create_local_resource(ResourceID, check_config(InitialConfig))
+    ),
+    ?assertEqual(
+        ok,
+        emqx_resource:remove_local(ResourceID)
+    ).
+
 perform_lifecycle_check(ResourceID, InitialConfig, TestConfig) ->
     #{
         channel := Channel
     } = get_channel_connection(TestConfig),
-    {ok, #{config := CheckedConfig}} =
+    CheckedConfig = check_config(InitialConfig),
-        emqx_resource:check_config(emqx_bridge_rabbitmq_connector, InitialConfig),
+    #{
-    {ok, #{
         state := #{poolname := PoolName} = State,
         status := InitialStatus
-    }} =
+    } = create_local_resource(ResourceID, CheckedConfig),
-        emqx_resource:create_local(
-            ResourceID,
-            ?CONNECTOR_RESOURCE_GROUP,
-            emqx_bridge_rabbitmq_connector,
-            CheckedConfig,
-            #{}
-        ),
     ?assertEqual(InitialStatus, connected),
     %% Instance should match the state and status of the just started resource
     {ok, ?CONNECTOR_RESOURCE_GROUP, #{
@@ -184,6 +194,21 @@ perform_lifecycle_check(ResourceID, InitialConfig, TestConfig) ->
 % %% Helpers
 % %%------------------------------------------------------------------------------
 
+check_config(Config) ->
+    {ok, #{config := CheckedConfig}} =
+        emqx_resource:check_config(emqx_bridge_rabbitmq_connector, Config),
+    CheckedConfig.
+
+create_local_resource(ResourceID, CheckedConfig) ->
+    {ok, Bridge} = emqx_resource:create_local(
+        ResourceID,
+        ?CONNECTOR_RESOURCE_GROUP,
+        emqx_bridge_rabbitmq_connector,
+        CheckedConfig,
+        #{}
+    ),
+    Bridge.
+
 perform_query(PoolName, Channel) ->
     %% Send message to queue:
     ok = emqx_resource:query(PoolName, {query, test_data()}),
@@ -216,16 +241,19 @@ receive_simple_test_message(Channel) ->
     end.
 
 rabbitmq_config() ->
+    rabbitmq_config(#{}).
+
+rabbitmq_config(Overrides) ->
     Config =
         #{
             server => rabbit_mq_host(),
             port => 5672,
             username => <<"guest">>,
-            password => <<"guest">>,
+            password => rabbit_mq_password(),
             exchange => rabbit_mq_exchange(),
             routing_key => rabbit_mq_routing_key()
         },
-    #{<<"config">> => Config}.
+    #{<<"config">> => maps:merge(Config, Overrides)}.
 
 test_data() ->
     #{<<"msg_field">> => <<"Hello">>}.

apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_rocketmq, [
     {description, "EMQX Enterprise RocketMQ Bridge"},
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {registered, []},
     {applications, [kernel, stdlib, emqx_resource, rocketmq]},
     {env, []},

apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl

@@ -48,13 +48,8 @@ fields(config) ->
                 binary(),
                 #{default => <<>>, desc => ?DESC("access_key")}
             )},
-        {secret_key,
+        {secret_key, emqx_schema_secret:mk(#{default => <<>>, desc => ?DESC("secret_key")})},
-            mk(
+        {security_token, emqx_schema_secret:mk(#{default => <<>>, desc => ?DESC(security_token)})},
-                binary(),
-                #{default => <<>>, desc => ?DESC("secret_key"), sensitive => true}
-            )},
-        {security_token,
-            mk(binary(), #{default => <<>>, desc => ?DESC(security_token), sensitive => true})},
         {sync_timeout,
             mk(
                 emqx_schema:timeout_duration(),
@@ -294,21 +289,19 @@ make_producer_opts(
         acl_info => emqx_secret:wrap(ACLInfo)
     }.
 
-acl_info(<<>>, <<>>, <<>>) ->
+acl_info(<<>>, _, _) ->
     #{};
-acl_info(AccessKey, SecretKey, <<>>) when is_binary(AccessKey), is_binary(SecretKey) ->
+acl_info(AccessKey, SecretKey, SecurityToken) when is_binary(AccessKey) ->
-    #{
+    Info = #{
         access_key => AccessKey,
-        secret_key => SecretKey
+        secret_key => emqx_maybe:define(emqx_secret:unwrap(SecretKey), <<>>)
-    };
+    },
-acl_info(AccessKey, SecretKey, SecurityToken) when
+    case emqx_maybe:define(emqx_secret:unwrap(SecurityToken), <<>>) of
-    is_binary(AccessKey), is_binary(SecretKey), is_binary(SecurityToken)
+        <<>> ->
-->
+            Info;
-    #{
+        Token ->
-        access_key => AccessKey,
+            Info#{security_token => Token}
-        secret_key => SecretKey,
+    end;
-        security_token => SecurityToken
-    };
 acl_info(_, _, _) ->
     #{}.
 

apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_sqlserver, [
     {description, "EMQX Enterprise SQL Server Bridge"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {applications, [kernel, stdlib, emqx_resource, odbc]},
     {env, []},