yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/release-54'

This commit is contained in:
Ivan Dyachkov 2023-11-14 19:38:21 +01:00
parent 88637f81d1 e80600ca0f
commit 7c0e345d3a
339 changed files with 25754 additions and 1949 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.ci/docker-compose-file/credentials.env Normal file
View File

@ -0,0 +1,7 @@
MONGO_USERNAME=emqx
MONGO_PASSWORD=passw0rd
MONGO_AUTHSOURCE=admin
# See "Environment Variables" @ https://hub.docker.com/_/mongo
MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME}
MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD}

3
.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml
View File

@ -9,6 +9,9 @@ services:
- emqx_bridge
ports:
- "27017:27017"
env_file:
- .env
- credentials.env
command:
--ipv6
--bind_ip_all

1
.ci/docker-compose-file/docker-compose.yaml
View File

@ -5,6 +5,7 @@ services:
container_name: erlang
image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04}
env_file:
- credentials.env
- conf.env
environment:
GITHUB_ACTIONS: ${GITHUB_ACTIONS:-}

2
README-CN.md
View File

@ -77,7 +77,7 @@ EMQX Cloud 文档：[docs.emqx.com/zh/cloud/latest/](https://docs.emqx.com/zh/cl
优雅的跨平台 MQTT 5.0 客户端工具，提供了桌面端、命令行、Web 三种版本，帮助您更快的开发和调试 MQTT 服务和应用。
- [车联网平台搭建从入门到精通 ](https://www.emqx.com/zh/blog/category/internet-of-vehicles)
- [车联网平台搭建从入门到精通](https://www.emqx.com/zh/blog/category/internet-of-vehicles)
结合 EMQ 在车联网领域的实践经验，从协议选择等理论知识，到平台架构设计等实战操作，分享如何搭建一个可靠、高效、符合行业场景需求的车联网平台。

3
apps/emqx/include/emqx.hrl
View File

@ -39,9 +39,6 @@
%% System topic
-define(SYSTOP, <<"$SYS/">>).
%% Queue topic
-define(QUEUE, <<"$queue/">>).
%%--------------------------------------------------------------------
%% alarms
%%--------------------------------------------------------------------

20
apps/emqx/include/emqx_mqtt.hrl
View File

@ -55,6 +55,17 @@
%% MQTT-3.1.1 and MQTT-5.0 [MQTT-4.7.3-3]
-define(MAX_TOPIC_LEN, 65535).
%%--------------------------------------------------------------------
%% MQTT Share-Sub Internal
%%--------------------------------------------------------------------
-record(share, {group :: emqx_types:group(), topic :: emqx_types:topic()}).
%% guards
-define(IS_TOPIC(T),
(is_binary(T) orelse is_record(T, share))
).
%%--------------------------------------------------------------------
%% MQTT QoS Levels
%%--------------------------------------------------------------------
@ -661,13 +672,10 @@ end).
-define(PACKET(Type), #mqtt_packet{header = #mqtt_packet_header{type = Type}}).
-define(SHARE, "$share").
-define(QUEUE, "$queue").
-define(SHARE(Group, Topic), emqx_topic:join([<<?SHARE>>, Group, Topic])).
-define(IS_SHARE(Topic),
case Topic of
<<?SHARE, _/binary>> -> true;
_ -> false
end
).
-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
-define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty).
-define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty).

1
apps/emqx/include/emqx_trace.hrl
View File

@ -32,6 +32,5 @@
-define(SHARD, ?COMMON_SHARD).
-define(MAX_SIZE, 30).
-define(OWN_KEYS, [level, filters, filter_default, handlers]).
-endif.

1
apps/emqx/include/http_api.hrl
View File

@ -17,6 +17,7 @@
%% HTTP API Auth
-define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
-define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET').
-define(API_KEY_NOT_ALLOW_MSG, <<"This API Key don't have permission to access this resource">>).
%% Bad Request
-define(BAD_REQUEST, 'BAD_REQUEST').

34
apps/emqx/include/logger.hrl
View File

@ -40,7 +40,9 @@
end
).
-define(AUDIT_HANDLER, emqx_audit).
-define(TRACE_FILTER, emqx_trace_filter).
-define(OWN_KEYS, [level, filters, filter_default, handlers]).
-define(TRACE(Tag, Msg, Meta), ?TRACE(debug, Tag, Msg, Meta)).
@ -61,25 +63,35 @@
)
end).
-define(AUDIT(_Level_, _From_, _Meta_), begin
case emqx_config:get([log, audit], #{enable => false}) of
#{enable := false} ->
-ifdef(EMQX_RELEASE_EDITION).
-if(?EMQX_RELEASE_EDITION == ee).
-define(AUDIT(_LevelFun_, _MetaFun_), begin
case logger_config:get(logger, ?AUDIT_HANDLER) of
{error, {not_found, _}} ->
ok;
#{enable := true, level := _AllowLevel_} ->
{ok, Handler = #{level := _AllowLevel_}} ->
_Level_ = _LevelFun_,
case logger:compare_levels(_AllowLevel_, _Level_) of
_R_ when _R_ == lt; _R_ == eq ->
emqx_trace:log(
_Level_,
[{emqx_audit, fun(L, _) -> L end, undefined, undefined}],
_Msg = undefined,
_Meta_#{from => _From_}
);
gt ->
emqx_audit:log(_Level_, _MetaFun_, Handler);
_ ->
ok
end
end
end).
-else.
%% Only for compile pass, ce edition will not call it
-define(AUDIT(_L_, _M_), _ = {_L_, _M_}).
-endif.
-else.
%% Only for compile pass, ce edition will not call it
-define(AUDIT(_L_, _M_), _ = {_L_, _M_}).
-endif.
%% print to 'user' group leader
-define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
-define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)).

17
apps/emqx/src/bhvrs/emqx_db_backup.erl
View File

@ -16,4 +16,21 @@
-module(emqx_db_backup).
-type traverse_break_reason() :: over | migrate.
-callback backup_tables() -> [mria:table()].
%% validate the backup
%% return `ok` to traverse the next item
%% return `{ok, over}` to finish the traverse
%% return `{ok, migrate}` to call the migration callback
-callback validate_mnesia_backup(tuple()) ->
ok
| {ok, traverse_break_reason()}
| {error, term()}.
-callback migrate_mnesia_backup(tuple()) -> {ok, tuple()} | {error, term()}.
-optional_callbacks([validate_mnesia_backup/1, migrate_mnesia_backup/1]).
-export_type([traverse_break_reason/0]).

7
apps/emqx/src/config/emqx_config_logger.erl
View File

@ -23,8 +23,9 @@
-export([post_config_update/5]).
-export([filter_audit/2]).
-include("logger.hrl").
-define(LOG, [log]).
-define(AUDIT_HANDLER, emqx_audit).
add_handler() ->
ok = emqx_config_handler:add_handler(?LOG, ?MODULE),
@ -95,6 +96,10 @@ update_log_handlers(NewHandlers) ->
ok = application:set_env(kernel, logger, NewHandlers),
ok.
%% Don't remove audit log handler here, we need record this removed action into audit log file.
%% we will remove audit log handler after audit log is record in emqx_audit:log/3.
update_log_handler({removed, ?AUDIT_HANDLER}) ->
ok;
update_log_handler({removed, Id}) ->
log_to_console("Config override: ~s is removed~n", [id_for_log(Id)]),
logger:remove_handler(Id);

88
apps/emqx/src/emqx_broker.erl
View File

@ -118,18 +118,20 @@ create_tabs() ->
%% Subscribe API
%%------------------------------------------------------------------------------
-spec subscribe(emqx_types:topic()) -> ok.
subscribe(Topic) when is_binary(Topic) ->
-spec subscribe(emqx_types:topic() | emqx_types:share()) -> ok.
subscribe(Topic) when ?IS_TOPIC(Topic) ->
subscribe(Topic, undefined).
-spec subscribe(emqx_types:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok.
subscribe(Topic, SubId) when is_binary(Topic), ?IS_SUBID(SubId) ->
-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid() | emqx_types:subopts()) ->
ok.
subscribe(Topic, SubId) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId) ->
subscribe(Topic, SubId, ?DEFAULT_SUBOPTS);
subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) ->
subscribe(Topic, SubOpts) when ?IS_TOPIC(Topic), is_map(SubOpts) ->
subscribe(Topic, undefined, SubOpts).
-spec subscribe(emqx_types:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok.
subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) ->
-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid(), emqx_types:subopts()) ->
ok.
subscribe(Topic, SubId, SubOpts0) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) ->
SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0),
_ = emqx_trace:subscribe(Topic, SubId, SubOpts),
SubPid = self(),
@ -151,13 +153,13 @@ with_subid(undefined, SubOpts) ->
with_subid(SubId, SubOpts) ->
maps:put(subid, SubId, SubOpts).
%% @private
do_subscribe(Topic, SubPid, SubOpts) ->
true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}),
Group = maps:get(share, SubOpts, undefined),
do_subscribe(Group, Topic, SubPid, SubOpts).
do_subscribe2(Topic, SubPid, SubOpts).
do_subscribe(undefined, Topic, SubPid, SubOpts) ->
do_subscribe2(Topic, SubPid, SubOpts) when is_binary(Topic) ->
%% FIXME: subscribe shard bug
%% https://emqx.atlassian.net/browse/EMQX-10214
case emqx_broker_helper:get_sub_shard(SubPid, Topic) of
0 ->
true = ets:insert(?SUBSCRIBER, {Topic, SubPid}),
@ -168,34 +170,40 @@ do_subscribe(undefined, Topic, SubPid, SubOpts) ->
true = ets:insert(?SUBOPTION, {{Topic, SubPid}, maps:put(shard, I, SubOpts)}),
call(pick({Topic, I}), {subscribe, Topic, I})
end;
%% Shared subscription
do_subscribe(Group, Topic, SubPid, SubOpts) ->
do_subscribe2(Topic = #share{group = Group, topic = RealTopic}, SubPid, SubOpts) when
is_binary(RealTopic)
->
true = ets:insert(?SUBOPTION, {{Topic, SubPid}, SubOpts}),
emqx_shared_sub:subscribe(Group, Topic, SubPid).
emqx_shared_sub:subscribe(Group, RealTopic, SubPid).
%%--------------------------------------------------------------------
%% Unsubscribe API
%%--------------------------------------------------------------------
-spec unsubscribe(emqx_types:topic()) -> ok.
unsubscribe(Topic) when is_binary(Topic) ->
-spec unsubscribe(emqx_types:topic() | emqx_types:share()) -> ok.
unsubscribe(Topic) when ?IS_TOPIC(Topic) ->
SubPid = self(),
case ets:lookup(?SUBOPTION, {Topic, SubPid}) of
[{_, SubOpts}] ->
_ = emqx_broker_helper:reclaim_seq(Topic),
_ = emqx_trace:unsubscribe(Topic, SubOpts),
do_unsubscribe(Topic, SubPid, SubOpts);
[] ->
ok
end.
-spec do_unsubscribe(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) ->
ok.
do_unsubscribe(Topic, SubPid, SubOpts) ->
true = ets:delete(?SUBOPTION, {Topic, SubPid}),
true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}),
Group = maps:get(share, SubOpts, undefined),
do_unsubscribe(Group, Topic, SubPid, SubOpts).
do_unsubscribe2(Topic, SubPid, SubOpts).
do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
-spec do_unsubscribe2(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) ->
ok.
do_unsubscribe2(Topic, SubPid, SubOpts) when
is_binary(Topic), is_pid(SubPid), is_map(SubOpts)
->
_ = emqx_broker_helper:reclaim_seq(Topic),
case maps:get(shard, SubOpts, 0) of
0 ->
true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}),
@ -205,7 +213,9 @@ do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}),
cast(pick({Topic, I}), {unsubscribed, Topic, I})
end;
do_unsubscribe(Group, Topic, SubPid, _SubOpts) ->
do_unsubscribe2(#share{group = Group, topic = Topic}, SubPid, _SubOpts) when
is_binary(Group), is_binary(Topic), is_pid(SubPid)
->
emqx_shared_sub:unsubscribe(Group, Topic, SubPid).
%%--------------------------------------------------------------------
@ -306,7 +316,9 @@ aggre([], true, Acc) ->
lists:usort(Acc).
%% @doc Forward message to another node.
-spec forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode :: sync | async) ->
-spec forward(
node(), emqx_types:topic() | emqx_types:share(), emqx_types:delivery(), RpcMode :: sync | async
) ->
emqx_types:deliver_result().
forward(Node, To, Delivery, async) ->
true = emqx_broker_proto_v1:forward_async(Node, To, Delivery),
@ -329,7 +341,8 @@ forward(Node, To, Delivery, sync) ->
Result
end.
-spec dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result().
-spec dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) ->
emqx_types:deliver_result().
dispatch(Topic, Delivery = #delivery{}) when is_binary(Topic) ->
case emqx:is_running() of
true ->
@ -353,7 +366,11 @@ inc_dropped_cnt(Msg) ->
end.
-compile({inline, [subscribers/1]}).
-spec subscribers(emqx_types:topic() | {shard, emqx_types:topic(), non_neg_integer()}) ->
-spec subscribers(
emqx_types:topic()
| emqx_types:share()
| {shard, emqx_types:topic() | emqx_types:share(), non_neg_integer()}
) ->
[pid()].
subscribers(Topic) when is_binary(Topic) ->
lookup_value(?SUBSCRIBER, Topic, []);
@ -372,7 +389,7 @@ subscriber_down(SubPid) ->
SubOpts when is_map(SubOpts) ->
_ = emqx_broker_helper:reclaim_seq(Topic),
true = ets:delete(?SUBOPTION, {Topic, SubPid}),
do_unsubscribe(undefined, Topic, SubPid, SubOpts);
do_unsubscribe2(Topic, SubPid, SubOpts);
undefined ->
ok
end
@ -386,7 +403,7 @@ subscriber_down(SubPid) ->
%%--------------------------------------------------------------------
-spec subscriptions(pid() | emqx_types:subid()) ->
[{emqx_types:topic(), emqx_types:subopts()}].
[{emqx_types:topic() | emqx_types:share(), emqx_types:subopts()}].
subscriptions(SubPid) when is_pid(SubPid) ->
[
{Topic, lookup_value(?SUBOPTION, {Topic, SubPid}, #{})}
@ -400,20 +417,22 @@ subscriptions(SubId) ->
[]
end.
-spec subscriptions_via_topic(emqx_types:topic()) -> [emqx_types:subopts()].
-spec subscriptions_via_topic(emqx_types:topic() | emqx_types:share()) -> [emqx_types:subopts()].
subscriptions_via_topic(Topic) ->
MatchSpec = [{{{Topic, '_'}, '_'}, [], ['$_']}],
ets:select(?SUBOPTION, MatchSpec).
-spec subscribed(pid() | emqx_types:subid(), emqx_types:topic()) -> boolean().
-spec subscribed(
pid() | emqx_types:subid(), emqx_types:topic() | emqx_types:share()
) -> boolean().
subscribed(SubPid, Topic) when is_pid(SubPid) ->
ets:member(?SUBOPTION, {Topic, SubPid});
subscribed(SubId, Topic) when ?IS_SUBID(SubId) ->
SubPid = emqx_broker_helper:lookup_subpid(SubId),
ets:member(?SUBOPTION, {Topic, SubPid}).
-spec get_subopts(pid(), emqx_types:topic()) -> maybe(emqx_types:subopts()).
get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) ->
-spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> maybe(emqx_types:subopts()).
get_subopts(SubPid, Topic) when is_pid(SubPid), ?IS_TOPIC(Topic) ->
lookup_value(?SUBOPTION, {Topic, SubPid});
get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->
case emqx_broker_helper:lookup_subpid(SubId) of
@ -423,7 +442,7 @@ get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->
undefined
end.
-spec set_subopts(emqx_types:topic(), emqx_types:subopts()) -> boolean().
-spec set_subopts(emqx_types:topic() | emqx_types:share(), emqx_types:subopts()) -> boolean().
set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) ->
set_subopts(self(), Topic, NewOpts).
@ -437,7 +456,7 @@ set_subopts(SubPid, Topic, NewOpts) ->
false
end.
-spec topics() -> [emqx_types:topic()].
-spec topics() -> [emqx_types:topic() | emqx_types:share()].
topics() ->
emqx_router:topics().
@ -542,7 +561,8 @@ code_change(_OldVsn, State, _Extra) ->
%% Internal functions
%%--------------------------------------------------------------------
-spec do_dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result().
-spec do_dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) ->
emqx_types:deliver_result().
do_dispatch(Topic, #delivery{message = Msg}) ->
DispN = lists:foldl(
fun(Sub, N) ->
@ -560,6 +580,8 @@ do_dispatch(Topic, #delivery{message = Msg}) ->
{ok, DispN}
end.
%% Donot dispatch to share subscriber here.
%% we do it in `emqx_shared_sub.erl` with configured strategy
do_dispatch(SubPid, Topic, Msg) when is_pid(SubPid) ->
case erlang:is_process_alive(SubPid) of
true ->

301
apps/emqx/src/emqx_channel.erl
View File

@ -476,60 +476,27 @@ handle_in(
ok = emqx_metrics:inc('packets.pubcomp.missed'),
{ok, Channel}
end;
handle_in(
SubPkt = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
Channel = #channel{clientinfo = ClientInfo}
) ->
case emqx_packet:check(SubPkt) of
ok ->
TopicFilters0 = parse_topic_filters(TopicFilters),
TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0),
TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel),
HasAuthzDeny = lists:any(
fun({_TopicFilter, ReasonCode}) ->
ReasonCode =:= ?RC_NOT_AUTHORIZED
end,
TupleTopicFilters0
),
DenyAction = emqx:get_config([authorization, deny_action], ignore),
case DenyAction =:= disconnect andalso HasAuthzDeny of
true ->
handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel);
false ->
TopicFilters2 = [
TopicFilter
|| {TopicFilter, ?RC_SUCCESS} <- TupleTopicFilters0
handle_in(SubPkt = ?SUBSCRIBE_PACKET(PacketId, _Properties, _TopicFilters0), Channel0) ->
Pipe = pipeline(
[
fun check_subscribe/2,
fun enrich_subscribe/2,
%% TODO && FIXME (EMQX-10786): mount topic before authz check.
fun check_sub_authzs/2,
fun check_sub_caps/2
],
TopicFilters3 = run_hooks(
'client.subscribe',
[ClientInfo, Properties],
TopicFilters2
SubPkt,
Channel0
),
{TupleTopicFilters1, NChannel} = process_subscribe(
TopicFilters3,
Properties,
Channel
),
TupleTopicFilters2 =
lists:foldl(
fun
({{Topic, Opts = #{deny_subscription := true}}, _QoS}, Acc) ->
Key = {Topic, maps:without([deny_subscription], Opts)},
lists:keyreplace(Key, 1, Acc, {Key, ?RC_UNSPECIFIED_ERROR});
(Tuple = {Key, _Value}, Acc) ->
lists:keyreplace(Key, 1, Acc, Tuple)
end,
TupleTopicFilters0,
TupleTopicFilters1
),
ReasonCodes2 = [
ReasonCode
|| {_TopicFilter, ReasonCode} <- TupleTopicFilters2
],
handle_out(suback, {PacketId, ReasonCodes2}, NChannel)
end;
{error, ReasonCode} ->
handle_out(disconnect, ReasonCode, Channel)
case Pipe of
{ok, NPkt = ?SUBSCRIBE_PACKET(_PacketId, TFChecked), Channel} ->
{TFSubedWithNRC, NChannel} = process_subscribe(run_sub_hooks(NPkt, Channel), Channel),
ReasonCodes = gen_reason_codes(TFChecked, TFSubedWithNRC),
handle_out(suback, {PacketId, ReasonCodes}, NChannel);
{error, {disconnect, RC}, Channel} ->
%% funcs in pipeline always cause action: `disconnect`
%% And Only one ReasonCode in DISCONNECT packet
handle_out(disconnect, RC, Channel)
end;
handle_in(
Packet = ?UNSUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
@ -540,7 +507,7 @@ handle_in(
TopicFilters1 = run_hooks(
'client.unsubscribe',
[ClientInfo, Properties],
parse_topic_filters(TopicFilters)
parse_raw_topic_filters(TopicFilters)
),
{ReasonCodes, NChannel} = process_unsubscribe(TopicFilters1, Properties, Channel),
handle_out(unsuback, {PacketId, ReasonCodes}, NChannel);
@ -782,32 +749,14 @@ after_message_acked(ClientInfo, Msg, PubAckProps) ->
%% Process Subscribe
%%--------------------------------------------------------------------
-compile({inline, [process_subscribe/3]}).
process_subscribe(TopicFilters, SubProps, Channel) ->
process_subscribe(TopicFilters, SubProps, Channel, []).
process_subscribe(TopicFilters, Channel) ->
process_subscribe(TopicFilters, Channel, []).
process_subscribe([], _SubProps, Channel, Acc) ->
process_subscribe([], Channel, Acc) ->
{lists:reverse(Acc), Channel};
process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Acc) ->
case check_sub_caps(TopicFilter, SubOpts, Channel) of
ok ->
{ReasonCode, NChannel} = do_subscribe(
TopicFilter,
SubOpts#{sub_props => SubProps},
Channel
),
process_subscribe(More, SubProps, NChannel, [{Topic, ReasonCode} | Acc]);
{error, ReasonCode} ->
?SLOG(
warning,
#{
msg => "cannot_subscribe_topic_filter",
reason => emqx_reason_codes:name(ReasonCode)
},
#{topic => TopicFilter}
),
process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc])
end.
process_subscribe([Filter = {TopicFilter, SubOpts} | More], Channel, Acc) ->
{NReasonCode, NChannel} = do_subscribe(TopicFilter, SubOpts, Channel),
process_subscribe(More, NChannel, [{Filter, NReasonCode} | Acc]).
do_subscribe(
TopicFilter,
@ -818,11 +767,13 @@ do_subscribe(
session = Session
}
) ->
%% TODO && FIXME (EMQX-10786): mount topic before authz check.
NTopicFilter = emqx_mountpoint:mount(MountPoint, TopicFilter),
NSubOpts = enrich_subopts(maps:merge(?DEFAULT_SUBOPTS, SubOpts), Channel),
case emqx_session:subscribe(ClientInfo, NTopicFilter, NSubOpts, Session) of
case emqx_session:subscribe(ClientInfo, NTopicFilter, SubOpts, Session) of
{ok, NSession} ->
{QoS, Channel#channel{session = NSession}};
%% TODO && FIXME (EMQX-11216): QoS as ReasonCode(max granted QoS) for now
RC = QoS,
{RC, Channel#channel{session = NSession}};
{error, RC} ->
?SLOG(
warning,
@ -835,6 +786,30 @@ do_subscribe(
{RC, Channel}
end.
gen_reason_codes(TFChecked, TFSubedWitNhRC) ->
do_gen_reason_codes([], TFChecked, TFSubedWitNhRC).
%% Initial RC is `RC_SUCCESS | RC_NOT_AUTHORIZED`, generated by check_sub_authzs/2
%% And then TF with `RC_SUCCESS` will passing through `process_subscribe/2` and
%% NRC should override the initial RC.
do_gen_reason_codes(Acc, [], []) ->
lists:reverse(Acc);
do_gen_reason_codes(
Acc,
[{_, ?RC_SUCCESS} | RestTF],
[{_, NRC} | RestWithNRC]
) ->
%% will passing through `process_subscribe/2`
%% use NRC to override IintialRC
do_gen_reason_codes([NRC | Acc], RestTF, RestWithNRC);
do_gen_reason_codes(
Acc,
[{_, InitialRC} | Rest],
RestWithNRC
) ->
%% InitialRC is not `RC_SUCCESS`, use it.
do_gen_reason_codes([InitialRC | Acc], Rest, RestWithNRC).
%%--------------------------------------------------------------------
%% Process Unsubscribe
%%--------------------------------------------------------------------
@ -1213,13 +1188,8 @@ handle_call(Req, Channel) ->
ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}.
handle_info({subscribe, TopicFilters}, Channel) ->
{_, NChannel} = lists:foldl(
fun({TopicFilter, SubOpts}, {_, ChannelAcc}) ->
do_subscribe(TopicFilter, SubOpts, ChannelAcc)
end,
{[], Channel},
parse_topic_filters(TopicFilters)
),
NTopicFilters = enrich_subscribe(TopicFilters, Channel),
{_TopicFiltersWithRC, NChannel} = process_subscribe(NTopicFilters, Channel),
{ok, NChannel};
handle_info({unsubscribe, TopicFilters}, Channel) ->
{_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel),
@ -1857,49 +1827,156 @@ check_pub_caps(
) ->
emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic => Topic}).
%%--------------------------------------------------------------------
%% Check Subscribe Packet
check_subscribe(SubPkt, _Channel) ->
case emqx_packet:check(SubPkt) of
ok -> ok;
{error, RC} -> {error, {disconnect, RC}}
end.
%%--------------------------------------------------------------------
%% Check Sub Authorization
check_sub_authzs(TopicFilters, Channel) ->
check_sub_authzs(TopicFilters, Channel, []).
check_sub_authzs(
[TopicFilter = {Topic, _} | More],
Channel = #channel{clientinfo = ClientInfo},
Acc
?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters0),
Channel = #channel{clientinfo = ClientInfo}
) ->
CheckResult = do_check_sub_authzs(TopicFilters0, ClientInfo),
HasAuthzDeny = lists:any(
fun({{_TopicFilter, _SubOpts}, ReasonCode}) ->
ReasonCode =:= ?RC_NOT_AUTHORIZED
end,
CheckResult
),
DenyAction = emqx:get_config([authorization, deny_action], ignore),
case DenyAction =:= disconnect andalso HasAuthzDeny of
true ->
{error, {disconnect, ?RC_NOT_AUTHORIZED}, Channel};
false ->
{ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}
end.
do_check_sub_authzs(TopicFilters, ClientInfo) ->
do_check_sub_authzs(ClientInfo, TopicFilters, []).
do_check_sub_authzs(_ClientInfo, [], Acc) ->
lists:reverse(Acc);
do_check_sub_authzs(ClientInfo, [TopicFilter = {Topic, _SubOpts} | More], Acc) ->
%% subsclibe authz check only cares the real topic filter when shared-sub
%% e.g. only check <<"t/#">> for <<"$share/g/t/#">>
Action = authz_action(TopicFilter),
case emqx_access_control:authorize(ClientInfo, Action, Topic) of
case
emqx_access_control:authorize(
ClientInfo,
Action,
emqx_topic:get_shared_real_topic(Topic)
)
of
%% TODO: support maximum QoS granted
%% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7]
%% Not implemented yet:
%% {allow, RC} -> do_check_sub_authzs(ClientInfo, More, [{TopicFilter, RC} | Acc]);
allow ->
check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]);
do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_SUCCESS} | Acc]);
deny ->
check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
end;
check_sub_authzs([], _Channel, Acc) ->
lists:reverse(Acc).
do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
end.
%%--------------------------------------------------------------------
%% Check Sub Caps
check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = ClientInfo}) ->
emqx_mqtt_caps:check_sub(ClientInfo, TopicFilter, SubOpts).
check_sub_caps(
?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters),
Channel = #channel{clientinfo = ClientInfo}
) ->
CheckResult = do_check_sub_caps(ClientInfo, TopicFilters),
{ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}.
do_check_sub_caps(ClientInfo, TopicFilters) ->
do_check_sub_caps(ClientInfo, TopicFilters, []).
do_check_sub_caps(_ClientInfo, [], Acc) ->
lists:reverse(Acc);
do_check_sub_caps(ClientInfo, [TopicFilter = {{Topic, SubOpts}, ?RC_SUCCESS} | More], Acc) ->
case emqx_mqtt_caps:check_sub(ClientInfo, Topic, SubOpts) of
ok ->
do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]);
{error, NRC} ->
?SLOG(
warning,
#{
msg => "cannot_subscribe_topic_filter",
reason => emqx_reason_codes:name(NRC)
},
#{topic => Topic}
),
do_check_sub_caps(ClientInfo, More, [{{Topic, SubOpts}, NRC} | Acc])
end;
do_check_sub_caps(ClientInfo, [TopicFilter = {{_Topic, _SubOpts}, _OtherRC} | More], Acc) ->
do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]).
%%--------------------------------------------------------------------
%% Enrich SubId
%% Run Subscribe Hooks
enrich_subopts_subid(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
[{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
enrich_subopts_subid(_Properties, TopicFilters) ->
TopicFilters.
run_sub_hooks(
?SUBSCRIBE_PACKET(_PacketId, Properties, TopicFilters0),
_Channel = #channel{clientinfo = ClientInfo}
) ->
TopicFilters = [
TopicFilter
|| {TopicFilter, ?RC_SUCCESS} <- TopicFilters0
],
_NTopicFilters = run_hooks('client.subscribe', [ClientInfo, Properties], TopicFilters).
%%--------------------------------------------------------------------
%% Enrich SubOpts
enrich_subopts(SubOpts, _Channel = ?IS_MQTT_V5) ->
SubOpts;
enrich_subopts(SubOpts, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) ->
%% for api subscribe without sub-authz check and sub-caps check.
enrich_subscribe(TopicFilters, Channel) when is_list(TopicFilters) ->
do_enrich_subscribe(#{}, TopicFilters, Channel);
%% for mqtt clients sent subscribe packet.
enrich_subscribe(?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), Channel) ->
NTopicFilters = do_enrich_subscribe(Properties, TopicFilters, Channel),
{ok, ?SUBSCRIBE_PACKET(PacketId, Properties, NTopicFilters), Channel}.
do_enrich_subscribe(Properties, TopicFilters, Channel) ->
_NTopicFilters = run_fold(
[
%% TODO: do try catch with reason code here
fun(TFs, _) -> parse_raw_topic_filters(TFs) end,
fun enrich_subopts_subid/2,
fun enrich_subopts_porps/2,
fun enrich_subopts_flags/2
],
TopicFilters,
#{sub_props => Properties, channel => Channel}
).
enrich_subopts_subid(TopicFilters, #{sub_props := #{'Subscription-Identifier' := SubId}}) ->
[{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
enrich_subopts_subid(TopicFilters, _State) ->
TopicFilters.
enrich_subopts_porps(TopicFilters, #{sub_props := SubProps}) ->
[{Topic, SubOpts#{sub_props => SubProps}} || {Topic, SubOpts} <- TopicFilters].
enrich_subopts_flags(TopicFilters, #{channel := Channel}) ->
do_enrich_subopts_flags(TopicFilters, Channel).
do_enrich_subopts_flags(TopicFilters, ?IS_MQTT_V5) ->
[{Topic, merge_default_subopts(SubOpts)} || {Topic, SubOpts} <- TopicFilters];
do_enrich_subopts_flags(TopicFilters, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) ->
Rap = flag(IsBridge),
NL = flag(get_mqtt_conf(Zone, ignore_loop_deliver)),
SubOpts#{rap => flag(IsBridge), nl => NL}.
[
{Topic, (merge_default_subopts(SubOpts))#{rap => Rap, nl => NL}}
|| {Topic, SubOpts} <- TopicFilters
].
merge_default_subopts(SubOpts) ->
maps:merge(?DEFAULT_SUBOPTS, SubOpts).
%%--------------------------------------------------------------------
%% Enrich ConnAck Caps
@ -2089,8 +2166,8 @@ maybe_shutdown(Reason, _Intent = shutdown, Channel) ->
%%--------------------------------------------------------------------
%% Parse Topic Filters
-compile({inline, [parse_topic_filters/1]}).
parse_topic_filters(TopicFilters) ->
%% [{<<"$share/group/topic">>, _SubOpts = #{}} | _]
parse_raw_topic_filters(TopicFilters) ->
lists:map(fun emqx_topic:parse/1, TopicFilters).
%%--------------------------------------------------------------------

47
apps/emqx/src/emqx_mountpoint.erl
View File

@ -17,6 +17,7 @@
-module(emqx_mountpoint).
-include("emqx.hrl").
-include("emqx_mqtt.hrl").
-include("emqx_placeholder.hrl").
-include("types.hrl").
@ -34,38 +35,54 @@
-spec mount(maybe(mountpoint()), Any) -> Any when
Any ::
emqx_types:topic()
| emqx_types:share()
| emqx_types:message()
| emqx_types:topic_filters().
mount(undefined, Any) ->
Any;
mount(MountPoint, Topic) when is_binary(Topic) ->
prefix(MountPoint, Topic);
mount(MountPoint, Msg = #message{topic = Topic}) ->
Msg#message{topic = prefix(MountPoint, Topic)};
mount(MountPoint, Topic) when ?IS_TOPIC(Topic) ->
prefix_maybe_share(MountPoint, Topic);
mount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
Msg#message{topic = prefix_maybe_share(MountPoint, Topic)};
mount(MountPoint, TopicFilters) when is_list(TopicFilters) ->
[{prefix(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
[{prefix_maybe_share(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
%% @private
-compile({inline, [prefix/2]}).
prefix(MountPoint, Topic) ->
<<MountPoint/binary, Topic/binary>>.
-spec prefix_maybe_share(maybe(mountpoint()), Any) -> Any when
Any ::
emqx_types:topic()
| emqx_types:share().
prefix_maybe_share(MountPoint, Topic) when
is_binary(MountPoint) andalso is_binary(Topic)
->
<<MountPoint/binary, Topic/binary>>;
prefix_maybe_share(MountPoint, #share{group = Group, topic = Topic}) when
is_binary(MountPoint) andalso is_binary(Topic)
->
#share{group = Group, topic = prefix_maybe_share(MountPoint, Topic)}.
-spec unmount(maybe(mountpoint()), Any) -> Any when
Any ::
emqx_types:topic()
| emqx_types:share()
| emqx_types:message().
unmount(undefined, Any) ->
Any;
unmount(MountPoint, Topic) when is_binary(Topic) ->
unmount(MountPoint, Topic) when ?IS_TOPIC(Topic) ->
unmount_maybe_share(MountPoint, Topic);
unmount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
Msg#message{topic = unmount_maybe_share(MountPoint, Topic)}.
unmount_maybe_share(MountPoint, Topic) when
is_binary(MountPoint) andalso is_binary(Topic)
->
case string:prefix(Topic, MountPoint) of
nomatch -> Topic;
Topic1 -> Topic1
end;
unmount(MountPoint, Msg = #message{topic = Topic}) ->
case string:prefix(Topic, MountPoint) of
nomatch -> Msg;
Topic1 -> Msg#message{topic = Topic1}
end.
unmount_maybe_share(MountPoint, TopicFilter = #share{topic = Topic}) when
is_binary(MountPoint) andalso is_binary(Topic)
->
TopicFilter#share{topic = unmount_maybe_share(MountPoint, Topic)}.
-spec replvar(maybe(mountpoint()), map()) -> maybe(mountpoint()).
replvar(undefined, _Vars) ->

15
apps/emqx/src/emqx_mqtt_caps.erl
View File

@ -102,16 +102,19 @@ do_check_pub(_Flags, _Caps) ->
-spec check_sub(
emqx_types:clientinfo(),
emqx_types:topic(),
emqx_types:topic() | emqx_types:share(),
emqx_types:subopts()
) ->
ok_or_error(emqx_types:reason_code()).
check_sub(ClientInfo = #{zone := Zone}, Topic, SubOpts) ->
Caps = emqx_config:get_zone_conf(Zone, [mqtt]),
Flags = #{
%% TODO: qos check
%% (max_qos_allowed, Map) ->
%% max_qos_allowed => maps:get(max_qos_allowed, Caps, 2),
topic_levels => emqx_topic:levels(Topic),
is_wildcard => emqx_topic:wildcard(Topic),
is_shared => maps:is_key(share, SubOpts),
is_shared => erlang:is_record(Topic, share),
is_exclusive => maps:get(is_exclusive, SubOpts, false)
},
do_check_sub(Flags, Caps, ClientInfo, Topic).
@ -126,13 +129,19 @@ do_check_sub(#{is_shared := true}, #{shared_subscription := false}, _, _) ->
{error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED};
do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := false}, _, _) ->
{error, ?RC_TOPIC_FILTER_INVALID};
do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) ->
do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) when
is_binary(Topic)
->
case emqx_exclusive_subscription:check_subscribe(ClientInfo, Topic) of
deny ->
{error, ?RC_QUOTA_EXCEEDED};
_ ->
ok
end;
%% for max_qos_allowed
%% see: RC_GRANTED_QOS_0, RC_GRANTED_QOS_1, RC_GRANTED_QOS_2
%% do_check_sub(_, _) ->
%% {ok, RC};
do_check_sub(_Flags, _Caps, _, _) ->
ok.

1
apps/emqx/src/emqx_reason_codes.erl
View File

@ -177,6 +177,7 @@ compat(connack, 16#9D) -> ?CONNACK_SERVER;
compat(connack, 16#9F) -> ?CONNACK_SERVER;
compat(suback, Code) when Code =< ?QOS_2 -> Code;
compat(suback, Code) when Code >= 16#80 -> 16#80;
%% TODO: 16#80(qos0) 16#81(qos1) 16#82(qos2) for mqtt-v3.1.1
compat(unsuback, _Code) -> undefined;
compat(_Other, _Code) -> undefined.

85
apps/emqx/src/emqx_schema_secret.erl Normal file
View File

@ -0,0 +1,85 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc HOCON schema that defines _secret_ concept.
-module(emqx_schema_secret).
-include_lib("emqx/include/logger.hrl").
-include_lib("typerefl/include/types.hrl").
-export([mk/1]).
%% HOCON Schema API
-export([convert_secret/2]).
%% @doc Secret value.
-type t() :: binary().
%% @doc Source of the secret value.
%% * "file://...": file path to a file containing secret value.
%% * other binaries: secret value itself.
-type source() :: iodata().
-type secret() :: binary() | function().
-reflect_type([secret/0]).
-define(SCHEMA, #{
required => false,
format => <<"password">>,
sensitive => true,
converter => fun ?MODULE:convert_secret/2
}).
-dialyzer({nowarn_function, source/1}).
%%
-spec mk(#{atom() => _}) -> hocon_schema:field_schema().
mk(Overrides = #{}) ->
hoconsc:mk(secret(), maps:merge(?SCHEMA, Overrides)).
convert_secret(undefined, #{}) ->
undefined;
convert_secret(Secret, #{make_serializable := true}) ->
unicode:characters_to_binary(source(Secret));
convert_secret(Secret, #{}) when is_function(Secret, 0) ->
Secret;
convert_secret(Secret, #{}) when is_integer(Secret) ->
wrap(integer_to_binary(Secret));
convert_secret(Secret, #{}) ->
try unicode:characters_to_binary(Secret) of
String when is_binary(String) ->
wrap(String);
{error, _, _} ->
throw(invalid_string)
catch
error:_ ->
throw(invalid_type)
end.
-spec wrap(source()) -> emqx_secret:t(t()).
wrap(<<"file://", Filename/binary>>) ->
emqx_secret:wrap_load({file, Filename});
wrap(Secret) ->
emqx_secret:wrap(Secret).
-spec source(emqx_secret:t(t())) -> source().
source(Secret) when is_function(Secret) ->
source(emqx_secret:term(Secret));
source({file, Filename}) ->
<<"file://", Filename/binary>>;
source(Secret) ->
Secret.

31
apps/emqx/src/emqx_secret.erl
View File

@ -19,23 +19,52 @@
-module(emqx_secret).
%% API:
-export([wrap/1, unwrap/1]).
-export([wrap/1, wrap_load/1, unwrap/1, term/1]).
-export_type([t/1]).
-opaque t(T) :: T | fun(() -> t(T)).
%% Secret loader module.
%% Any changes related to processing of secrets should be made there.
-define(LOADER, emqx_secret_loader).
%%================================================================================
%% API funcions
%%================================================================================
%% @doc Wrap a term in a secret closure.
%% This effectively hides the term from any term formatting / printing code.
-spec wrap(T) -> t(T).
wrap(Term) ->
fun() ->
Term
end.
%% @doc Wrap a loader function call over a term in a secret closure.
%% This is slightly more flexible form of `wrap/1` with the same basic purpose.
-spec wrap_load(emqx_secret_loader:source()) -> t(_).
wrap_load(Source) ->
fun() ->
apply(?LOADER, load, [Source])
end.
%% @doc Unwrap a secret closure, revealing the secret.
%% This is either `Term` or `Module:Function(Term)` depending on how it was wrapped.
-spec unwrap(t(T)) -> T.
unwrap(Term) when is_function(Term, 0) ->
%% Handle potentially nested funs
unwrap(Term());
unwrap(Term) ->
Term.
%% @doc Inspect the term wrapped in a secret closure.
-spec term(t(_)) -> _Term.
term(Wrap) when is_function(Wrap, 0) ->
case erlang:fun_info(Wrap, module) of
{module, ?MODULE} ->
{env, Env} = erlang:fun_info(Wrap, env),
lists:last(Env);
_ ->
error(badarg, [Wrap])
end.

42
apps/emqx/src/emqx_secret_loader.erl Normal file
View File

@ -0,0 +1,42 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_secret_loader).
%% API
-export([load/1]).
-export([file/1]).
-export_type([source/0]).
-type source() :: {file, file:filename_all()}.
-spec load(source()) -> binary() | no_return().
load({file, Filename}) ->
file(Filename).
-spec file(file:filename_all()) -> binary() | no_return().
file(Filename) ->
case file:read_file(Filename) of
{ok, Secret} ->
string:trim(Secret, trailing);
{error, Reason} ->
throw(#{
msg => failed_to_read_secret_file,
path => Filename,
reason => emqx_utils:explain_posix(Reason)
})
end.

12
apps/emqx/src/emqx_session.erl
View File

@ -267,7 +267,7 @@ destroy(Session) ->
-spec subscribe(
clientinfo(),
emqx_types:topic(),
emqx_types:topic() | emqx_types:share(),
emqx_types:subopts(),
t()
) ->
@ -287,7 +287,7 @@ subscribe(ClientInfo, TopicFilter, SubOpts, Session) ->
-spec unsubscribe(
clientinfo(),
emqx_types:topic(),
emqx_types:topic() | emqx_types:share(),
emqx_types:subopts(),
t()
) ->
@ -418,7 +418,13 @@ enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) ->
end.
enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) ->
SubOpts = ?IMPL(Session):get_subscription(Topic, Session),
SubOpts =
case Msg of
#message{headers = #{redispatch_to := ?REDISPATCH_TO(Group, T)}} ->
?IMPL(Session):get_subscription(emqx_topic:make_shared_record(Group, T), Session);
_ ->
?IMPL(Session):get_subscription(Topic, Session)
end,
enrich_message(ClientInfo, Msg, SubOpts, UpgradeQoS).
enrich_message(

2
apps/emqx/src/emqx_session_mem.erl
View File

@ -316,7 +316,7 @@ unsubscribe(
{error, ?RC_NO_SUBSCRIPTION_EXISTED}
end.
-spec get_subscription(emqx_types:topic(), session()) ->
-spec get_subscription(emqx_types:topic() | emqx_types:share(), session()) ->
emqx_types:subopts() | undefined.
get_subscription(Topic, #session{subscriptions = Subs}) ->
maps:get(Topic, Subs, undefined).

18
apps/emqx/src/emqx_shared_sub.erl
View File

@ -95,7 +95,6 @@
-define(ACK, shared_sub_ack).
-define(NACK(Reason), {shared_sub_nack, Reason}).
-define(NO_ACK, no_ack).
-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
-define(SUBSCRIBER_DOWN, noproc).
-type redispatch_to() :: ?REDISPATCH_TO(emqx_types:group(), emqx_types:topic()).
@ -234,19 +233,16 @@ without_group_ack(Msg) ->
get_group_ack(Msg) ->
emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK).
with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) ->
Msg;
%% always add `redispatch_to` header to the message
%% for QOS_0 msgs, redispatch_to is not needed and filtered out in is_redispatch_needed/1
with_redispatch_to(Msg, Group, Topic) ->
emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg).
%% @hidden Redispatch is needed only for the messages with redispatch_to header added.
is_redispatch_needed(#message{} = Msg) ->
case get_redispatch_to(Msg) of
?REDISPATCH_TO(_, _) ->
true;
_ ->
false
end.
%% @hidden Redispatch is needed only for the messages which not QOS_0
is_redispatch_needed(#message{qos = ?QOS_0}) ->
false;
is_redispatch_needed(#message{headers = #{redispatch_to := ?REDISPATCH_TO(_, _)}}) ->
true.
%% @doc Redispatch shared deliveries to other members in the group.
redispatch(Messages0) ->

94
apps/emqx/src/emqx_topic.erl
View File

@ -36,9 +36,16 @@
parse/2
]).
-export([
maybe_format_share/1,
get_shared_real_topic/1,
make_shared_record/2
]).
-type topic() :: emqx_types:topic().
-type word() :: emqx_types:word().
-type words() :: emqx_types:words().
-type share() :: emqx_types:share().
%% Guards
-define(MULTI_LEVEL_WILDCARD_NOT_LAST(C, REST),
@ -50,7 +57,9 @@
%%--------------------------------------------------------------------
%% @doc Is wildcard topic?
-spec wildcard(topic() | words()) -> true | false.
-spec wildcard(topic() | share() | words()) -> true | false.
wildcard(#share{topic = Topic}) when is_binary(Topic) ->
wildcard(Topic);
wildcard(Topic) when is_binary(Topic) ->
wildcard(words(Topic));
wildcard([]) ->
@ -64,7 +73,7 @@ wildcard([_H | T]) ->
%% @doc Match Topic name with filter.
-spec match(Name, Filter) -> boolean() when
Name :: topic() | words(),
Name :: topic() | share() | words(),
Filter :: topic() | words().
match(<<$$, _/binary>>, <<$+, _/binary>>) ->
false;
@ -72,6 +81,10 @@ match(<<$$, _/binary>>, <<$#, _/binary>>) ->
false;
match(Name, Filter) when is_binary(Name), is_binary(Filter) ->
match(words(Name), words(Filter));
match(#share{} = Name, Filter) ->
match_share(Name, Filter);
match(Name, #share{} = Filter) ->
match_share(Name, Filter);
match([], []) ->
true;
match([H | T1], [H | T2]) ->
@ -87,12 +100,29 @@ match([_H1 | _], []) ->
match([], [_H | _T2]) ->
false.
-spec match_share(Name, Filter) -> boolean() when
Name :: share(),
Filter :: topic() | share().
match_share(#share{topic = Name}, Filter) when is_binary(Filter) ->
%% only match real topic filter for normal topic filter.
match(words(Name), words(Filter));
match_share(#share{group = Group, topic = Name}, #share{group = Group, topic = Filter}) ->
%% Matching real topic filter When subed same share group.
match(words(Name), words(Filter));
match_share(#share{}, _) ->
%% Otherwise, non-matched.
false;
match_share(Name, #share{topic = Filter}) when is_binary(Name) ->
%% Only match real topic filter for normal topic_filter/topic_name.
match(Name, Filter).
-spec match_any(Name, [Filter]) -> boolean() when
Name :: topic() | words(),
Filter :: topic() | words().
match_any(Topic, Filters) ->
lists:any(fun(Filter) -> match(Topic, Filter) end, Filters).
%% TODO: validate share topic #share{} for emqx_trace.erl
%% @doc Validate topic name or filter
-spec validate(topic() | {name | filter, topic()}) -> true.
validate(Topic) when is_binary(Topic) ->
@ -107,7 +137,7 @@ validate(_, <<>>) ->
validate(_, Topic) when is_binary(Topic) andalso (size(Topic) > ?MAX_TOPIC_LEN) ->
%% MQTT-5.0 [MQTT-4.7.3-3]
error(topic_too_long);
validate(filter, SharedFilter = <<"$share/", _Rest/binary>>) ->
validate(filter, SharedFilter = <<?SHARE, "/", _Rest/binary>>) ->
validate_share(SharedFilter);
validate(filter, Filter) when is_binary(Filter) ->
validate2(words(Filter));
@ -139,12 +169,12 @@ validate3(<<C/utf8, _Rest/binary>>) when C == $#; C == $+; C == 0 ->
validate3(<<_/utf8, Rest/binary>>) ->
validate3(Rest).
validate_share(<<"$share/", Rest/binary>>) when
validate_share(<<?SHARE, "/", Rest/binary>>) when
Rest =:= <<>> orelse Rest =:= <<"/">>
->
%% MQTT-5.0 [MQTT-4.8.2-1]
error(?SHARE_EMPTY_FILTER);
validate_share(<<"$share/", Rest/binary>>) ->
validate_share(<<?SHARE, "/", Rest/binary>>) ->
case binary:split(Rest, <<"/">>) of
%% MQTT-5.0 [MQTT-4.8.2-1]
[<<>>, _] ->
@ -156,7 +186,7 @@ validate_share(<<"$share/", Rest/binary>>) ->
validate_share(ShareName, Filter)
end.
validate_share(_, <<"$share/", _Rest/binary>>) ->
validate_share(_, <<?SHARE, "/", _Rest/binary>>) ->
error(?SHARE_RECURSIVELY);
validate_share(ShareName, Filter) ->
case binary:match(ShareName, [<<"+">>, <<"#">>]) of
@ -185,7 +215,9 @@ bin('#') -> <<"#">>;
bin(B) when is_binary(B) -> B;
bin(L) when is_list(L) -> list_to_binary(L).
-spec levels(topic()) -> pos_integer().
-spec levels(topic() | share()) -> pos_integer().
levels(#share{topic = Topic}) when is_binary(Topic) ->
levels(Topic);
levels(Topic) when is_binary(Topic) ->
length(tokens(Topic)).
@ -197,6 +229,8 @@ tokens(Topic) ->
%% @doc Split Topic Path to Words
-spec words(topic()) -> words().
words(#share{topic = Topic}) when is_binary(Topic) ->
words(Topic);
words(Topic) when is_binary(Topic) ->
[word(W) || W <- tokens(Topic)].
@ -237,26 +271,29 @@ do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
do_join(TopicAcc, [Word | Words]) ->
do_join(<<TopicAcc/binary, "/", (bin(Word))/binary>>, Words).
-spec parse(topic() | {topic(), map()}) -> {topic(), #{share => binary()}}.
-spec parse(topic() | {topic(), map()}) -> {topic() | share(), map()}.
parse(TopicFilter) when is_binary(TopicFilter) ->
parse(TopicFilter, #{});
parse({TopicFilter, Options}) when is_binary(TopicFilter) ->
parse(TopicFilter, Options).
-spec parse(topic(), map()) -> {topic(), map()}.
parse(TopicFilter = <<"$queue/", _/binary>>, #{share := _Group}) ->
error({invalid_topic_filter, TopicFilter});
parse(TopicFilter = <<"$share/", _/binary>>, #{share := _Group}) ->
error({invalid_topic_filter, TopicFilter});
parse(<<"$queue/", TopicFilter/binary>>, Options) ->
parse(TopicFilter, Options#{share => <<"$queue">>});
parse(TopicFilter = <<"$share/", Rest/binary>>, Options) ->
-spec parse(topic() | share(), map()) -> {topic() | share(), map()}.
%% <<"$queue/[real_topic_filter]>">> equivalent to <<"$share/$queue/[real_topic_filter]">>
%% So the head of `real_topic_filter` MUST NOT be `<<$queue>>` or `<<$share>>`
parse(#share{topic = Topic = <<?QUEUE, "/", _/binary>>}, _Options) ->
error({invalid_topic_filter, Topic});
parse(#share{topic = Topic = <<?SHARE, "/", _/binary>>}, _Options) ->
error({invalid_topic_filter, Topic});
parse(<<?QUEUE, "/", Topic/binary>>, Options) ->
parse(#share{group = <<?QUEUE>>, topic = Topic}, Options);
parse(TopicFilter = <<?SHARE, "/", Rest/binary>>, Options) ->
case binary:split(Rest, <<"/">>) of
[_Any] ->
error({invalid_topic_filter, TopicFilter});
[ShareName, Filter] ->
case binary:match(ShareName, [<<"+">>, <<"#">>]) of
nomatch -> parse(Filter, Options#{share => ShareName});
%% `Group` could be `$share` or `$queue`
[Group, Topic] ->
case binary:match(Group, [<<"+">>, <<"#">>]) of
nomatch -> parse(#share{group = Group, topic = Topic}, Options);
_ -> error({invalid_topic_filter, TopicFilter})
end
end;
@ -267,5 +304,22 @@ parse(TopicFilter = <<"$exclusive/", Topic/binary>>, Options) ->
_ ->
{Topic, Options#{is_exclusive => true}}
end;
parse(TopicFilter, Options) ->
parse(TopicFilter, Options) when
?IS_TOPIC(TopicFilter)
->
{TopicFilter, Options}.
get_shared_real_topic(#share{topic = TopicFilter}) ->
TopicFilter;
get_shared_real_topic(TopicFilter) when is_binary(TopicFilter) ->
TopicFilter.
make_shared_record(Group, Topic) ->
#share{group = Group, topic = Topic}.
maybe_format_share(#share{group = <<?QUEUE>>, topic = Topic}) ->
join([<<?QUEUE>>, Topic]);
maybe_format_share(#share{group = Group, topic = Topic}) ->
join([<<?SHARE>>, Group, Topic]);
maybe_format_share(Topic) ->
join([Topic]).

2
apps/emqx/src/emqx_trace/emqx_trace.erl
View File

@ -105,7 +105,7 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
ignore ->
ignore;
Log ->
case logger_config:get(ets:whereis(logger), Id) of
case logger_config:get(logger, Id) of
{ok, #{module := Module} = HandlerConfig0} ->
HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
try

10
apps/emqx/src/emqx_types.erl
View File

@ -40,6 +40,10 @@
words/0
]).
-export_type([
share/0
]).
-export_type([
socktype/0,
sockstate/0,
@ -136,11 +140,14 @@
-type subid() :: binary() | atom().
-type group() :: binary() | undefined.
%% '_' for match spec
-type group() :: binary() | '_'.
-type topic() :: binary().
-type word() :: '' | '+' | '#' | binary().
-type words() :: list(word()).
-type share() :: #share{}.
-type socktype() :: tcp | udp | ssl | proxy | atom().
-type sockstate() :: idle | running | blocked | closed.
-type conninfo() :: #{
@ -207,7 +214,6 @@
rap := 0 | 1,
nl := 0 | 1,
qos := qos(),
share => binary(),
atom() => term()
}.
-type reason_code() :: 0..16#FF.

33
apps/emqx/test/emqx_broker_SUITE.erl
View File

@ -299,14 +299,19 @@ t_nosub_pub(Config) when is_list(Config) ->
?assertEqual(1, emqx_metrics:val('messages.dropped')).
t_shared_subscribe({init, Config}) ->
emqx_broker:subscribe(<<"topic">>, <<"clientid">>, #{share => <<"group">>}),
emqx_broker:subscribe(
emqx_topic:make_shared_record(<<"group">>, <<"topic">>), <<"clientid">>, #{}
),
ct:sleep(100),
Config;
t_shared_subscribe(Config) when is_list(Config) ->
emqx_broker:safe_publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)),
?assert(
receive
{deliver, <<"topic">>, #message{payload = <<"hello">>}} ->
{deliver, <<"topic">>, #message{
headers = #{redispatch_to := ?REDISPATCH_TO(<<"group">>, <<"topic">>)},
payload = <<"hello">>
}} ->
true;
Msg ->
ct:pal("Msg: ~p", [Msg]),
@ -316,7 +321,7 @@ t_shared_subscribe(Config) when is_list(Config) ->
end
);
t_shared_subscribe({'end', _Config}) ->
emqx_broker:unsubscribe(<<"$share/group/topic">>).
emqx_broker:unsubscribe(emqx_topic:make_shared_record(<<"group">>, <<"topic">>)).
t_shared_subscribe_2({init, Config}) ->
Config;
@ -723,24 +728,6 @@ t_connack_auth_error(Config) when is_list(Config) ->
?assertEqual(2, emqx_metrics:val('packets.connack.auth_error')),
ok.
t_handle_in_empty_client_subscribe_hook({init, Config}) ->
Hook = {?MODULE, client_subscribe_delete_all_hook, []},
ok = emqx_hooks:put('client.subscribe', Hook, _Priority = 100),
Config;
t_handle_in_empty_client_subscribe_hook({'end', _Config}) ->
emqx_hooks:del('client.subscribe', {?MODULE, client_subscribe_delete_all_hook}),
ok;
t_handle_in_empty_client_subscribe_hook(Config) when is_list(Config) ->
{ok, C} = emqtt:start_link(),
{ok, _} = emqtt:connect(C),
try
{ok, _, RCs} = emqtt:subscribe(C, <<"t">>),
?assertEqual([?RC_UNSPECIFIED_ERROR], RCs),
ok
after
emqtt:disconnect(C)
end.
authenticate_deny(_Credentials, _Default) ->
{stop, {error, bad_username_or_password}}.
@ -800,7 +787,3 @@ recv_msgs(Count, Msgs) ->
after 100 ->
Msgs
end.
client_subscribe_delete_all_hook(_ClientInfo, _Username, TopicFilter) ->
EmptyFilters = [{T, Opts#{deny_subscription => true}} || {T, Opts} <- TopicFilter],
{stop, EmptyFilters}.

11
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -456,7 +456,7 @@ t_process_subscribe(_) ->
ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end),
TopicFilters = [TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}],
{[{TopicFilter, ?RC_SUCCESS}], _Channel} =
emqx_channel:process_subscribe(TopicFilters, #{}, channel()).
emqx_channel:process_subscribe(TopicFilters, channel()).
t_process_unsubscribe(_) ->
ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end),
@ -914,7 +914,13 @@ t_check_pub_alias(_) ->
t_check_sub_authzs(_) ->
emqx_config:put_zone_conf(default, [authorization, enable], true),
TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS},
[{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()).
SubPkt = ?SUBSCRIBE_PACKET(1, #{}, [TopicFilter]),
CheckedSubPkt = ?SUBSCRIBE_PACKET(1, #{}, [{TopicFilter, ?RC_SUCCESS}]),
Channel = channel(),
?assertEqual(
{ok, CheckedSubPkt, Channel},
emqx_channel:check_sub_authzs(SubPkt, Channel)
).
t_enrich_connack_caps(_) ->
ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]),
@ -1061,6 +1067,7 @@ clientinfo(InitProps) ->
clientid => <<"clientid">>,
username => <<"username">>,
is_superuser => false,
is_bridge => false,
mountpoint => undefined
},
InitProps

6
apps/emqx/test/emqx_common_test_http.erl
View File

@ -34,6 +34,9 @@
-define(DEFAULT_APP_KEY, <<"default_app_key">>).
-define(DEFAULT_APP_SECRET, <<"default_app_secret">>).
%% from emqx_dashboard/include/emqx_dashboard_rbac.hrl
-define(ROLE_API_SUPERUSER, <<"administrator">>).
request_api(Method, Url, Auth) ->
request_api(Method, Url, [], Auth, []).
@ -96,7 +99,8 @@ create_default_app() ->
?DEFAULT_APP_SECRET,
true,
ExpiredAt,
<<"default app key for test">>
<<"default app key for test">>,
?ROLE_API_SUPERUSER
).
delete_default_app() ->

39
apps/emqx/test/emqx_mountpoint_SUITE.erl
View File

@ -29,6 +29,7 @@
).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("eunit/include/eunit.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
@ -52,6 +53,27 @@ t_mount(_) ->
mount(<<"device/1/">>, TopicFilters)
).
t_mount_share(_) ->
T = {TopicFilter, Opts} = emqx_topic:parse(<<"$share/group/topic">>),
TopicFilters = [T],
?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
%% should not mount share topic when make message.
Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>),
?assertEqual(
TopicFilter,
mount(undefined, TopicFilter)
),
?assertEqual(
#share{group = <<"group">>, topic = <<"device/1/topic">>},
mount(<<"device/1/">>, TopicFilter)
),
?assertEqual(
[{#share{group = <<"group">>, topic = <<"device/1/topic">>}, Opts}],
mount(<<"device/1/">>, TopicFilters)
).
t_unmount(_) ->
Msg = emqx_message:make(<<"clientid">>, <<"device/1/topic">>, <<"payload">>),
?assertEqual(<<"topic">>, unmount(undefined, <<"topic">>)),
@ -61,6 +83,23 @@ t_unmount(_) ->
?assertEqual(<<"device/1/topic">>, unmount(<<"device/2/">>, <<"device/1/topic">>)),
?assertEqual(Msg#message{topic = <<"device/1/topic">>}, unmount(<<"device/2/">>, Msg)).
t_unmount_share(_) ->
{TopicFilter, _Opts} = emqx_topic:parse(<<"$share/group/topic">>),
MountedTopicFilter = #share{group = <<"group">>, topic = <<"device/1/topic">>},
?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}),
%% should not unmount share topic when make message.
Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>),
?assertEqual(
TopicFilter,
unmount(undefined, TopicFilter)
),
?assertEqual(
#share{group = <<"group">>, topic = <<"topic">>},
unmount(<<"device/1/">>, MountedTopicFilter)
).
t_replvar(_) ->
?assertEqual(undefined, replvar(undefined, #{})),
?assertEqual(

4
apps/emqx/test/emqx_mqtt_caps_SUITE.erl
View File

@ -76,6 +76,8 @@ t_check_sub(_) ->
),
?assertEqual(
{error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED},
emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true})
emqx_mqtt_caps:check_sub(
ClientInfo, #share{group = <<"group">>, topic = <<"topic">>}, SubOpts
)
),
emqx_config:put([zones], OldConf).

8
apps/emqx/test/emqx_proper_types.erl
View File

@ -511,13 +511,7 @@ peercert() ->
conn_mod() ->
oneof([
emqx_connection,
emqx_ws_connection,
emqx_coap_mqtt_adapter,
emqx_sn_gateway,
emqx_lwm2m_protocol,
emqx_gbt32960_conn,
emqx_jt808_connection,
emqx_tcp_connection
emqx_ws_connection
]).
proto_name() ->

76
apps/emqx/test/emqx_secret_tests.erl Normal file
View File

@ -0,0 +1,76 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_secret_tests).
-include_lib("eunit/include/eunit.hrl").
wrap_unwrap_test() ->
?assertEqual(
42,
emqx_secret:unwrap(emqx_secret:wrap(42))
).
unwrap_immediate_test() ->
?assertEqual(
42,
emqx_secret:unwrap(42)
).
wrap_unwrap_load_test_() ->
Secret = <<"foobaz">>,
{
setup,
fun() -> write_temp_file(Secret) end,
fun(Filename) -> file:delete(Filename) end,
fun(Filename) ->
?_assertEqual(
Secret,
emqx_secret:unwrap(emqx_secret:wrap_load({file, Filename}))
)
end
}.
wrap_load_term_test() ->
?assertEqual(
{file, "no/such/file/i/swear"},
emqx_secret:term(emqx_secret:wrap_load({file, "no/such/file/i/swear"}))
).
wrap_unwrap_missing_file_test() ->
?assertThrow(
#{msg := failed_to_read_secret_file, reason := "No such file or directory"},
emqx_secret:unwrap(emqx_secret:wrap_load({file, "no/such/file/i/swear"}))
).
wrap_term_test() ->
?assertEqual(
42,
emqx_secret:term(emqx_secret:wrap(42))
).
external_fun_term_error_test() ->
Term = {foo, bar},
?assertError(
badarg,
emqx_secret:term(fun() -> Term end)
).
write_temp_file(Bytes) ->
Ts = erlang:system_time(millisecond),
Filename = filename:join("/tmp", ?MODULE_STRING ++ integer_to_list(-Ts)),
ok = file:write_file(Filename, Bytes),
Filename.

189
apps/emqx/test/emqx_shared_sub_SUITE.erl
View File

@ -137,7 +137,8 @@ t_random_basic(Config) when is_list(Config) ->
ClientId = <<"ClientId">>,
Topic = <<"foo">>,
Payload = <<"hello">>,
emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}),
Group = <<"group1">>,
emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}),
MsgQoS2 = emqx_message:make(ClientId, 2, Topic, Payload),
%% wait for the subscription to show up
ct:sleep(200),
@ -402,7 +403,7 @@ t_hash(Config) when is_list(Config) ->
ok = ensure_config(hash_clientid, false),
test_two_messages(hash_clientid).
t_hash_clinetid(Config) when is_list(Config) ->
t_hash_clientid(Config) when is_list(Config) ->
ok = ensure_config(hash_clientid, false),
test_two_messages(hash_clientid).
@ -528,14 +529,15 @@ last_message(ExpectedPayload, Pids, Timeout) ->
t_dispatch(Config) when is_list(Config) ->
ok = ensure_config(random),
Topic = <<"foo">>,
Group = <<"group1">>,
?assertEqual(
{error, no_subscribers},
emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}})
),
emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}),
emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}),
?assertEqual(
{ok, 1},
emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}})
).
t_uncovered_func(Config) when is_list(Config) ->
@ -991,37 +993,110 @@ t_session_kicked(Config) when is_list(Config) ->
?assertEqual([], collect_msgs(0)),
ok.
%% FIXME: currently doesn't work
%% t_different_groups_same_topic({init, Config}) ->
%% TestName = atom_to_binary(?FUNCTION_NAME),
%% ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
%% {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
%% {ok, _} = emqtt:connect(C),
%% [{client, C}, {clientid, ClientId} | Config];
%% t_different_groups_same_topic({'end', Config}) ->
%% C = ?config(client, Config),
%% emqtt:stop(C),
%% ok;
%% t_different_groups_same_topic(Config) when is_list(Config) ->
%% C = ?config(client, Config),
%% ClientId = ?config(clientid, Config),
%% %% Subscribe and unsubscribe to both $queue and $shared topics
%% Topic = <<"t/1">>,
%% SharedTopic0 = <<"$share/aa/", Topic/binary>>,
%% SharedTopic1 = <<"$share/bb/", Topic/binary>>,
%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic0, 2}),
%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic1, 2}),
-define(UPDATE_SUB_QOS(ConnPid, Topic, QoS),
?assertMatch({ok, _, [QoS]}, emqtt:subscribe(ConnPid, {Topic, QoS}))
).
%% Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
%% emqx:publish(Message0),
%% ?assertMatch([ {publish, #{payload := <<"hi">>}}
%% , {publish, #{payload := <<"hi">>}}
%% ], collect_msgs(5_000), #{routes => ets:tab2list(emqx_route)}),
t_different_groups_same_topic({init, Config}) ->
TestName = atom_to_binary(?FUNCTION_NAME),
ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
{ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(C),
[{client, C}, {clientid, ClientId} | Config];
t_different_groups_same_topic({'end', Config}) ->
C = ?config(client, Config),
emqtt:stop(C),
ok;
t_different_groups_same_topic(Config) when is_list(Config) ->
C = ?config(client, Config),
ClientId = ?config(clientid, Config),
%% Subscribe and unsubscribe to different group `aa` and `bb` with same topic
GroupA = <<"aa">>,
GroupB = <<"bb">>,
Topic = <<"t/1">>,
%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic0),
%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic1),
SharedTopicGroupA = ?SHARE(GroupA, Topic),
?UPDATE_SUB_QOS(C, SharedTopicGroupA, ?QOS_2),
SharedTopicGroupB = ?SHARE(GroupB, Topic),
?UPDATE_SUB_QOS(C, SharedTopicGroupB, ?QOS_2),
%% ok.
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
begin
?assertEqual(2, length(emqx_router:match_routes(Topic)))
end
),
Message0 = emqx_message:make(ClientId, ?QOS_2, Topic, <<"hi">>),
emqx:publish(Message0),
?assertMatch(
[
{publish, #{payload := <<"hi">>}},
{publish, #{payload := <<"hi">>}}
],
collect_msgs(5_000),
#{routes => ets:tab2list(emqx_route)}
),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB),
ok.
t_different_groups_update_subopts({init, Config}) ->
TestName = atom_to_binary(?FUNCTION_NAME),
ClientId = <<TestName/binary, (integer_to_binary(erlang:unique_integer()))/binary>>,
{ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(C),
[{client, C}, {clientid, ClientId} | Config];
t_different_groups_update_subopts({'end', Config}) ->
C = ?config(client, Config),
emqtt:stop(C),
ok;
t_different_groups_update_subopts(Config) when is_list(Config) ->
C = ?config(client, Config),
ClientId = ?config(clientid, Config),
%% Subscribe and unsubscribe to different group `aa` and `bb` with same topic
Topic = <<"t/1">>,
GroupA = <<"aa">>,
GroupB = <<"bb">>,
SharedTopicGroupA = ?SHARE(GroupA, Topic),
SharedTopicGroupB = ?SHARE(GroupB, Topic),
Fun = fun(Group, QoS) ->
?UPDATE_SUB_QOS(C, ?SHARE(Group, Topic), QoS),
?assertMatch(
#{qos := QoS},
emqx_broker:get_subopts(ClientId, emqx_topic:make_shared_record(Group, Topic))
)
end,
[Fun(Group, QoS) || QoS <- [?QOS_0, ?QOS_1, ?QOS_2], Group <- [GroupA, GroupB]],
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
begin
?assertEqual(2, length(emqx_router:match_routes(Topic)))
end
),
Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>),
emqx:publish(Message0),
?assertMatch(
[
{publish, #{payload := <<"hi">>}},
{publish, #{payload := <<"hi">>}}
],
collect_msgs(5_000),
#{routes => ets:tab2list(emqx_route)}
),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB),
ok.
t_queue_subscription({init, Config}) ->
TestName = atom_to_binary(?FUNCTION_NAME),
@ -1038,23 +1113,19 @@ t_queue_subscription({'end', Config}) ->
t_queue_subscription(Config) when is_list(Config) ->
C = ?config(client, Config),
ClientId = ?config(clientid, Config),
%% Subscribe and unsubscribe to both $queue and $shared topics
%% Subscribe and unsubscribe to both $queue share and $share/<group> with same topic
Topic = <<"t/1">>,
QueueTopic = <<"$queue/", Topic/binary>>,
SharedTopic = <<"$share/aa/", Topic/binary>>,
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {QueueTopic, 2}),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {SharedTopic, 2}),
%% FIXME: we should actually see 2 routes, one for each group
%% ($queue and aa), but currently the latest subscription
%% overwrites the existing one.
?UPDATE_SUB_QOS(C, QueueTopic, ?QOS_2),
?UPDATE_SUB_QOS(C, SharedTopic, ?QOS_2),
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
begin
ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
%% FIXME: should ensure we have 2 subscriptions
[_] = emqx_router:lookup_routes(Topic)
?assertEqual(2, length(emqx_router:match_routes(Topic)))
end
),
@ -1063,37 +1134,29 @@ t_queue_subscription(Config) when is_list(Config) ->
emqx:publish(Message0),
?assertMatch(
[
{publish, #{payload := <<"hi">>}},
{publish, #{payload := <<"hi">>}}
%% FIXME: should receive one message from each group
%% , {publish, #{payload := <<"hi">>}}
],
collect_msgs(5_000)
collect_msgs(5_000),
#{routes => ets:tab2list(emqx_route)}
),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, QueueTopic),
%% FIXME: return code should be success instead of 17 ("no_subscription_existed")
{ok, _, [?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(C, SharedTopic),
{ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopic),
%% FIXME: this should eventually be true, but currently we leak
%% the previous group subscription...
%% ?retry(
%% _Sleep0 = 100,
%% _Attempts0 = 50,
%% begin
%% ct:pal("routes: ~p", [ets:tab2list(emqx_route)]),
%% [] = emqx_router:lookup_routes(Topic)
%% end
%% ),
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
begin
?assertEqual(0, length(emqx_router:match_routes(Topic)))
end
),
ct:sleep(500),
Message1 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hello">>),
emqx:publish(Message1),
%% FIXME: we should *not* receive any messages...
%% ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}),
%% This is from the leaked group...
?assertMatch([{publish, #{topic := Topic}}], collect_msgs(1_000), #{
routes => ets:tab2list(emqx_route)
}),
%% we should *not* receive any messages.
?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}),
ok.

12
apps/emqx/test/emqx_topic_SUITE.erl
View File

@ -238,11 +238,11 @@ long_topic() ->
t_parse(_) ->
?assertError(
{invalid_topic_filter, <<"$queue/t">>},
parse(<<"$queue/t">>, #{share => <<"g">>})
parse(#share{group = <<"$queue">>, topic = <<"$queue/t">>}, #{})
),
?assertError(
{invalid_topic_filter, <<"$share/g/t">>},
parse(<<"$share/g/t">>, #{share => <<"g">>})
parse(#share{group = <<"g">>, topic = <<"$share/g/t">>}, #{})
),
?assertError(
{invalid_topic_filter, <<"$share/t">>},
@ -254,8 +254,12 @@ t_parse(_) ->
),
?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)),
?assertEqual({<<"a/b/+/#">>, #{qos => 1}}, parse({<<"a/b/+/#">>, #{qos => 1}})),
?assertEqual({<<"topic">>, #{share => <<"$queue">>}}, parse(<<"$queue/topic">>)),
?assertEqual({<<"topic">>, #{share => <<"group">>}}, parse(<<"$share/group/topic">>)),
?assertEqual(
{#share{group = <<"$queue">>, topic = <<"topic">>}, #{}}, parse(<<"$queue/topic">>)
),
?assertEqual(
{#share{group = <<"group">>, topic = <<"topic">>}, #{}}, parse(<<"$share/group/topic">>)
),
%% The '$local' and '$fastlane' topics have been deprecated.
?assertEqual({<<"$local/topic">>, #{}}, parse(<<"$local/topic">>)),
?assertEqual({<<"$local/$queue/topic">>, #{}}, parse(<<"$local/$queue/topic">>)),

3
apps/emqx/test/emqx_topic_index_SUITE.erl
View File

@ -209,9 +209,6 @@ t_match_fast_forward(Config) ->
M:insert(<<"a/b/1/2/3/4/5/6/7/8/9/#">>, id1, <<>>, Tab),
M:insert(<<"z/y/x/+/+">>, id2, <<>>, Tab),
M:insert(<<"a/b/c/+">>, id3, <<>>, Tab),
% dbg:tracer(),
% dbg:p(all, c),
% dbg:tpl({ets, next, '_'}, x),
?assertEqual(id1, id(match(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab))),
?assertEqual([id1], [id(X) || X <- matches(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab)]).

94
apps/emqx_audit/BSL.txt Normal file
View File

@ -0,0 +1,94 @@
Business Source License 1.1
Licensor: Hangzhou EMQ Technologies Co., Ltd.
Licensed Work: EMQX Enterprise Edition
The Licensed Work is (c) 2023
Hangzhou EMQ Technologies Co., Ltd.
Additional Use Grant: Students and educators are granted right to copy,
modify, and create derivative work for research
or education.
Change Date: 2027-02-01
Change License: Apache License, Version 2.0
For information about alternative licensing arrangements for the Software,
please contact Licensor: https://www.emqx.com/en/contact
Notice
The Business Source License (this document, or the “License”) is not an Open
Source license. However, the Licensed Work will eventually be made available
under an Open Source License, as stated in this License.
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
“Business Source License” is a trademark of MariaDB Corporation Ab.
-----------------------------------------------------------------------------
Business Source License 1.1
Terms
The Licensor hereby grants you the right to copy, modify, create derivative
works, redistribute, and make non-production use of the Licensed Work. The
Licensor may make an Additional Use Grant, above, permitting limited
production use.
Effective on the Change Date, or the fourth anniversary of the first publicly
available distribution of a specific version of the Licensed Work under this
License, whichever comes first, the Licensor hereby grants you rights under
the terms of the Change License, and the rights granted in the paragraph
above terminate.
If your use of the Licensed Work does not comply with the requirements
currently in effect as described in this License, you must purchase a
commercial license from the Licensor, its affiliated entities, or authorized
resellers, or you must refrain from using the Licensed Work.
All copies of the original and modified Licensed Work, and derivative works
of the Licensed Work, are subject to this License. This License applies
separately for each version of the Licensed Work and the Change Date may vary
for each version of the Licensed Work released by Licensor.
You must conspicuously display this License on each original or modified copy
of the Licensed Work. If you receive the Licensed Work in original or
modified form from a third party, the terms and conditions set forth in this
License apply to your use of that work.
Any use of the Licensed Work in violation of this License will automatically
terminate your rights under this License for the current and all other
versions of the Licensed Work.
This License does not grant you any right in any trademark or logo of
Licensor or its affiliates (provided that you may use a trademark or logo of
Licensor as expressly required by this License).
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
TITLE.
MariaDB hereby grants you permission to use this License’s text to license
your works, and to refer to it using the trademark “Business Source License”,
as long as you comply with the Covenants of Licensor below.
Covenants of Licensor
In consideration of the right to use this License’s text and the “Business
Source License” name and trademark, Licensor covenants to MariaDB, and to all
other recipients of the licensed work to be provided by Licensor:
1. To specify as the Change License the GPL Version 2.0 or any later version,
or a license that is compatible with GPL Version 2.0 or a later version,
where “compatible” means that software provided under the Change License can
be included in a program with software provided under GPL Version 2.0 or a
later version. Licensor may specify additional Change Licenses without
limitation.
2. To either: (a) specify an additional grant of rights to use that does not
impose any additional restriction on the right granted in this License, as
the Additional Use Grant; or (b) insert the text “None”.
3. To specify a Change Date.
4. Not to modify this License in any other way.

5
apps/emqx_audit/README.md Normal file
View File

@ -0,0 +1,5 @@
emqx_audit
=====
Audit log for EMQX, empowers users to efficiently access the desired audit trail data
and facilitates auditing, compliance, troubleshooting, and security analysis.

26
apps/emqx_audit/include/emqx_audit.hrl Normal file
View File

@ -0,0 +1,26 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-define(AUDIT, emqx_audit).
-record(?AUDIT, {
%% basic info
created_at,
node,
from,
source,
source_ip,
%% operation info
operation_id,
operation_type,
args,
operation_result,
failure,
%% request detail
http_method,
http_request,
http_status_code,
duration_ms,
extra
}).

5
apps/emqx_audit/rebar.config Normal file
View File

@ -0,0 +1,5 @@
{erl_opts, [debug_info]}.
{deps, [
{emqx, {path, "../emqx"}},
{emqx_utils, {path, "../emqx_utils"}}
]}.

10
apps/emqx_audit/src/emqx_audit.app.src Normal file
View File

@ -0,0 +1,10 @@
{application, emqx_audit, [
{description, "Audit log for EMQX"},
{vsn, "0.1.0"},
{registered, []},
{mod, {emqx_audit_app, []}},
{applications, [kernel, stdlib, emqx]},
{env, []},
{modules, []},
{links, []}
]}.

245
apps/emqx_audit/src/emqx_audit.erl Normal file
View File

@ -0,0 +1,245 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_audit).
-behaviour(gen_server).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl").
-include("emqx_audit.hrl").
%% API
-export([start_link/0]).
-export([log/3]).
-export([trans_clean_expired/2]).
%% gen_server callbacks
-export([
init/1,
handle_continue/2,
handle_call/3,
handle_cast/2,
handle_info/2,
terminate/2,
code_change/3
]).
-define(FILTER_REQ, [cert, host_info, has_sent_resp, pid, path_info, peer, ref, sock, streamid]).
-ifdef(TEST).
-define(INTERVAL, 100).
-else.
-define(INTERVAL, 10000).
-endif.
to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) ->
#?AUDIT{
operation_id = <<"">>,
operation_type = atom_to_binary(Cmd),
args = Args,
operation_result = <<"">>,
failure = <<"">>,
duration_ms = DurationMs,
from = cli,
source = <<"">>,
source_ip = <<"">>,
http_status_code = <<"">>,
http_method = <<"">>,
http_request = <<"">>
};
to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api ->
#{
source := Source,
source_ip := SourceIp,
%% operation info
operation_id := OperationId,
operation_type := OperationType,
operation_result := OperationResult,
%% request detail
http_status_code := StatusCode,
http_method := Method,
http_request := Request,
duration_ms := DurationMs
} = Log,
#?AUDIT{
from = From,
source = Source,
source_ip = SourceIp,
%% operation info
operation_id = OperationId,
operation_type = OperationType,
operation_result = OperationResult,
failure = maps:get(failure, Log, <<"">>),
%% request detail
http_status_code = StatusCode,
http_method = Method,
http_request = Request,
duration_ms = DurationMs,
args = <<"">>
};
to_audit(#{from := erlang_console, function := F, args := Args}) ->
#?AUDIT{
from = erlang_console,
source = <<"">>,
source_ip = <<"">>,
%% operation info
operation_id = <<"">>,
operation_type = <<"">>,
operation_result = <<"">>,
failure = <<"">>,
%% request detail
http_status_code = <<"">>,
http_method = <<"">>,
http_request = <<"">>,
duration_ms = 0,
args = iolist_to_binary(io_lib:format("~p: ~p~n", [F, Args]))
}.
log(_Level, undefined, _Handler) ->
ok;
log(Level, Meta1, Handler) ->
Meta2 = Meta1#{time => logger:timestamp(), level => Level},
log_to_file(Level, Meta2, Handler),
log_to_db(Meta2),
remove_handler_when_disabled().
remove_handler_when_disabled() ->
case emqx_config:get([log, audit, enable], false) of
true ->
ok;
false ->
_ = logger:remove_handler(?AUDIT_HANDLER),
ok
end.
log_to_db(Log) ->
Audit0 = to_audit(Log),
Audit = Audit0#?AUDIT{
node = node(),
created_at = erlang:system_time(microsecond)
},
mria:dirty_write(?AUDIT, Audit).
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
init([]) ->
ok = mria:create_table(?AUDIT, [
{type, ordered_set},
{rlog_shard, ?COMMON_SHARD},
{storage, disc_copies},
{record_name, ?AUDIT},
{attributes, record_info(fields, ?AUDIT)}
]),
{ok, #{}, {continue, setup}}.
handle_continue(setup, State) ->
ok = mria:wait_for_tables([?AUDIT]),
NewState = State#{role => mria_rlog:role()},
?AUDIT(alert, #{
cmd => emqx,
args => ["start"],
version => emqx_release:version(),
from => cli,
duration_ms => 0
}),
{noreply, NewState, interval(NewState)}.
handle_call(_Request, _From, State) ->
{reply, ignore, State, interval(State)}.
handle_cast(_Request, State) ->
{noreply, State, interval(State)}.
handle_info(timeout, State) ->
ExtraWait = clean_expired_logs(),
{noreply, State, interval(State) + ExtraWait};
handle_info(_Info, State) ->
{noreply, State, interval(State)}.
terminate(_Reason, _State) ->
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
%%%===================================================================
%%% Internal functions
%%%===================================================================
%% if clean_expired transaction aborted, it will be scheduled with extra 60 seconds.
clean_expired_logs() ->
MaxSize = max_size(),
Oldest = mnesia:dirty_first(?AUDIT),
CurSize = mnesia:table_info(?AUDIT, size),
case CurSize - MaxSize of
DelSize when DelSize > 0 ->
case
mria:transaction(
?COMMON_SHARD,
fun ?MODULE:trans_clean_expired/2,
[Oldest, DelSize]
)
of
{atomic, ok} ->
0;
{aborted, Reason} ->
?SLOG(error, #{
msg => "clean_expired_audit_aborted",
reason => Reason,
delete_size => DelSize,
current_size => CurSize,
max_count => MaxSize
}),
60000
end;
_ ->
0
end.
trans_clean_expired(Oldest, DelCount) ->
First = mnesia:first(?AUDIT),
%% Other node already clean from the oldest record.
%% ensure not delete twice, otherwise records that should not be deleted will be deleted.
case First =:= Oldest of
true -> do_clean_expired(First, DelCount);
false -> ok
end.
do_clean_expired(_, DelSize) when DelSize =< 0 -> ok;
do_clean_expired('$end_of_table', _DelSize) ->
ok;
do_clean_expired(CurKey, DeleteSize) ->
mnesia:delete(?AUDIT, CurKey, sticky_write),
do_clean_expired(mnesia:next(?AUDIT, CurKey), DeleteSize - 1).
max_size() ->
emqx_conf:get([log, audit, max_filter_size], 5000).
interval(#{role := replicant}) -> hibernate;
interval(#{role := core}) -> ?INTERVAL + rand:uniform(?INTERVAL).
log_to_file(Level, Meta, #{module := Module} = Handler) ->
Log = #{level => Level, meta => Meta, msg => undefined},
Handler1 = maps:without(?OWN_KEYS, Handler),
try
erlang:apply(Module, log, [Log, Handler1])
catch
C:R:S ->
case logger:remove_handler(?AUDIT_HANDLER) of
ok ->
logger:internal_log(
error, {removed_failing_handler, ?AUDIT_HANDLER, C, R, S}
);
{error, {not_found, _}} ->
ok;
{error, Reason} ->
logger:internal_log(
error,
{removed_handler_failed, ?AUDIT_HANDLER, Reason, C, R, S}
)
end
end.

398
apps/emqx_audit/src/emqx_audit_api.erl Normal file
View File

@ -0,0 +1,398 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_audit_api).
-behaviour(minirest_api).
%% API
-export([api_spec/0, paths/0, schema/1, namespace/0, fields/1]).
-export([audit/2]).
-export([qs2ms/2, format/1]).
-include_lib("emqx/include/logger.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-include("emqx_audit.hrl").
-import(hoconsc, [mk/2, ref/2, array/1]).
-define(TAGS, ["Audit"]).
-define(AUDIT_QS_SCHEMA, [
{<<"node">>, atom},
{<<"from">>, atom},
{<<"source">>, binary},
{<<"source_ip">>, binary},
{<<"operation_id">>, binary},
{<<"operation_type">>, binary},
{<<"operation_result">>, atom},
{<<"http_status_code">>, integer},
{<<"http_method">>, atom},
{<<"gte_created_at">>, timestamp},
{<<"lte_created_at">>, timestamp},
{<<"gte_duration_ms">>, timestamp},
{<<"lte_duration_ms">>, timestamp}
]).
-define(DISABLE_MSG, <<"Audit is disabled">>).
namespace() -> "audit".
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
paths() ->
["/audit"].
schema("/audit") ->
#{
'operationId' => audit,
get => #{
tags => ?TAGS,
description => ?DESC(audit_get),
parameters => [
{node,
?HOCON(binary(), #{
in => query,
required => false,
example => <<"emqx@127.0.0.1">>,
desc => ?DESC(filter_node)
})},
{from,
?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{
in => query,
required => false,
example => <<"dashboard">>,
desc => ?DESC(filter_from)
})},
{source,
?HOCON(binary(), #{
in => query,
required => false,
example => <<"admin">>,
desc => ?DESC(filter_source)
})},
{source_ip,
?HOCON(binary(), #{
in => query,
required => false,
example => <<"127.0.0.1">>,
desc => ?DESC(filter_source_ip)
})},
{operation_id,
?HOCON(binary(), #{
in => query,
required => false,
example => <<"/rules/{id}">>,
desc => ?DESC(filter_operation_id)
})},
{operation_type,
?HOCON(binary(), #{
in => query,
example => <<"rules">>,
required => false,
desc => ?DESC(filter_operation_type)
})},
{operation_result,
?HOCON(?ENUM([success, failure]), #{
in => query,
example => failure,
required => false,
desc => ?DESC(filter_operation_result)
})},
{http_status_code,
?HOCON(integer(), #{
in => query,
example => 200,
required => false,
desc => ?DESC(filter_http_status_code)
})},
{http_method,
?HOCON(?ENUM([post, put, delete]), #{
in => query,
example => post,
required => false,
desc => ?DESC(filter_http_method)
})},
{gte_duration_ms,
?HOCON(integer(), #{
in => query,
example => 0,
required => false,
desc => ?DESC(filter_gte_duration_ms)
})},
{lte_duration_ms,
?HOCON(integer(), #{
in => query,
example => 1000,
required => false,
desc => ?DESC(filter_lte_duration_ms)
})},
{gte_created_at,
?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
in => query,
required => false,
example => <<"2023-10-15T00:00:00.820384+08:00">>,
desc => ?DESC(filter_gte_created_at)
})},
{lte_created_at,
?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
in => query,
example => <<"2023-10-16T00:00:00.820384+08:00">>,
required => false,
desc => ?DESC(filter_lte_created_at)
})},
ref(emqx_dashboard_swagger, page),
ref(emqx_dashboard_swagger, limit)
],
summary => <<"List audit logs">>,
responses => #{
200 =>
emqx_dashboard_swagger:schema_with_example(
array(?REF(audit_list)),
audit_log_list_example()
),
400 => emqx_dashboard_swagger:error_codes(
['BAD_REQUEST'],
?DISABLE_MSG
)
}
}
}.
fields(audit_list) ->
[
{data, mk(array(?REF(audit)), #{desc => ?DESC("audit_resp")})},
{meta, mk(ref(emqx_dashboard_swagger, meta), #{})}
];
fields(audit) ->
[
{created_at,
?HOCON(
emqx_utils_calendar:epoch_millisecond(),
#{
desc => "The time when the log is created"
}
)},
{node,
?HOCON(binary(), #{
desc => "The node name to which the log is created"
})},
{from,
?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{
desc => "The source type of the log"
})},
{source,
?HOCON(binary(), #{
desc => "The source of the log"
})},
{source_ip,
?HOCON(binary(), #{
desc => "The source ip of the log"
})},
{operation_id,
?HOCON(binary(), #{
desc => "The operation id of the log"
})},
{operation_type,
?HOCON(binary(), #{
desc => "The operation type of the log"
})},
{operation_result,
?HOCON(?ENUM([success, failure]), #{
desc => "The operation result of the log"
})},
{http_status_code,
?HOCON(integer(), #{
desc => "The http status code of the log"
})},
{http_method,
?HOCON(?ENUM([post, put, delete]), #{
desc => "The http method of the log"
})},
{duration_ms,
?HOCON(integer(), #{
desc => "The duration of the log"
})},
{args,
?HOCON(?ARRAY(binary()), #{
desc => "The args of the log"
})},
{failure,
?HOCON(?ARRAY(binary()), #{
desc => "The failure of the log"
})},
{http_request,
?HOCON(?REF(http_request), #{
desc => "The http request of the log"
})}
];
fields(http_request) ->
[
{bindings, ?HOCON(map(), #{})},
{body, ?HOCON(map(), #{})},
{headers, ?HOCON(map(), #{})},
{method, ?HOCON(?ENUM([post, put, delete]), #{})}
].
audit(get, #{query_string := QueryString}) ->
case emqx_config:get([log, audit, enable], false) of
false ->
{400, #{code => 'BAD_REQUEST', message => ?DISABLE_MSG}};
true ->
case
emqx_mgmt_api:node_query(
node(),
?AUDIT,
QueryString,
?AUDIT_QS_SCHEMA,
fun ?MODULE:qs2ms/2,
fun ?MODULE:format/1
)
of
{error, page_limit_invalid} ->
{400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}};
{error, Node, Error} ->
Message = list_to_binary(
io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])
),
{500, #{code => <<"NODE_DOWN">>, message => Message}};
Result ->
{200, Result}
end
end.
qs2ms(_Tab, {Qs, _}) ->
#{
match_spec => gen_match_spec(Qs, #?AUDIT{_ = '_'}, []),
fuzzy_fun => undefined
}.
gen_match_spec([], Audit, Conn) ->
[{Audit, Conn, ['$_']}];
gen_match_spec([{node, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{node = T}, Conn);
gen_match_spec([{from, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{from = T}, Conn);
gen_match_spec([{source, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{source = T}, Conn);
gen_match_spec([{source_ip, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{source_ip = T}, Conn);
gen_match_spec([{operation_id, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{operation_id = T}, Conn);
gen_match_spec([{operation_type, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{operation_type = T}, Conn);
gen_match_spec([{operation_result, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{operation_result = T}, Conn);
gen_match_spec([{http_status_code, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{http_status_code = T}, Conn);
gen_match_spec([{http_method, '=:=', T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{http_method = T}, Conn);
gen_match_spec([{created_at, Hold, T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [{'$1', Hold, T} | Conn]);
gen_match_spec([{created_at, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [
{'$1', Hold1, T1}, {'$1', Hold2, T2} | Conn
]);
gen_match_spec([{duration_ms, Hold, T} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [{'$2', Hold, T} | Conn]);
gen_match_spec([{duration_ms, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) ->
gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [
{'$2', Hold1, T1}, {'$2', Hold2, T2} | Conn
]).
format(Audit) ->
#?AUDIT{
created_at = CreatedAt,
node = Node,
from = From,
source = Source,
source_ip = SourceIp,
operation_id = OperationId,
operation_type = OperationType,
operation_result = OperationResult,
http_status_code = HttpStatusCode,
http_method = HttpMethod,
duration_ms = DurationMs,
args = Args,
failure = Failure,
http_request = HttpRequest
} = Audit,
#{
created_at => emqx_utils_calendar:epoch_to_rfc3339(CreatedAt, microsecond),
node => Node,
from => From,
source => Source,
source_ip => SourceIp,
operation_id => OperationId,
operation_type => OperationType,
operation_result => OperationResult,
http_status_code => HttpStatusCode,
http_method => HttpMethod,
duration_ms => DurationMs,
args => Args,
failure => Failure,
http_request => HttpRequest
}.
audit_log_list_example() ->
#{
data => [api_example(), cli_example()],
meta => #{
<<"count">> => 2,
<<"hasnext">> => false,
<<"limit">> => 50,
<<"page">> => 1
}
}.
api_example() ->
#{
<<"args">> => "",
<<"created_at">> => "2023-10-17T10:41:20.383993+08:00",
<<"duration_ms">> => 0,
<<"failure">> => "",
<<"from">> => "dashboard",
<<"http_method">> => "post",
<<"http_request">> => #{
<<"bindings">> => #{},
<<"body">> => #{
<<"password">> => "******",
<<"username">> => "admin"
},
<<"headers">> => #{
<<"accept">> => "*/*",
<<"authorization">> => "******",
<<"connection">> => "keep-alive",
<<"content-length">> => "45",
<<"content-type">> => "application/json"
},
<<"method">> => "post"
},
<<"http_status_code">> => 200,
<<"node">> => "emqx@127.0.0.1",
<<"operation_id">> => "/login",
<<"operation_result">> => "success",
<<"operation_type">> => "login",
<<"source">> => "admin",
<<"source_ip">> => "127.0.0.1"
}.
cli_example() ->
#{
<<"args">> => [<<"show">>, <<"log">>],
<<"created_at">> => "2023-10-17T10:45:13.100426+08:00",
<<"duration_ms">> => 7,
<<"failure">> => "",
<<"from">> => "cli",
<<"http_method">> => "",
<<"http_request">> => "",
<<"http_status_code">> => "",
<<"node">> => "emqx@127.0.0.1",
<<"operation_id">> => "",
<<"operation_result">> => "",
<<"operation_type">> => "conf",
<<"source">> => "",
<<"source_ip">> => ""
}.

15
apps/emqx_audit/src/emqx_audit_app.erl Normal file
View File

@ -0,0 +1,15 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_audit_app).
-behaviour(application).
-export([start/2, stop/1]).
start(_StartType, _StartArgs) ->
emqx_audit_sup:start_link().
stop(_State) ->
ok.

33
apps/emqx_audit/src/emqx_audit_sup.erl Normal file
View File

@ -0,0 +1,33 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_audit_sup).
-behaviour(supervisor).
-export([start_link/0]).
-export([init/1]).
-define(SERVER, ?MODULE).
start_link() ->
supervisor:start_link({local, ?SERVER}, ?MODULE, []).
init([]) ->
SupFlags = #{
strategy => one_for_all,
intensity => 10,
period => 10
},
ChildSpecs = [
#{
id => emqx_audit,
start => {emqx_audit, start_link, []},
type => worker,
restart => transient,
shutdown => 1000
}
],
{ok, {SupFlags, ChildSpecs}}.

248
apps/emqx_audit/test/emqx_audit_api_SUITE.erl Normal file
View File

@ -0,0 +1,248 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_audit_api_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
all() ->
[
{group, audit, [sequence]}
].
groups() ->
[
{audit, [sequence], common_tests()}
].
common_tests() ->
emqx_common_test_helpers:all(?MODULE).
-define(CONF_DEFAULT, #{
node =>
#{
name => "emqx1@127.0.0.1",
cookie => "emqxsecretcookie",
data_dir => "data"
},
log => #{
audit =>
#{
enable => true,
ignore_high_frequency_request => true,
level => info,
max_filter_size => 15,
rotation_count => 2,
rotation_size => "10MB",
time_offset => "system"
}
}
}).
init_per_suite(Config) ->
_ = application:load(emqx_conf),
emqx_config:erase_all(),
emqx_mgmt_api_test_util:init_suite([emqx_ctl, emqx_conf, emqx_audit]),
ok = emqx_common_test_helpers:load_config(emqx_enterprise_schema, ?CONF_DEFAULT),
emqx_config:save_schema_mod_and_names(emqx_enterprise_schema),
ok = emqx_config_logger:refresh_config(),
application:set_env(emqx, boot_modules, []),
emqx_conf_cli:load(),
Config.
end_per_suite(_) ->
emqx_mgmt_api_test_util:end_suite([emqx_audit, emqx_conf, emqx_ctl]).
t_http_api(_) ->
process_flag(trap_exit, true),
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
{ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(),
NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1),
{ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones),
?assertMatch(#{<<"max_qos_allowed">> := 1}, Res),
{ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
?assertMatch(
#{
<<"data">> := [
#{
<<"from">> := <<"rest_api">>,
<<"operation_id">> := <<"/configs/global_zone">>,
<<"source_ip">> := <<"127.0.0.1">>,
<<"source">> := _,
<<"http_request">> := #{
<<"method">> := <<"put">>,
<<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}},
<<"bindings">> := _,
<<"headers">> := #{<<"authorization">> := <<"******">>}
},
<<"http_status_code">> := 200,
<<"operation_result">> := <<"success">>,
<<"operation_type">> := <<"configs">>
}
]
},
emqx_utils_json:decode(Res1, [return_maps])
),
ok.
t_disabled(_) ->
Enable = [log, audit, enable],
?assertEqual(true, emqx:get_config(Enable)),
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
{ok, _} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
Size1 = mnesia:table_info(emqx_audit, size),
{ok, Logs} = emqx_mgmt_api_configs_SUITE:get_config("log"),
Logs1 = emqx_utils_maps:deep_put([<<"audit">>, <<"max_filter_size">>], Logs, 100),
NewLogs = emqx_utils_maps:deep_put([<<"audit">>, <<"enable">>], Logs1, false),
{ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", NewLogs),
{ok, GetLog1} = emqx_mgmt_api_configs_SUITE:get_config("log"),
?assertEqual(NewLogs, GetLog1),
?assertMatch(
{error, _},
emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader)
),
Size2 = mnesia:table_info(emqx_audit, size),
%% Record the audit disable action, so the size + 1
?assertEqual(Size1 + 1, Size2),
{ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(),
NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_topic_levels">>], Zones, 111),
{ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones),
?assertMatch(#{<<"max_topic_levels">> := 111}, Res),
Size3 = mnesia:table_info(emqx_audit, size),
%% Don't record mqtt update request.
?assertEqual(Size2, Size3),
%% enabled again
{ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", Logs1),
{ok, GetLog2} = emqx_mgmt_api_configs_SUITE:get_config("log"),
?assertEqual(Logs1, GetLog2),
Size4 = mnesia:table_info(emqx_audit, size),
?assertEqual(Size3 + 1, Size4),
ok.
t_cli(_Config) ->
ok = emqx_ctl:run_command(["conf", "show", "log"]),
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
{ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader),
#{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]),
?assertMatch(
[
#{
<<"from">> := <<"cli">>,
<<"operation_id">> := <<"">>,
<<"source_ip">> := <<"">>,
<<"operation_type">> := <<"conf">>,
<<"args">> := [<<"show">>, <<"log">>],
<<"node">> := _,
<<"source">> := <<"">>,
<<"http_request">> := <<"">>
}
],
Data
),
%% check filter
{ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader),
#{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]),
?assertEqual(Data, Data1),
{ok, Res2} = emqx_mgmt_api_test_util:request_api(
get, AuditPath, "from=erlang_console", AuthHeader
),
?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])),
ok.
t_max_size(_Config) ->
{ok, _} = emqx:update_config([log, audit, max_filter_size], 1000),
SizeFun =
fun() ->
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
Limit = "limit=1000",
{ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, Limit, AuthHeader),
#{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]),
erlang:length(Data)
end,
InitSize = SizeFun(),
lists:foreach(
fun(_) ->
ok = emqx_ctl:run_command(["conf", "show", "log"])
end,
lists:duplicate(100, 1)
),
timer:sleep(110),
Size1 = SizeFun(),
?assert(Size1 - InitSize >= 100, {Size1, InitSize}),
{ok, _} = emqx:update_config([log, audit, max_filter_size], 10),
%% wait for clean_expired
timer:sleep(250),
ExpectSize = emqx:get_config([log, audit, max_filter_size]),
Size2 = SizeFun(),
?assertEqual(ExpectSize, Size2, {sys:get_state(emqx_audit)}),
ok.
t_kickout_clients_without_log(_) ->
process_flag(trap_exit, true),
AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
{ok, AuditLogs1} = emqx_mgmt_api_test_util:request_api(get, AuditPath),
kickout_clients(),
{ok, AuditLogs2} = emqx_mgmt_api_test_util:request_api(get, AuditPath),
?assertEqual(AuditLogs1, AuditLogs2),
ok.
kickout_clients() ->
ClientId1 = <<"client1">>,
ClientId2 = <<"client2">>,
ClientId3 = <<"client3">>,
{ok, C1} = emqtt:start_link(#{
clientid => ClientId1,
proto_ver => v5,
properties => #{'Session-Expiry-Interval' => 120}
}),
{ok, _} = emqtt:connect(C1),
{ok, C2} = emqtt:start_link(#{clientid => ClientId2}),
{ok, _} = emqtt:connect(C2),
{ok, C3} = emqtt:start_link(#{clientid => ClientId3}),
{ok, _} = emqtt:connect(C3),
timer:sleep(300),
%% get /clients
ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]),
{ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]),
ClientsMeta = maps:get(<<"meta">>, ClientsResponse),
ClientsPage = maps:get(<<"page">>, ClientsMeta),
ClientsLimit = maps:get(<<"limit">>, ClientsMeta),
ClientsCount = maps:get(<<"count">>, ClientsMeta),
?assertEqual(ClientsPage, 1),
?assertEqual(ClientsLimit, emqx_mgmt:default_row_limit()),
?assertEqual(ClientsCount, 3),
%% kickout clients
KickoutPath = emqx_mgmt_api_test_util:api_path(["clients", "kickout", "bulk"]),
KickoutBody = [ClientId1, ClientId2, ClientId3],
{ok, 204, _} = emqx_mgmt_api_test_util:request_api_with_body(post, KickoutPath, KickoutBody),
{ok, Clients2} = emqx_mgmt_api_test_util:request_api(get, ClientsPath),
ClientsResponse2 = emqx_utils_json:decode(Clients2, [return_maps]),
?assertMatch(#{<<"data">> := []}, ClientsResponse2).

4
apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl
View File

@ -26,10 +26,6 @@
-define(AUTHN_BACKEND, ldap).
-define(AUTHN_BACKEND_BIN, <<"ldap">>).
-define(AUTHN_BACKEND_BIND, ldap_bind).
-define(AUTHN_BACKEND_BIND_BIN, <<"ldap_bind">>).
-define(AUTHN_TYPE, {?AUTHN_MECHANISM, ?AUTHN_BACKEND}).
-define(AUTHN_TYPE_BIND, {?AUTHN_MECHANISM, ?AUTHN_BACKEND_BIND}).
-endif.

2
apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl
View File

@ -25,12 +25,10 @@
start(_StartType, _StartArgs) ->
ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_ldap),
ok = emqx_authn:register_provider(?AUTHN_TYPE, emqx_authn_ldap),
ok = emqx_authn:register_provider(?AUTHN_TYPE_BIND, emqx_authn_ldap_bind),
{ok, Sup} = emqx_auth_ldap_sup:start_link(),
{ok, Sup}.
stop(_State) ->
ok = emqx_authn:deregister_provider(?AUTHN_TYPE),
ok = emqx_authn:deregister_provider(?AUTHN_TYPE_BIND),
ok = emqx_authz:unregister_source(?AUTHZ_TYPE),
ok.

185
apps/emqx_auth_ldap/src/emqx_authn_ldap.erl
View File

@ -16,19 +16,10 @@
-module(emqx_authn_ldap).
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("eldap/include/eldap.hrl").
-behaviour(emqx_authn_provider).
%% a compatible attribute for version 4.x
-define(ISENABLED_ATTR, "isEnabled").
-define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]).
%% TODO
%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512
%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112
-export([
create/2,
update/2,
@ -69,163 +60,25 @@ authenticate(#{auth_method := _}, _) ->
ignore;
authenticate(#{password := undefined}, _) ->
{error, bad_username_or_password};
authenticate(
#{password := Password} = Credential,
#{
password_attribute := PasswordAttr,
is_superuser_attribute := IsSuperuserAttr,
query_timeout := Timeout,
resource_id := ResourceId
} = State
) ->
case
emqx_resource:simple_sync_query(
ResourceId,
{query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout}
)
of
{ok, []} ->
ignore;
{ok, [Entry]} ->
is_enabled(Password, Entry, State);
{error, Reason} ->
?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{
resource => ResourceId,
timeout => Timeout,
reason => Reason
}),
ignore
authenticate(Credential, #{method := #{type := Type}} = State) ->
case Type of
hash ->
emqx_authn_ldap_hash:authenticate(Credential, State);
bind ->
emqx_authn_ldap_bind:authenticate(Credential, State)
end.
%% it used the deprecated config form
parse_config(
#{password_attribute := PasswordAttr, is_superuser_attribute := IsSuperuserAttr} = Config0
) ->
Config = maps:without([password_attribute, is_superuser_attribute], Config0),
parse_config(Config#{
method => #{
type => hash,
password_attribute => PasswordAttr,
is_superuser_attribute => IsSuperuserAttr
}
});
parse_config(Config) ->
maps:with([query_timeout, password_attribute, is_superuser_attribute], Config).
%% To compatible v4.x
is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) ->
IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"),
case emqx_authn_utils:to_bool(IsEnabled) of
true ->
ensure_password(Password, Entry, State);
_ ->
{error, user_disabled}
end.
ensure_password(
Password,
#eldap_entry{attributes = Attributes} = Entry,
#{password_attribute := PasswordAttr} = State
) ->
case get_value(PasswordAttr, Attributes) of
undefined ->
{error, no_password};
[LDAPPassword | _] ->
extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State)
end.
%% RFC 2307 format password
%% https://datatracker.ietf.org/doc/html/rfc2307
extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) ->
case
re:run(
LDAPPassword,
"{([^{}]+)}(.+)",
[{capture, all_but_first, list}, global]
)
of
{match, [[HashTypeStr, PasswordHashStr]]} ->
case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of
{ok, HashType} ->
PasswordHash = to_binary(PasswordHashStr),
is_valid_algorithm(HashType, PasswordHash, Password, Entry, State);
_Error ->
{error, invalid_hash_type}
end;
_ ->
OnFail(LDAPPassword, Password, Entry, State)
end.
is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) ->
case lists:member(HashType, ?VALID_ALGORITHMS) of
true ->
verify_password(HashType, PasswordHash, Password, Entry, State);
_ ->
{error, {invalid_hash_type, HashType}}
end.
%% this password is in LDIF format which is base64 encoding
try_decode_password(LDAPPassword, Password, Entry, State) ->
case safe_base64_decode(LDAPPassword) of
{ok, Decode} ->
extract_hash_algorithm(
Decode,
Password,
fun(_, _, _, _) ->
{error, invalid_password}
end,
Entry,
State
);
{error, Reason} ->
{error, {invalid_password, Reason}}
end.
%% sha with salt
%% https://www.openldap.org/faq/data/cache/347.html
verify_password(ssha, PasswordData, Password, Entry, State) ->
case safe_base64_decode(PasswordData) of
{ok, <<PasswordHash:20/binary, Salt/binary>>} ->
verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State);
{ok, _} ->
{error, invalid_ssha_password};
{error, Reason} ->
{error, {invalid_password, Reason}}
end;
verify_password(
Algorithm,
Base64HashData,
Password,
Entry,
State
) ->
verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State).
verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) ->
PasswordHash = hash_password(Algorithm, Salt, Position, Password),
case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of
true ->
{ok, is_superuser(Entry, State)};
_ ->
{error, bad_username_or_password}
end.
is_superuser(Entry, #{is_superuser_attribute := Attr} = _State) ->
Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"),
#{is_superuser => emqx_authn_utils:to_bool(Value)}.
safe_base64_decode(Data) ->
try
{ok, base64:decode(Data)}
catch
_:Reason ->
{error, {invalid_base64_data, Reason}}
end.
get_lower_bin_value(Key, Proplists, Default) ->
[Value | _] = get_value(Key, Proplists, [Default]),
to_binary(string:to_lower(Value)).
to_binary(Value) ->
erlang:list_to_binary(Value).
hash_password(Algorithm, _Salt, disable, Password) ->
hash_password(Algorithm, Password);
hash_password(Algorithm, Salt, suffix, Password) ->
hash_password(Algorithm, <<Password/binary, Salt/binary>>).
hash_password(Algorithm, Data) ->
crypto:hash(Algorithm, Data).
compare_password(hash, LDAPPasswordHash, PasswordHash) ->
emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash);
compare_password(base64, Base64HashData, PasswordHash) ->
emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)).
maps:with([query_timeout, method], Config).

21
apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl
View File

@ -20,32 +20,13 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("eldap/include/eldap.hrl").
-behaviour(emqx_authn_provider).
-export([
create/2,
update/2,
authenticate/2,
destroy/1
authenticate/2
]).
%%------------------------------------------------------------------------------
%% APIs
%%------------------------------------------------------------------------------
create(_AuthenticatorID, Config) ->
emqx_authn_ldap:do_create(?MODULE, Config).
update(Config, State) ->
emqx_authn_ldap:update(Config, State).
destroy(State) ->
emqx_authn_ldap:destroy(State).
authenticate(#{auth_method := _}, _) ->
ignore;
authenticate(#{password := undefined}, _) ->
{error, bad_username_or_password};
authenticate(
#{password := _Password} = Credential,
#{

66
apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl
View File

@ -1,66 +0,0 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authn_ldap_bind_schema).
-behaviour(emqx_authn_schema).
-export([
fields/1,
desc/1,
refs/0,
select_union_member/1,
namespace/0
]).
-include("emqx_auth_ldap.hrl").
-include_lib("hocon/include/hoconsc.hrl").
namespace() -> "authn".
refs() ->
[?R_REF(ldap_bind)].
select_union_member(#{
<<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIND_BIN
}) ->
refs();
select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIND_BIN}) ->
throw(#{
reason => "unknown_mechanism",
expected => ?AUTHN_MECHANISM
});
select_union_member(_) ->
undefined.
fields(ldap_bind) ->
[
{mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)},
{backend, emqx_authn_schema:backend(?AUTHN_BACKEND_BIND)},
{query_timeout, fun query_timeout/1}
] ++
emqx_authn_schema:common_fields() ++
emqx_ldap:fields(config) ++ emqx_ldap:fields(bind_opts).
desc(ldap_bind) ->
?DESC(ldap_bind);
desc(_) ->
undefined.
query_timeout(type) -> emqx_schema:timeout_duration_ms();
query_timeout(desc) -> ?DESC(?FUNCTION_NAME);
query_timeout(default) -> <<"5s">>;
query_timeout(_) -> undefined.

197
apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl Normal file
View File

@ -0,0 +1,197 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authn_ldap_hash).
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("eldap/include/eldap.hrl").
%% a compatible attribute for version 4.x
-define(ISENABLED_ATTR, "isEnabled").
-define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]).
%% TODO
%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512
%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112
-export([
authenticate/2
]).
-import(proplists, [get_value/2, get_value/3]).
%%------------------------------------------------------------------------------
%% APIs
%%------------------------------------------------------------------------------
authenticate(
#{password := Password} = Credential,
#{
method := #{
password_attribute := PasswordAttr,
is_superuser_attribute := IsSuperuserAttr
},
query_timeout := Timeout,
resource_id := ResourceId
} = State
) ->
case
emqx_resource:simple_sync_query(
ResourceId,
{query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout}
)
of
{ok, []} ->
ignore;
{ok, [Entry]} ->
is_enabled(Password, Entry, State);
{error, Reason} ->
?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{
resource => ResourceId,
timeout => Timeout,
reason => Reason
}),
ignore
end.
%% To compatible v4.x
is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) ->
IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"),
case emqx_authn_utils:to_bool(IsEnabled) of
true ->
ensure_password(Password, Entry, State);
_ ->
{error, user_disabled}
end.
ensure_password(
Password,
#eldap_entry{attributes = Attributes} = Entry,
#{method := #{password_attribute := PasswordAttr}} = State
) ->
case get_value(PasswordAttr, Attributes) of
undefined ->
{error, no_password};
[LDAPPassword | _] ->
extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State)
end.
%% RFC 2307 format password
%% https://datatracker.ietf.org/doc/html/rfc2307
extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) ->
case
re:run(
LDAPPassword,
"{([^{}]+)}(.+)",
[{capture, all_but_first, list}, global]
)
of
{match, [[HashTypeStr, PasswordHashStr]]} ->
case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of
{ok, HashType} ->
PasswordHash = to_binary(PasswordHashStr),
is_valid_algorithm(HashType, PasswordHash, Password, Entry, State);
_Error ->
{error, invalid_hash_type}
end;
_ ->
OnFail(LDAPPassword, Password, Entry, State)
end.
is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) ->
case lists:member(HashType, ?VALID_ALGORITHMS) of
true ->
verify_password(HashType, PasswordHash, Password, Entry, State);
_ ->
{error, {invalid_hash_type, HashType}}
end.
%% this password is in LDIF format which is base64 encoding
try_decode_password(LDAPPassword, Password, Entry, State) ->
case safe_base64_decode(LDAPPassword) of
{ok, Decode} ->
extract_hash_algorithm(
Decode,
Password,
fun(_, _, _, _) ->
{error, invalid_password}
end,
Entry,
State
);
{error, Reason} ->
{error, {invalid_password, Reason}}
end.
%% sha with salt
%% https://www.openldap.org/faq/data/cache/347.html
verify_password(ssha, PasswordData, Password, Entry, State) ->
case safe_base64_decode(PasswordData) of
{ok, <<PasswordHash:20/binary, Salt/binary>>} ->
verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State);
{ok, _} ->
{error, invalid_ssha_password};
{error, Reason} ->
{error, {invalid_password, Reason}}
end;
verify_password(
Algorithm,
Base64HashData,
Password,
Entry,
State
) ->
verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State).
verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) ->
PasswordHash = hash_password(Algorithm, Salt, Position, Password),
case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of
true ->
{ok, is_superuser(Entry, State)};
_ ->
{error, bad_username_or_password}
end.
is_superuser(Entry, #{method := #{is_superuser_attribute := Attr}} = _State) ->
Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"),
#{is_superuser => emqx_authn_utils:to_bool(Value)}.
safe_base64_decode(Data) ->
try
{ok, base64:decode(Data)}
catch
_:Reason ->
{error, {invalid_base64_data, Reason}}
end.
get_lower_bin_value(Key, Proplists, Default) ->
[Value | _] = get_value(Key, Proplists, [Default]),
to_binary(string:to_lower(Value)).
to_binary(Value) ->
erlang:list_to_binary(Value).
hash_password(Algorithm, _Salt, disable, Password) ->
hash_password(Algorithm, Password);
hash_password(Algorithm, Salt, suffix, Password) ->
hash_password(Algorithm, <<Password/binary, Salt/binary>>).
hash_password(Algorithm, Data) ->
crypto:hash(Algorithm, Data).
compare_password(hash, LDAPPasswordHash, PasswordHash) ->
emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash);
compare_password(base64, Base64HashData, PasswordHash) ->
emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)).

61
apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl
View File

@ -32,7 +32,7 @@
namespace() -> "authn".
refs() ->
[?R_REF(ldap)].
[?R_REF(ldap), ?R_REF(ldap_deprecated)].
select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN}) ->
refs();
@ -44,12 +44,34 @@ select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIN}) ->
select_union_member(_) ->
undefined.
fields(ldap_deprecated) ->
common_fields() ++
[
{password_attribute, password_attribute()},
{is_superuser_attribute, is_superuser_attribute()}
];
fields(ldap) ->
common_fields() ++
[
{method,
?HOCON(
?UNION([?R_REF(hash_method), ?R_REF(bind_method)]),
#{desc => ?DESC(method)}
)}
];
fields(hash_method) ->
[
{type, method_type(hash)},
{password_attribute, password_attribute()},
{is_superuser_attribute, is_superuser_attribute()}
];
fields(bind_method) ->
[{type, method_type(bind)}] ++ emqx_ldap:fields(bind_opts).
common_fields() ->
[
{mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)},
{backend, emqx_authn_schema:backend(?AUTHN_BACKEND)},
{password_attribute, fun password_attribute/1},
{is_superuser_attribute, fun is_superuser_attribute/1},
{query_timeout, fun query_timeout/1}
] ++
emqx_authn_schema:common_fields() ++
@ -57,18 +79,35 @@ fields(ldap) ->
desc(ldap) ->
?DESC(ldap);
desc(ldap_deprecated) ->
?DESC(ldap_deprecated);
desc(hash_method) ->
?DESC(hash_method);
desc(bind_method) ->
?DESC(bind_method);
desc(_) ->
undefined.
password_attribute(type) -> string();
password_attribute(desc) -> ?DESC(?FUNCTION_NAME);
password_attribute(default) -> <<"userPassword">>;
password_attribute(_) -> undefined.
method_type(Type) ->
?HOCON(?ENUM([Type]), #{desc => ?DESC(?FUNCTION_NAME), default => Type}).
is_superuser_attribute(type) -> string();
is_superuser_attribute(desc) -> ?DESC(?FUNCTION_NAME);
is_superuser_attribute(default) -> <<"isSuperuser">>;
is_superuser_attribute(_) -> undefined.
password_attribute() ->
?HOCON(
string(),
#{
desc => ?DESC(?FUNCTION_NAME),
default => <<"userPassword">>
}
).
is_superuser_attribute() ->
?HOCON(
string(),
#{
desc => ?DESC(?FUNCTION_NAME),
default => <<"isSuperuser">>
}
).
query_timeout(type) -> emqx_schema:timeout_duration_ms();
query_timeout(desc) -> ?DESC(?FUNCTION_NAME);

36
apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl
View File

@ -70,6 +70,29 @@ end_per_suite(Config) ->
%% Tests
%%------------------------------------------------------------------------------
t_create_with_deprecated_cfg(_Config) ->
AuthConfig = deprecated_raw_ldap_auth_config(),
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, AuthConfig}
),
{ok, [#{provider := emqx_authn_ldap, state := State}]} = emqx_authn_chains:list_authenticators(
?GLOBAL
),
?assertMatch(
#{
method := #{
type := hash,
is_superuser_attribute := _,
password_attribute := "not_the_default_value"
}
},
State
),
emqx_authn_test_lib:delete_config(?ResourceID).
t_create(_Config) ->
AuthConfig = raw_ldap_auth_config(),
@ -225,6 +248,19 @@ raw_ldap_auth_config() ->
<<"pool_size">> => 8
}.
deprecated_raw_ldap_auth_config() ->
#{
<<"mechanism">> => <<"password_based">>,
<<"backend">> => <<"ldap">>,
<<"server">> => ldap_server(),
<<"is_superuser_attribute">> => <<"isSuperuser">>,
<<"password_attribute">> => <<"not_the_default_value">>,
<<"base_dn">> => <<"uid=${username},ou=testdevice,dc=emqx,dc=io">>,
<<"username">> => <<"cn=root,dc=emqx,dc=io">>,
<<"password">> => <<"public">>,
<<"pool_size">> => 8
}.
user_seeds() ->
New = fun(Username, Password, Result) ->
#{

17
apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl
View File

@ -27,7 +27,7 @@
-define(LDAP_RESOURCE, <<"emqx_authn_ldap_bind_SUITE">>).
-define(PATH, [authentication]).
-define(ResourceID, <<"password_based:ldap_bind">>).
-define(ResourceID, <<"password_based:ldap">>).
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -78,7 +78,7 @@ t_create(_Config) ->
{create_authenticator, ?GLOBAL, AuthConfig}
),
{ok, [#{provider := emqx_authn_ldap_bind}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
{ok, [#{provider := emqx_authn_ldap}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
emqx_authn_test_lib:delete_config(?ResourceID).
t_create_invalid(_Config) ->
@ -146,10 +146,10 @@ t_destroy(_Config) ->
{create_authenticator, ?GLOBAL, AuthConfig}
),
{ok, [#{provider := emqx_authn_ldap_bind, state := State}]} =
{ok, [#{provider := emqx_authn_ldap, state := State}]} =
emqx_authn_chains:list_authenticators(?GLOBAL),
{ok, _} = emqx_authn_ldap_bind:authenticate(
{ok, _} = emqx_authn_ldap:authenticate(
#{
username => <<"mqttuser0001">>,
password => <<"mqttuser0001">>
@ -165,7 +165,7 @@ t_destroy(_Config) ->
% Authenticator should not be usable anymore
?assertMatch(
ignore,
emqx_authn_ldap_bind:authenticate(
emqx_authn_ldap:authenticate(
#{
username => <<"mqttuser0001">>,
password => <<"mqttuser0001">>
@ -199,7 +199,7 @@ t_update(_Config) ->
% We update with config with correct query, provider should update and work properly
{ok, _} = emqx:update_config(
?PATH,
{update_authenticator, ?GLOBAL, <<"password_based:ldap_bind">>, CorrectConfig}
{update_authenticator, ?GLOBAL, <<"password_based:ldap">>, CorrectConfig}
),
{ok, _} = emqx_access_control:authenticate(
@ -218,14 +218,17 @@ t_update(_Config) ->
raw_ldap_auth_config() ->
#{
<<"mechanism">> => <<"password_based">>,
<<"backend">> => <<"ldap_bind">>,
<<"backend">> => <<"ldap">>,
<<"server">> => ldap_server(),
<<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>,
<<"filter">> => <<"(uid=${username})">>,
<<"username">> => <<"cn=root,dc=emqx,dc=io">>,
<<"password">> => <<"public">>,
<<"pool_size">> => 8,
<<"method">> => #{
<<"type">> => <<"bind">>,
<<"bind_password">> => <<"${password}">>
}
}.
user_seeds() ->

16
apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl
View File

@ -278,6 +278,10 @@ raw_mongo_auth_config() ->
<<"server">> => mongo_server(),
<<"w_mode">> => <<"unsafe">>,
<<"auth_source">> => mongo_authsource(),
<<"username">> => mongo_username(),
<<"password">> => mongo_password(),
<<"filter">> => #{<<"username">> => <<"${username}">>},
<<"password_hash_field">> => <<"password_hash">>,
<<"salt_field">> => <<"salt">>,
@ -464,9 +468,21 @@ mongo_config() ->
{database, <<"mqtt">>},
{host, ?MONGO_HOST},
{port, ?MONGO_DEFAULT_PORT},
{auth_source, mongo_authsource()},
{login, mongo_username()},
{password, mongo_password()},
{register, ?MONGO_CLIENT}
].
mongo_authsource() ->
iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")).
mongo_username() ->
iolist_to_binary(os:getenv("MONGO_USERNAME", "")).
mongo_password() ->
iolist_to_binary(os:getenv("MONGO_PASSWORD", "")).
start_apps(Apps) ->
lists:foreach(fun application:ensure_all_started/1, Apps).

16
apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl
View File

@ -397,6 +397,10 @@ raw_mongo_authz_config() ->
<<"collection">> => <<"acl">>,
<<"server">> => mongo_server(),
<<"auth_source">> => mongo_authsource(),
<<"username">> => mongo_username(),
<<"password">> => mongo_password(),
<<"filter">> => #{<<"username">> => <<"${username}">>}
}.
@ -408,9 +412,21 @@ mongo_config() ->
{database, <<"mqtt">>},
{host, ?MONGO_HOST},
{port, ?MONGO_DEFAULT_PORT},
{auth_source, mongo_authsource()},
{login, mongo_username()},
{password, mongo_password()},
{register, ?MONGO_CLIENT}
].
mongo_authsource() ->
iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")).
mongo_username() ->
iolist_to_binary(os:getenv("MONGO_USERNAME", "")).
mongo_password() ->
iolist_to_binary(os:getenv("MONGO_PASSWORD", "")).
start_apps(Apps) ->
lists:foreach(fun application:ensure_all_started/1, Apps).

3
apps/emqx_bridge/src/emqx_bridge.erl
View File

@ -93,7 +93,8 @@
T == iotdb;
T == kinesis_producer;
T == greptimedb;
T == azure_event_hub_producer
T == azure_event_hub_producer;
T == syskeeper_forwarder
).
-define(ROOT_KEY, bridges).

3
apps/emqx_bridge/src/emqx_bridge_resource.erl
View File

@ -356,9 +356,10 @@ parse_confs(<<"iotdb">>, Name, Conf) ->
authentication :=
#{
username := Username,
password := Password
password := Secret
}
} = Conf,
Password = emqx_secret:unwrap(Secret),
BasicToken = base64:encode(<<Username/binary, ":", Password/binary>>),
%% This version atom correspond to the macro ?VSN_1_1_X in
%% emqx_bridge_iotdb.hrl. It would be better to use the macro directly, but

2
apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_cassandra, [
{description, "EMQX Enterprise Cassandra Bridge"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{applications, [
kernel,

27
apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl
View File

@ -70,7 +70,7 @@ cassandra_db_fields() ->
{keyspace, fun keyspace/1},
{pool_size, fun emqx_connector_schema_lib:pool_size/1},
{username, fun emqx_connector_schema_lib:username/1},
{password, fun emqx_connector_schema_lib:password/1},
{password, emqx_connector_schema_lib:password_field()},
{auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1}
].
@ -111,14 +111,14 @@ on_start(
emqx_schema:parse_servers(Servers0, ?DEFAULT_SERVER_OPTION)
),
Options = [
Options =
maps:to_list(maps:with([username, password], Config)) ++
[
{nodes, Servers},
{keyspace, Keyspace},
{auto_reconnect, ?AUTO_RECONNECT_INTERVAL},
{pool_size, PoolSize}
],
Options1 = maybe_add_opt(username, Config, Options),
Options2 = maybe_add_opt(password, Config, Options1, _IsSensitive = true),
SslOpts =
case maps:get(enable, SSL) of
@ -131,7 +131,7 @@ on_start(
[]
end,
State = parse_prepare_cql(Config),
case emqx_resource_pool:start(InstId, ?MODULE, Options2 ++ SslOpts) of
case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of
ok ->
{ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})};
{error, Reason} ->
@ -387,6 +387,7 @@ conn_opts(Opts) ->
conn_opts([], Acc) ->
Acc;
conn_opts([{password, Password} | Opts], Acc) ->
%% TODO: teach `ecql` to accept 0-arity closures as passwords.
conn_opts(Opts, [{password, emqx_secret:unwrap(Password)} | Acc]);
conn_opts([Opt | Opts], Acc) ->
conn_opts(Opts, [Opt | Acc]).
@ -512,19 +513,3 @@ maybe_assign_type(V) when is_integer(V) ->
maybe_assign_type(V) when is_float(V) -> {double, V};
maybe_assign_type(V) ->
V.
maybe_add_opt(Key, Conf, Opts) ->
maybe_add_opt(Key, Conf, Opts, _IsSensitive = false).
maybe_add_opt(Key, Conf, Opts, IsSensitive) ->
case Conf of
#{Key := Val} ->
[{Key, maybe_wrap(IsSensitive, Val)} | Opts];
_ ->
Opts
end.
maybe_wrap(false = _IsSensitive, Val) ->
Val;
maybe_wrap(true, Val) ->
emqx_secret:wrap(Val).

63
apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl
View File

@ -40,10 +40,9 @@ all() ->
].
groups() ->
TCs = emqx_common_test_helpers:all(?MODULE),
[
{auth, TCs},
{noauth, TCs}
{auth, [t_lifecycle, t_start_passfile]},
{noauth, [t_lifecycle]}
].
cassandra_servers(CassandraHost) ->
@ -115,32 +114,37 @@ end_per_testcase(_, _Config) ->
t_lifecycle(Config) ->
perform_lifecycle_check(
<<"emqx_connector_cassandra_SUITE">>,
<<?MODULE_STRING>>,
cassandra_config(Config)
).
show(X) ->
erlang:display(X),
X.
show(Label, What) ->
erlang:display({Label, What}),
What.
t_start_passfile(Config) ->
ResourceID = atom_to_binary(?FUNCTION_NAME),
PasswordFilename = filename:join(?config(priv_dir, Config), "passfile"),
ok = file:write_file(PasswordFilename, ?CASSA_PASSWORD),
InitialConfig = emqx_utils_maps:deep_merge(
cassandra_config(Config),
#{
<<"config">> => #{
password => iolist_to_binary(["file://", PasswordFilename])
}
}
),
?assertMatch(
#{status := connected},
create_local_resource(ResourceID, check_config(InitialConfig))
),
?assertEqual(
ok,
emqx_resource:remove_local(ResourceID)
).
perform_lifecycle_check(ResourceId, InitialConfig) ->
{ok, #{config := CheckedConfig}} =
emqx_resource:check_config(?CASSANDRA_RESOURCE_MOD, InitialConfig),
{ok, #{
CheckedConfig = check_config(InitialConfig),
#{
state := #{pool_name := PoolName} = State,
status := InitialStatus
}} =
emqx_resource:create_local(
ResourceId,
?CONNECTOR_RESOURCE_GROUP,
?CASSANDRA_RESOURCE_MOD,
CheckedConfig,
#{}
),
} = create_local_resource(ResourceId, CheckedConfig),
?assertEqual(InitialStatus, connected),
% Instance should match the state and status of the just started resource
{ok, ?CONNECTOR_RESOURCE_GROUP, #{
@ -191,6 +195,21 @@ perform_lifecycle_check(ResourceId, InitialConfig) ->
%% utils
%%--------------------------------------------------------------------
check_config(Config) ->
{ok, #{config := CheckedConfig}} = emqx_resource:check_config(?CASSANDRA_RESOURCE_MOD, Config),
CheckedConfig.
create_local_resource(ResourceId, CheckedConfig) ->
{ok, Bridge} =
emqx_resource:create_local(
ResourceId,
?CONNECTOR_RESOURCE_GROUP,
?CASSANDRA_RESOURCE_MOD,
CheckedConfig,
#{}
),
Bridge.
cassandra_config(Config) ->
Host = ?config(cassa_host, Config),
AuthOpts = maps:from_list(?config(cassa_auth_opts, Config)),

2
apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_clickhouse, [
{description, "EMQX Enterprise ClickHouse Bridge"},
{vsn, "0.2.3"},
{vsn, "0.2.4"},
{registered, []},
{applications, [
kernel,

3
apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
View File

@ -145,7 +145,7 @@ on_start(
Options = [
{url, URL},
{user, maps:get(username, Config, "default")},
{key, emqx_secret:wrap(maps:get(password, Config, "public"))},
{key, maps:get(password, Config, emqx_secret:wrap("public"))},
{database, DB},
{auto_reconnect, ?AUTO_RECONNECT_INTERVAL},
{pool_size, PoolSize},
@ -243,6 +243,7 @@ connect(Options) ->
URL = iolist_to_binary(emqx_http_lib:normalize(proplists:get_value(url, Options))),
User = proplists:get_value(user, Options),
Database = proplists:get_value(database, Options),
%% TODO: teach `clickhouse` to accept 0-arity closures as passwords.
Key = emqx_secret:unwrap(proplists:get_value(key, Options)),
Pool = proplists:get_value(pool, Options),
PoolSize = proplists:get_value(pool_size, Options),

36
apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl
View File

@ -10,10 +10,12 @@
-include("emqx_connector.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("stdlib/include/assert.hrl").
-include_lib("common_test/include/ct.hrl").
-define(APP, emqx_bridge_clickhouse).
-define(CLICKHOUSE_HOST, "clickhouse").
-define(CLICKHOUSE_RESOURCE_MOD, emqx_bridge_clickhouse_connector).
-define(CLICKHOUSE_PASSWORD, "public").
%% This test SUITE requires a running clickhouse instance. If you don't want to
%% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script
@ -57,7 +59,7 @@ init_per_suite(Config) ->
clickhouse:start_link([
{url, clickhouse_url()},
{user, <<"default">>},
{key, "public"},
{key, ?CLICKHOUSE_PASSWORD},
{pool, tmp_pool}
]),
{ok, _, _} = clickhouse:query(Conn, <<"CREATE DATABASE IF NOT EXISTS mqtt">>, #{}),
@ -92,6 +94,31 @@ t_lifecycle(_Config) ->
clickhouse_config()
).
t_start_passfile(Config) ->
ResourceID = atom_to_binary(?FUNCTION_NAME),
PasswordFilename = filename:join(?config(priv_dir, Config), "passfile"),
ok = file:write_file(PasswordFilename, <<?CLICKHOUSE_PASSWORD>>),
InitialConfig = clickhouse_config(#{
password => iolist_to_binary(["file://", PasswordFilename])
}),
{ok, #{config := ResourceConfig}} =
emqx_resource:check_config(?CLICKHOUSE_RESOURCE_MOD, InitialConfig),
?assertMatch(
{ok, #{status := connected}},
emqx_resource:create_local(
ResourceID,
?CONNECTOR_RESOURCE_GROUP,
?CLICKHOUSE_RESOURCE_MOD,
ResourceConfig,
#{}
)
),
?assertEqual(
ok,
emqx_resource:remove_local(ResourceID)
),
ok.
show(X) ->
erlang:display(X),
X.
@ -168,12 +195,15 @@ perform_lifecycle_check(ResourceID, InitialConfig) ->
% %%------------------------------------------------------------------------------
clickhouse_config() ->
clickhouse_config(#{}).
clickhouse_config(Overrides) ->
Config =
#{
auto_reconnect => true,
database => <<"mqtt">>,
username => <<"default">>,
password => <<"public">>,
password => <<?CLICKHOUSE_PASSWORD>>,
pool_size => 8,
url => iolist_to_binary(
io_lib:format(
@ -186,7 +216,7 @@ clickhouse_config() ->
),
connect_timeout => <<"10s">>
},
#{<<"config">> => Config}.
#{<<"config">> => maps:merge(Config, Overrides)}.
test_query_no_params() ->
{query, <<"SELECT 1">>}.

2
apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_dynamo, [
{description, "EMQX Enterprise Dynamo Bridge"},
{vsn, "0.1.3"},
{vsn, "0.1.4"},
{registered, []},
{applications, [
kernel,

13
apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl
View File

@ -45,12 +45,10 @@ fields(config) ->
#{required => true, desc => ?DESC("aws_access_key_id")}
)},
{aws_secret_access_key,
mk(
binary(),
emqx_schema_secret:mk(
#{
required => true,
desc => ?DESC("aws_secret_access_key"),
sensitive => true
desc => ?DESC("aws_secret_access_key")
}
)},
{pool_size, fun emqx_connector_schema_lib:pool_size/1},
@ -89,7 +87,7 @@ on_start(
host => Host,
port => Port,
aws_access_key_id => to_str(AccessKeyID),
aws_secret_access_key => to_str(SecretAccessKey),
aws_secret_access_key => SecretAccessKey,
schema => Schema
}},
{pool_size, PoolSize}
@ -182,9 +180,8 @@ do_query(
end.
connect(Opts) ->
Options = proplists:get_value(config, Opts),
{ok, _Pid} = Result = emqx_bridge_dynamo_connector_client:start_link(Options),
Result.
Config = proplists:get_value(config, Opts),
{ok, _Pid} = emqx_bridge_dynamo_connector_client:start_link(Config).
parse_template(Config) ->
Templates =

19
apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl
View File

@ -20,8 +20,7 @@
handle_cast/2,
handle_info/2,
terminate/2,
code_change/3,
format_status/2
code_change/3
]).
-ifdef(TEST).
@ -62,11 +61,13 @@ start_link(Options) ->
%% Initialize dynamodb data bridge
init(#{
aws_access_key_id := AccessKeyID,
aws_secret_access_key := SecretAccessKey,
aws_secret_access_key := Secret,
host := Host,
port := Port,
schema := Schema
}) ->
%% TODO: teach `erlcloud` to to accept 0-arity closures as passwords.
SecretAccessKey = to_str(emqx_secret:unwrap(Secret)),
erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema),
{ok, #{}}.
@ -101,13 +102,6 @@ terminate(_Reason, _State) ->
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
-spec format_status(
Opt :: normal | terminate,
Status :: list()
) -> Status :: term().
format_status(_Opt, Status) ->
Status.
%%%===================================================================
%%% Internal functions
%%%===================================================================
@ -184,3 +178,8 @@ convert2binary(Value) when is_list(Value) ->
unicode:characters_to_binary(Value);
convert2binary(Value) when is_map(Value) ->
emqx_utils_json:encode(Value).
to_str(List) when is_list(List) ->
List;
to_str(Bin) when is_binary(Bin) ->
erlang:binary_to_list(Bin).

55
apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl
View File

@ -22,8 +22,6 @@
-define(BATCH_SIZE, 10).
-define(PAYLOAD, <<"HELLO">>).
-define(GET_CONFIG(KEY__, CFG__), proplists:get_value(KEY__, CFG__)).
%% How to run it locally (all commands are run in $PROJ_ROOT dir):
%% run ct in docker container
%% run script:
@ -84,7 +82,9 @@ end_per_group(_Group, _Config) ->
ok.
init_per_suite(Config) ->
Config.
SecretFile = filename:join(?config(priv_dir, Config), "secret"),
ok = file:write_file(SecretFile, <<?SECRET_ACCESS_KEY>>),
[{dynamo_secretfile, SecretFile} | Config].
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite(),
@ -158,32 +158,35 @@ common_init(ConfigT) ->
end.
dynamo_config(BridgeType, Config) ->
Port = integer_to_list(?GET_CONFIG(port, Config)),
Url = "http://" ++ ?GET_CONFIG(host, Config) ++ ":" ++ Port,
Host = ?config(host, Config),
Port = ?config(port, Config),
Name = atom_to_binary(?MODULE),
BatchSize = ?GET_CONFIG(batch_size, Config),
QueryMode = ?GET_CONFIG(query_mode, Config),
BatchSize = ?config(batch_size, Config),
QueryMode = ?config(query_mode, Config),
SecretFile = ?config(dynamo_secretfile, Config),
ConfigString =
io_lib:format(
"bridges.~s.~s {\n"
" enable = true\n"
" url = ~p\n"
" table = ~p\n"
" aws_access_key_id = ~p\n"
" aws_secret_access_key = ~p\n"
" resource_opts = {\n"
" request_ttl = 500ms\n"
" batch_size = ~b\n"
" query_mode = ~s\n"
" }\n"
"}",
"bridges.~s.~s {"
"\n enable = true"
"\n url = \"http://~s:~p\""
"\n table = ~p"
"\n aws_access_key_id = ~p"
"\n aws_secret_access_key = ~p"
"\n resource_opts = {"
"\n request_ttl = 500ms"
"\n batch_size = ~b"
"\n query_mode = ~s"
"\n }"
"\n }",
[
BridgeType,
Name,
Url,
Host,
Port,
?TABLE,
?ACCESS_KEY_ID,
?SECRET_ACCESS_KEY,
%% NOTE: using file-based secrets with HOCON configs
"file://" ++ SecretFile,
BatchSize,
QueryMode
]
@ -252,8 +255,8 @@ delete_table(_Config) ->
erlcloud_ddb2:delete_table(?TABLE_BIN).
setup_dynamo(Config) ->
Host = ?GET_CONFIG(host, Config),
Port = ?GET_CONFIG(port, Config),
Host = ?config(host, Config),
Port = ?config(port, Config),
erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, Host, Port, ?SCHEMA).
directly_setup_dynamo() ->
@ -313,7 +316,9 @@ t_setup_via_http_api_and_publish(Config) ->
PgsqlConfig0 = ?config(dynamo_config, Config),
PgsqlConfig = PgsqlConfig0#{
<<"name">> => Name,
<<"type">> => BridgeType
<<"type">> => BridgeType,
%% NOTE: using literal secret with HTTP API requests.
<<"aws_secret_access_key">> => <<?SECRET_ACCESS_KEY>>
},
?assertMatch(
{ok, _},
@ -400,7 +405,7 @@ t_simple_query(Config) ->
),
Request = {get_item, {<<"id">>, <<"not_exists">>}},
Result = query_resource(Config, Request),
case ?GET_CONFIG(batch_size, Config) of
case ?config(batch_size, Config) of
?BATCH_SIZE ->
?assertMatch({error, {unrecoverable_error, {invalid_request, _}}}, Result);
1 ->

11
apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl
View File

@ -147,13 +147,7 @@ fields(greptimedb) ->
[
{dbname, mk(binary(), #{required => true, desc => ?DESC("dbname")})},
{username, mk(binary(), #{desc => ?DESC("username")})},
{password,
mk(binary(), #{
desc => ?DESC("password"),
format => <<"password">>,
sensitive => true,
converter => fun emqx_schema:password_converter/2
})}
{password, emqx_schema_secret:mk(#{desc => ?DESC("password")})}
] ++ emqx_connector_schema_lib:ssl_fields().
server() ->
@ -302,7 +296,8 @@ ssl_config(SSL = #{enable := true}) ->
auth(#{username := Username, password := Password}) ->
[
{auth, {basic, #{username => str(Username), password => str(Password)}}}
%% TODO: teach `greptimedb` to accept 0-arity closures as passwords.
{auth, {basic, #{username => str(Username), password => emqx_secret:unwrap(Password)}}}
];
auth(_) ->
[].

16
apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
View File

@ -192,20 +192,14 @@ fields(influxdb_api_v1) ->
[
{database, mk(binary(), #{required => true, desc => ?DESC("database")})},
{username, mk(binary(), #{desc => ?DESC("username")})},
{password,
mk(binary(), #{
desc => ?DESC("password"),
format => <<"password">>,
sensitive => true,
converter => fun emqx_schema:password_converter/2
})}
{password, emqx_schema_secret:mk(#{desc => ?DESC("password")})}
] ++ emqx_connector_schema_lib:ssl_fields();
fields(influxdb_api_v2) ->
fields(common) ++
[
{bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})},
{org, mk(binary(), #{required => true, desc => ?DESC("org")})},
{token, mk(binary(), #{required => true, desc => ?DESC("token")})}
{token, emqx_schema_secret:mk(#{required => true, desc => ?DESC("token")})}
] ++ emqx_connector_schema_lib:ssl_fields().
server() ->
@ -363,7 +357,8 @@ protocol_config(#{
{version, v2},
{bucket, str(Bucket)},
{org, str(Org)},
{token, Token}
%% TODO: teach `influxdb` to accept 0-arity closures as passwords.
{token, emqx_secret:unwrap(Token)}
] ++ ssl_config(SSL).
ssl_config(#{enable := false}) ->
@ -383,7 +378,8 @@ username(_) ->
[].
password(#{password := Password}) ->
[{password, str(Password)}];
%% TODO: teach `influxdb` to accept 0-arity closures as passwords.
[{password, str(emqx_secret:unwrap(Password))}];
password(_) ->
[].

2
apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge_iotdb, [
{description, "EMQX Enterprise Apache IoTDB Bridge"},
{vsn, "0.1.3"},
{vsn, "0.1.4"},
{modules, [
emqx_bridge_iotdb,
emqx_bridge_iotdb_impl

7
apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl
View File

@ -51,12 +51,9 @@ fields(auth_basic) ->
[
{username, mk(binary(), #{required => true, desc => ?DESC("config_auth_basic_username")})},
{password,
mk(binary(), #{
emqx_schema_secret:mk(#{
required => true,
desc => ?DESC("config_auth_basic_password"),
format => <<"password">>,
sensitive => true,
converter => fun emqx_schema:password_converter/2
desc => ?DESC("config_auth_basic_password")
})}
].

6
apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
View File

@ -283,11 +283,9 @@ fields(auth_username_password) ->
})},
{username, mk(binary(), #{required => true, desc => ?DESC(auth_sasl_username)})},
{password,
mk(binary(), #{
emqx_connector_schema_lib:password_field(#{
required => true,
sensitive => true,
desc => ?DESC(auth_sasl_password),
converter => fun emqx_schema:password_converter/2
desc => ?DESC(auth_sasl_password)
})}
];
fields(auth_gssapi_kerberos) ->

4
apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl
View File

@ -31,8 +31,8 @@ make_client_id(BridgeType0, BridgeName0) ->
sasl(none) ->
undefined;
sasl(#{mechanism := Mechanism, username := Username, password := Password}) ->
{Mechanism, Username, emqx_secret:wrap(Password)};
sasl(#{mechanism := Mechanism, username := Username, password := Secret}) ->
{Mechanism, Username, Secret};
sasl(#{
kerberos_principal := Principal,
kerberos_keytab_file := KeyTabFile

119
apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl
View File

@ -30,29 +30,41 @@ all() ->
].
groups() ->
AllTCs = emqx_common_test_helpers:all(?MODULE),
SASLAuths = [
sasl_auth_plain,
sasl_auth_scram256,
sasl_auth_scram512,
sasl_auth_kerberos
SASLGroups = [
{sasl_auth_plain, testcases(sasl)},
{sasl_auth_scram256, testcases(sasl)},
{sasl_auth_scram512, testcases(sasl)},
{sasl_auth_kerberos, testcases(sasl_auth_kerberos)}
],
SASLAuthGroups = [{group, Type} || Type <- SASLAuths],
OnlyOnceTCs = only_once_tests(),
MatrixTCs = AllTCs -- OnlyOnceTCs,
SASLTests = [{Group, MatrixTCs} || Group <- SASLAuths],
SASLAuthGroups = [{group, Group} || {Group, _} <- SASLGroups],
[
{plain, MatrixTCs ++ OnlyOnceTCs},
{ssl, MatrixTCs},
{plain, testcases(plain)},
{ssl, testcases(common)},
{sasl_plain, SASLAuthGroups},
{sasl_ssl, SASLAuthGroups}
] ++ SASLTests.
| SASLGroups
].
sasl_only_tests() ->
[t_failed_creation_then_fixed].
%% tests that do not need to be run on all groups
only_once_tests() ->
testcases(all) ->
emqx_common_test_helpers:all(?MODULE);
testcases(plain) ->
%% NOTE: relevant only for a subset of SASL testcases
Exclude = [t_failed_creation_then_fixed],
testcases(all) -- Exclude;
testcases(common) ->
testcases(plain) -- testcases(once);
testcases(sasl) ->
testcases(all) -- testcases(once);
testcases(sasl_auth_kerberos) ->
%% NOTE: need a proxy to run these tests
Exclude = [
t_failed_creation_then_fixed,
t_on_get_status,
t_receive_after_recovery
],
testcases(sasl) -- Exclude;
testcases(once) ->
%% tests that do not need to be run on all groups
[
t_begin_offset_earliest,
t_bridge_rule_action_source,
@ -220,7 +232,7 @@ init_per_group(sasl_auth_kerberos, Config0) ->
(KV) ->
KV
end,
[{has_proxy, false}, {sasl_auth_mechanism, kerberos} | Config0]
[{sasl_auth_mechanism, kerberos} | Config0]
),
Config;
init_per_group(_Group, Config) ->
@ -264,43 +276,6 @@ end_per_group(Group, Config) when
end_per_group(_Group, _Config) ->
ok.
init_per_testcase(TestCase, Config) when
TestCase =:= t_failed_creation_then_fixed
->
KafkaType = ?config(kafka_type, Config),
AuthMechanism = ?config(sasl_auth_mechanism, Config),
IsSASL = lists:member(KafkaType, [sasl_plain, sasl_ssl]),
case {IsSASL, AuthMechanism} of
{true, kerberos} ->
[{skip_does_not_apply, true}];
{true, _} ->
common_init_per_testcase(TestCase, Config);
{false, _} ->
[{skip_does_not_apply, true}]
end;
init_per_testcase(TestCase, Config) when
TestCase =:= t_failed_creation_then_fixed
->
%% test with one partiton only for this case because
%% the wait probe may not be always sent to the same partition
HasProxy = proplists:get_value(has_proxy, Config, true),
case HasProxy of
false ->
[{skip_does_not_apply, true}];
true ->
common_init_per_testcase(TestCase, [{num_partitions, 1} | Config])
end;
init_per_testcase(TestCase, Config) when
TestCase =:= t_on_get_status;
TestCase =:= t_receive_after_recovery
->
HasProxy = proplists:get_value(has_proxy, Config, true),
case HasProxy of
false ->
[{skip_does_not_apply, true}];
true ->
common_init_per_testcase(TestCase, Config)
end;
init_per_testcase(t_cluster_group = TestCase, Config0) ->
Config = emqx_utils:merge_opts(Config0, [{num_partitions, 6}]),
common_init_per_testcase(TestCase, Config);
@ -393,10 +368,6 @@ common_init_per_testcase(TestCase, Config0) ->
].
end_per_testcase(_Testcase, Config) ->
case proplists:get_bool(skip_does_not_apply, Config) of
true ->
ok;
false ->
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
ProducersConfigs = ?config(kafka_producers, Config),
@ -414,9 +385,7 @@ end_per_testcase(_Testcase, Config) ->
%% in CI, apparently this needs more time since the
%% machines struggle with all the containers running...
emqx_common_test_helpers:call_janitor(60_000),
ok = snabbkaffe:stop(),
ok
end.
ok = snabbkaffe:stop().
%%------------------------------------------------------------------------------
%% Helper fns
@ -1391,14 +1360,6 @@ t_multiple_topic_mappings(Config) ->
ok.
t_on_get_status(Config) ->
case proplists:get_bool(skip_does_not_apply, Config) of
true ->
ok;
false ->
do_t_on_get_status(Config)
end.
do_t_on_get_status(Config) ->
ProxyPort = ?config(proxy_port, Config),
ProxyHost = ?config(proxy_host, Config),
ProxyName = ?config(proxy_name, Config),
@ -1421,14 +1382,6 @@ do_t_on_get_status(Config) ->
%% ensure that we can create and use the bridge successfully after
%% creating it with bad config.
t_failed_creation_then_fixed(Config) ->
case proplists:get_bool(skip_does_not_apply, Config) of
true ->
ok;
false ->
?check_trace(do_t_failed_creation_then_fixed(Config), [])
end.
do_t_failed_creation_then_fixed(Config) ->
ct:timetrap({seconds, 180}),
MQTTTopic = ?config(mqtt_topic, Config),
MQTTQoS = ?config(mqtt_qos, Config),
@ -1516,14 +1469,6 @@ do_t_failed_creation_then_fixed(Config) ->
%% recovering from a network partition will make the subscribers
%% consume the messages produced during the down time.
t_receive_after_recovery(Config) ->
case proplists:get_bool(skip_does_not_apply, Config) of
true ->
ok;
false ->
do_t_receive_after_recovery(Config)
end.
do_t_receive_after_recovery(Config) ->
ct:timetrap(120_000),
ProxyPort = ?config(proxy_port, Config),
ProxyHost = ?config(proxy_host, Config),

212
apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl
View File

@ -28,13 +28,8 @@
).
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx_dashboard/include/emqx_dashboard.hrl").
-define(HOST, "http://127.0.0.1:18083").
%% -define(API_VERSION, "v5").
-define(BASE_PATH, "/api/v5").
%% NOTE: it's "kafka", but not "kafka_producer"
@ -48,13 +43,6 @@
%%------------------------------------------------------------------------------
all() ->
case code:get_object_code(cthr) of
{Module, Code, Filename} ->
{module, Module} = code:load_binary(Module, Filename, Code),
ok;
error ->
error
end,
All0 = emqx_common_test_helpers:all(?MODULE),
All = All0 -- matrix_cases(),
Groups = lists:map(fun({G, _, _}) -> {group, G} end, groups()),
@ -105,23 +93,12 @@ init_per_suite(Config0) ->
emqx_connector,
emqx_bridge_kafka,
emqx_bridge,
emqx_rule_engine
emqx_rule_engine,
{emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
emqx_mgmt_api_test_util:init_suite(),
wait_until_kafka_is_up(),
%% Wait until bridges API is up
(fun WaitUntilRestApiUp() ->
case http_get(["bridges"]) of
{ok, 200, _Res} ->
ok;
Val ->
ct:pal("REST API for bridges not up. Wait and try again. Response: ~p", [Val]),
timer:sleep(1000),
WaitUntilRestApiUp()
end
end)(),
[{apps, Apps} | Config].
end_per_suite(Config) ->
@ -183,6 +160,7 @@ t_query_mode_async(CtConfig) ->
t_publish(matrix) ->
{publish, [
[tcp, none, key_dispatch, sync],
[ssl, plain_passfile, random, sync],
[ssl, scram_sha512, random, async],
[ssl, kerberos, random, sync]
]};
@ -200,9 +178,15 @@ t_publish(Config) ->
end,
Auth1 =
case Auth of
none -> "none";
scram_sha512 -> valid_sasl_scram512_settings();
kerberos -> valid_sasl_kerberos_settings()
none ->
"none";
plain_passfile ->
Passfile = filename:join(?config(priv_dir, Config), "passfile"),
valid_sasl_plain_passfile_settings(Passfile);
scram_sha512 ->
valid_sasl_scram512_settings();
kerberos ->
valid_sasl_kerberos_settings()
end,
ConnCfg = #{
"bootstrap_hosts" => Hosts,
@ -1018,112 +1002,89 @@ hocon_config(Args, ConfigTemplateFun) ->
),
Hocon.
%% erlfmt-ignore
hocon_config_template() ->
"""
bridges.kafka.{{ bridge_name }} {
bootstrap_hosts = \"{{ kafka_hosts_string }}\"
enable = true
authentication = {{{ authentication }}}
ssl = {{{ ssl }}}
local_topic = \"{{ local_topic }}\"
kafka = {
message = {
key = \"${clientid}\"
value = \"${.payload}\"
timestamp = \"${timestamp}\"
}
buffer = {
memory_overload_protection = false
}
partition_strategy = {{ partition_strategy }}
topic = \"{{ kafka_topic }}\"
query_mode = {{ query_mode }}
}
metadata_request_timeout = 5s
min_metadata_refresh_interval = 3s
socket_opts {
nodelay = true
}
connect_timeout = 5s
}
""".
"bridges.kafka.{{ bridge_name }} {"
"\n bootstrap_hosts = \"{{ kafka_hosts_string }}\""
"\n enable = true"
"\n authentication = {{{ authentication }}}"
"\n ssl = {{{ ssl }}}"
"\n local_topic = \"{{ local_topic }}\""
"\n kafka = {"
"\n message = {"
"\n key = \"${clientid}\""
"\n value = \"${.payload}\""
"\n timestamp = \"${timestamp}\""
"\n }"
"\n buffer = {"
"\n memory_overload_protection = false"
"\n }"
"\n partition_strategy = {{ partition_strategy }}"
"\n topic = \"{{ kafka_topic }}\""
"\n query_mode = {{ query_mode }}"
"\n }"
"\n metadata_request_timeout = 5s"
"\n min_metadata_refresh_interval = 3s"
"\n socket_opts {"
"\n nodelay = true"
"\n }"
"\n connect_timeout = 5s"
"\n }".
%% erlfmt-ignore
hocon_config_template_with_headers() ->
"""
bridges.kafka.{{ bridge_name }} {
bootstrap_hosts = \"{{ kafka_hosts_string }}\"
enable = true
authentication = {{{ authentication }}}
ssl = {{{ ssl }}}
local_topic = \"{{ local_topic }}\"
kafka = {
message = {
key = \"${clientid}\"
value = \"${.payload}\"
timestamp = \"${timestamp}\"
}
buffer = {
memory_overload_protection = false
}
kafka_headers = \"{{ kafka_headers }}\"
kafka_header_value_encode_mode: json
kafka_ext_headers: {{{ kafka_ext_headers }}}
partition_strategy = {{ partition_strategy }}
topic = \"{{ kafka_topic }}\"
query_mode = {{ query_mode }}
}
metadata_request_timeout = 5s
min_metadata_refresh_interval = 3s
socket_opts {
nodelay = true
}
connect_timeout = 5s
}
""".
"bridges.kafka.{{ bridge_name }} {"
"\n bootstrap_hosts = \"{{ kafka_hosts_string }}\""
"\n enable = true"
"\n authentication = {{{ authentication }}}"
"\n ssl = {{{ ssl }}}"
"\n local_topic = \"{{ local_topic }}\""
"\n kafka = {"
"\n message = {"
"\n key = \"${clientid}\""
"\n value = \"${.payload}\""
"\n timestamp = \"${timestamp}\""
"\n }"
"\n buffer = {"
"\n memory_overload_protection = false"
"\n }"
"\n kafka_headers = \"{{ kafka_headers }}\""
"\n kafka_header_value_encode_mode: json"
"\n kafka_ext_headers: {{{ kafka_ext_headers }}}"
"\n partition_strategy = {{ partition_strategy }}"
"\n topic = \"{{ kafka_topic }}\""
"\n query_mode = {{ query_mode }}"
"\n }"
"\n metadata_request_timeout = 5s"
"\n min_metadata_refresh_interval = 3s"
"\n socket_opts {"
"\n nodelay = true"
"\n }"
"\n connect_timeout = 5s"
"\n }".
%% erlfmt-ignore
hocon_config_template_authentication("none") ->
"none";
hocon_config_template_authentication(#{"mechanism" := _}) ->
"""
{
mechanism = {{ mechanism }}
password = {{ password }}
username = {{ username }}
}
""";
"{"
"\n mechanism = {{ mechanism }}"
"\n password = \"{{ password }}\""
"\n username = \"{{ username }}\""
"\n }";
hocon_config_template_authentication(#{"kerberos_principal" := _}) ->
"""
{
kerberos_principal = \"{{ kerberos_principal }}\"
kerberos_keytab_file = \"{{ kerberos_keytab_file }}\"
}
""".
"{"
"\n kerberos_principal = \"{{ kerberos_principal }}\""
"\n kerberos_keytab_file = \"{{ kerberos_keytab_file }}\""
"\n }".
%% erlfmt-ignore
hocon_config_template_ssl(Map) when map_size(Map) =:= 0 ->
"""
{
enable = false
}
""";
"{ enable = false }";
hocon_config_template_ssl(#{"enable" := "false"}) ->
"""
{
enable = false
}
""";
"{ enable = false }";
hocon_config_template_ssl(#{"enable" := "true"}) ->
"""
{
enable = true
cacertfile = \"{{{cacertfile}}}\"
certfile = \"{{{certfile}}}\"
keyfile = \"{{{keyfile}}}\"
}
""".
"{ enable = true"
"\n cacertfile = \"{{{cacertfile}}}\""
"\n certfile = \"{{{certfile}}}\""
"\n keyfile = \"{{{keyfile}}}\""
"\n }".
kafka_hosts_string(tcp, none) ->
kafka_hosts_string();
@ -1197,6 +1158,13 @@ valid_sasl_kerberos_settings() ->
"kerberos_keytab_file" => shared_secret(rig_keytab)
}.
valid_sasl_plain_passfile_settings(Passfile) ->
Auth = valid_sasl_plain_settings(),
ok = file:write_file(Passfile, maps:get("password", Auth)),
Auth#{
"password" := "file://" ++ Passfile
}.
kafka_hosts() ->
kpro:parse_endpoints(kafka_hosts_string()).

250
apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl
View File

@ -223,144 +223,136 @@ check_atom_key(Conf) when is_map(Conf) ->
%% Data section
%%===========================================================================
%% erlfmt-ignore
kafka_producer_old_hocon(_WithLocalTopic = true) ->
kafka_producer_old_hocon("mqtt {topic = \"mqtt/local\"}\n");
kafka_producer_old_hocon(_WithLocalTopic = false) ->
kafka_producer_old_hocon("mqtt {}\n");
kafka_producer_old_hocon(MQTTConfig) when is_list(MQTTConfig) ->
"""
bridges.kafka {
myproducer {
authentication = \"none\"
bootstrap_hosts = \"toxiproxy:9292\"
connect_timeout = \"5s\"
metadata_request_timeout = \"5s\"
min_metadata_refresh_interval = \"3s\"
producer {
kafka {
buffer {
memory_overload_protection = false
mode = \"memory\"
per_partition_limit = \"2GB\"
segment_bytes = \"100MB\"
}
compression = \"no_compression\"
max_batch_bytes = \"896KB\"
max_inflight = 10
message {
key = \"${.clientid}\"
timestamp = \"${.timestamp}\"
value = \"${.}\"
}
partition_count_refresh_interval = \"60s\"
partition_strategy = \"random\"
required_acks = \"all_isr\"
topic = \"test-topic-two-partitions\"
}
""" ++ MQTTConfig ++
"""
}
socket_opts {
nodelay = true
recbuf = \"1024KB\"
sndbuf = \"1024KB\"
}
ssl {enable = false, verify = \"verify_peer\"}
}
}
""".
[
"bridges.kafka {"
"\n myproducer {"
"\n authentication = \"none\""
"\n bootstrap_hosts = \"toxiproxy:9292\""
"\n connect_timeout = \"5s\""
"\n metadata_request_timeout = \"5s\""
"\n min_metadata_refresh_interval = \"3s\""
"\n producer {"
"\n kafka {"
"\n buffer {"
"\n memory_overload_protection = false"
"\n mode = \"memory\""
"\n per_partition_limit = \"2GB\""
"\n segment_bytes = \"100MB\""
"\n }"
"\n compression = \"no_compression\""
"\n max_batch_bytes = \"896KB\""
"\n max_inflight = 10"
"\n message {"
"\n key = \"${.clientid}\""
"\n timestamp = \"${.timestamp}\""
"\n value = \"${.}\""
"\n }"
"\n partition_count_refresh_interval = \"60s\""
"\n partition_strategy = \"random\""
"\n required_acks = \"all_isr\""
"\n topic = \"test-topic-two-partitions\""
"\n }",
MQTTConfig,
"\n }"
"\n socket_opts {"
"\n nodelay = true"
"\n recbuf = \"1024KB\""
"\n sndbuf = \"1024KB\""
"\n }"
"\n ssl {enable = false, verify = \"verify_peer\"}"
"\n }"
"\n}"
].
kafka_producer_new_hocon() ->
""
"\n"
"bridges.kafka {\n"
" myproducer {\n"
" authentication = \"none\"\n"
" bootstrap_hosts = \"toxiproxy:9292\"\n"
" connect_timeout = \"5s\"\n"
" metadata_request_timeout = \"5s\"\n"
" min_metadata_refresh_interval = \"3s\"\n"
" kafka {\n"
" buffer {\n"
" memory_overload_protection = false\n"
" mode = \"memory\"\n"
" per_partition_limit = \"2GB\"\n"
" segment_bytes = \"100MB\"\n"
" }\n"
" compression = \"no_compression\"\n"
" max_batch_bytes = \"896KB\"\n"
" max_inflight = 10\n"
" message {\n"
" key = \"${.clientid}\"\n"
" timestamp = \"${.timestamp}\"\n"
" value = \"${.}\"\n"
" }\n"
" partition_count_refresh_interval = \"60s\"\n"
" partition_strategy = \"random\"\n"
" required_acks = \"all_isr\"\n"
" topic = \"test-topic-two-partitions\"\n"
" }\n"
" local_topic = \"mqtt/local\"\n"
" socket_opts {\n"
" nodelay = true\n"
" recbuf = \"1024KB\"\n"
" sndbuf = \"1024KB\"\n"
" }\n"
" ssl {enable = false, verify = \"verify_peer\"}\n"
" resource_opts {\n"
" health_check_interval = 10s\n"
" }\n"
" }\n"
"}\n"
"".
"bridges.kafka {"
"\n myproducer {"
"\n authentication = \"none\""
"\n bootstrap_hosts = \"toxiproxy:9292\""
"\n connect_timeout = \"5s\""
"\n metadata_request_timeout = \"5s\""
"\n min_metadata_refresh_interval = \"3s\""
"\n kafka {"
"\n buffer {"
"\n memory_overload_protection = false"
"\n mode = \"memory\""
"\n per_partition_limit = \"2GB\""
"\n segment_bytes = \"100MB\""
"\n }"
"\n compression = \"no_compression\""
"\n max_batch_bytes = \"896KB\""
"\n max_inflight = 10"
"\n message {"
"\n key = \"${.clientid}\""
"\n timestamp = \"${.timestamp}\""
"\n value = \"${.}\""
"\n }"
"\n partition_count_refresh_interval = \"60s\""
"\n partition_strategy = \"random\""
"\n required_acks = \"all_isr\""
"\n topic = \"test-topic-two-partitions\""
"\n }"
"\n local_topic = \"mqtt/local\""
"\n socket_opts {"
"\n nodelay = true"
"\n recbuf = \"1024KB\""
"\n sndbuf = \"1024KB\""
"\n }"
"\n ssl {enable = false, verify = \"verify_peer\"}"
"\n resource_opts {"
"\n health_check_interval = 10s"
"\n }"
"\n }"
"\n}".
%% erlfmt-ignore
kafka_consumer_hocon() ->
"""
bridges.kafka_consumer.my_consumer {
enable = true
bootstrap_hosts = \"kafka-1.emqx.net:9292\"
connect_timeout = 5s
min_metadata_refresh_interval = 3s
metadata_request_timeout = 5s
authentication = {
mechanism = plain
username = emqxuser
password = password
}
kafka {
max_batch_bytes = 896KB
max_rejoin_attempts = 5
offset_commit_interval_seconds = 3s
offset_reset_policy = latest
}
topic_mapping = [
{
kafka_topic = \"kafka-topic-1\"
mqtt_topic = \"mqtt/topic/1\"
qos = 1
payload_template = \"${.}\"
},
{
kafka_topic = \"kafka-topic-2\"
mqtt_topic = \"mqtt/topic/2\"
qos = 2
payload_template = \"v = ${.value}\"
}
]
key_encoding_mode = none
value_encoding_mode = none
ssl {
enable = false
verify = verify_none
server_name_indication = \"auto\"
}
resource_opts {
health_check_interval = 10s
}
}
""".
"bridges.kafka_consumer.my_consumer {"
"\n enable = true"
"\n bootstrap_hosts = \"kafka-1.emqx.net:9292\""
"\n connect_timeout = 5s"
"\n min_metadata_refresh_interval = 3s"
"\n metadata_request_timeout = 5s"
"\n authentication = {"
"\n mechanism = plain"
"\n username = emqxuser"
"\n password = password"
"\n }"
"\n kafka {"
"\n max_batch_bytes = 896KB"
"\n max_rejoin_attempts = 5"
"\n offset_commit_interval_seconds = 3s"
"\n offset_reset_policy = latest"
"\n }"
"\n topic_mapping = ["
"\n {"
"\n kafka_topic = \"kafka-topic-1\""
"\n mqtt_topic = \"mqtt/topic/1\""
"\n qos = 1"
"\n payload_template = \"${.}\""
"\n },"
"\n {"
"\n kafka_topic = \"kafka-topic-2\""
"\n mqtt_topic = \"mqtt/topic/2\""
"\n qos = 2"
"\n payload_template = \"v = ${.value}\""
"\n }"
"\n ]"
"\n key_encoding_mode = none"
"\n value_encoding_mode = none"
"\n ssl {"
"\n enable = false"
"\n verify = verify_none"
"\n server_name_indication = \"auto\""
"\n }"
"\n resource_opts {"
"\n health_check_interval = 10s"
"\n }"
"\n }".
%% assert compatibility
bridge_schema_json_test() ->

2
apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_kinesis, [
{description, "EMQX Enterprise Amazon Kinesis Bridge"},
{vsn, "0.1.2"},
{vsn, "0.1.3"},
{registered, []},
{applications, [
kernel,

6
apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl
View File

@ -62,12 +62,10 @@ fields(connector_config) ->
}
)},
{aws_secret_access_key,
mk(
binary(),
emqx_schema_secret:mk(
#{
required => true,
desc => ?DESC("aws_secret_access_key"),
sensitive => true
desc => ?DESC("aws_secret_access_key")
}
)},
{endpoint,

12
apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl
View File

@ -97,7 +97,13 @@ init(#{
partition_key => PartitionKey,
stream_name => StreamName
},
New =
%% TODO: teach `erlcloud` to to accept 0-arity closures as passwords.
ok = erlcloud_config:configure(
to_str(AwsAccessKey),
to_str(emqx_secret:unwrap(AwsSecretAccessKey)),
Host,
Port,
Scheme,
fun(AccessKeyID, SecretAccessKey, HostAddr, HostPort, ConnectionScheme) ->
Config0 = erlcloud_kinesis:new(
AccessKeyID,
@ -107,9 +113,7 @@ init(#{
ConnectionScheme ++ "://"
),
Config0#aws_config{retry_num = MaxRetries}
end,
erlcloud_config:configure(
to_str(AwsAccessKey), to_str(AwsSecretAccessKey), Host, Port, Scheme, New
end
),
% check the connection
case erlcloud_kinesis:list_streams() of

2
apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl
View File

@ -15,7 +15,7 @@
-type config() :: #{
aws_access_key_id := binary(),
aws_secret_access_key := binary(),
aws_secret_access_key := emqx_secret:t(binary()),
endpoint := binary(),
stream_name := binary(),
partition_key := binary(),

61
apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl
View File

@ -11,10 +11,11 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(PRODUCER, emqx_bridge_kinesis_impl_producer).
-define(BRIDGE_TYPE, kinesis_producer).
-define(BRIDGE_TYPE_BIN, <<"kinesis_producer">>).
-define(KINESIS_PORT, 4566).
-define(KINESIS_ACCESS_KEY, "aws_access_key_id").
-define(KINESIS_SECRET_KEY, "aws_secret_access_key").
-define(TOPIC, <<"t/topic">>).
%%------------------------------------------------------------------------------
@ -38,6 +39,8 @@ init_per_suite(Config) ->
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy.emqx.net"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
ProxyName = "kinesis",
SecretFile = filename:join(?config(priv_dir, Config), "secret"),
ok = file:write_file(SecretFile, <<?KINESIS_SECRET_KEY>>),
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]),
{ok, _} = application:ensure_all_started(emqx_connector),
@ -46,6 +49,7 @@ init_per_suite(Config) ->
{proxy_host, ProxyHost},
{proxy_port, ProxyPort},
{kinesis_port, ?KINESIS_PORT},
{kinesis_secretfile, SecretFile},
{proxy_name, ProxyName}
| Config
].
@ -130,6 +134,7 @@ kinesis_config(Config) ->
Scheme = proplists:get_value(connection_scheme, Config, "http"),
ProxyHost = proplists:get_value(proxy_host, Config),
KinesisPort = proplists:get_value(kinesis_port, Config),
SecretFile = proplists:get_value(kinesis_secretfile, Config),
BatchSize = proplists:get_value(batch_size, Config, 100),
BatchTime = proplists:get_value(batch_time, Config, <<"500ms">>),
PayloadTemplate = proplists:get_value(payload_template, Config, "${payload}"),
@ -140,29 +145,32 @@ kinesis_config(Config) ->
Name = <<(atom_to_binary(?MODULE))/binary, (GUID)/binary>>,
ConfigString =
io_lib:format(
"bridges.kinesis_producer.~s {\n"
" enable = true\n"
" aws_access_key_id = \"aws_access_key_id\"\n"
" aws_secret_access_key = \"aws_secret_access_key\"\n"
" endpoint = \"~s://~s:~b\"\n"
" stream_name = \"~s\"\n"
" partition_key = \"~s\"\n"
" payload_template = \"~s\"\n"
" max_retries = ~b\n"
" pool_size = 1\n"
" resource_opts = {\n"
" health_check_interval = \"3s\"\n"
" request_ttl = 30s\n"
" resume_interval = 1s\n"
" metrics_flush_interval = \"700ms\"\n"
" worker_pool_size = 1\n"
" query_mode = ~s\n"
" batch_size = ~b\n"
" batch_time = \"~s\"\n"
" }\n"
"}\n",
"bridges.kinesis_producer.~s {"
"\n enable = true"
"\n aws_access_key_id = ~p"
"\n aws_secret_access_key = ~p"
"\n endpoint = \"~s://~s:~b\""
"\n stream_name = \"~s\""
"\n partition_key = \"~s\""
"\n payload_template = \"~s\""
"\n max_retries = ~b"
"\n pool_size = 1"
"\n resource_opts = {"
"\n health_check_interval = \"3s\""
"\n request_ttl = 30s"
"\n resume_interval = 1s"
"\n metrics_flush_interval = \"700ms\""
"\n worker_pool_size = 1"
"\n query_mode = ~s"
"\n batch_size = ~b"
"\n batch_time = \"~s\""
"\n }"
"\n }",
[
Name,
?KINESIS_ACCESS_KEY,
%% NOTE: using file-based secrets with HOCON configs.
"file://" ++ SecretFile,
Scheme,
ProxyHost,
KinesisPort,
@ -203,9 +211,6 @@ delete_bridge(Config) ->
ct:pal("deleting bridge ~p", [{Type, Name}]),
emqx_bridge:remove(Type, Name).
create_bridge_http(Config) ->
create_bridge_http(Config, _KinesisConfigOverrides = #{}).
create_bridge_http(Config, KinesisConfigOverrides) ->
TypeBin = ?BRIDGE_TYPE_BIN,
Name = ?config(kinesis_name, Config),
@ -489,7 +494,11 @@ to_bin(Str) when is_list(Str) ->
%%------------------------------------------------------------------------------
t_create_via_http(Config) ->
?assertMatch({ok, _}, create_bridge_http(Config)),
Overrides = #{
%% NOTE: using literal secret with HTTP API requests.
<<"aws_secret_access_key">> => <<?KINESIS_SECRET_KEY>>
},
?assertMatch({ok, _}, create_bridge_http(Config, Overrides)),
ok.
t_start_failed_then_fix(Config) ->

2
apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_mongodb, [
{description, "EMQX Enterprise MongoDB Bridge"},
{vsn, "0.2.1"},
{vsn, "0.2.2"},
{registered, []},
{applications, [
kernel,

3
apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl
View File

@ -6,9 +6,6 @@
-behaviour(emqx_resource).
-include_lib("emqx_connector/include/emqx_connector_tables.hrl").
-include_lib("emqx_resource/include/emqx_resource.hrl").
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").

137
apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl
View File

@ -11,6 +11,8 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-import(emqx_utils_conv, [bin/1]).
%%------------------------------------------------------------------------------
%% CT boilerplate
%%------------------------------------------------------------------------------
@ -96,14 +98,27 @@ init_per_group(Type = single, Config) ->
true ->
ok = start_apps(),
emqx_mgmt_api_test_util:init_suite(),
{Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, Config),
%% NOTE: `mongo-single` has auth enabled, see `credentials.env`.
AuthSource = bin(os:getenv("MONGO_AUTHSOURCE", "admin")),
Username = bin(os:getenv("MONGO_USERNAME", "")),
Password = bin(os:getenv("MONGO_PASSWORD", "")),
Passfile = filename:join(?config(priv_dir, Config), "passfile"),
ok = file:write_file(Passfile, Password),
NConfig = [
{mongo_authsource, AuthSource},
{mongo_username, Username},
{mongo_password, Password},
{mongo_passfile, Passfile}
| Config
],
{Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, NConfig),
[
{mongo_host, MongoHost},
{mongo_port, MongoPort},
{mongo_config, MongoConfig},
{mongo_type, Type},
{mongo_name, Name}
| Config
| NConfig
];
false ->
{skip, no_mongo}
@ -121,13 +136,13 @@ end_per_suite(_Config) ->
ok.
init_per_testcase(_Testcase, Config) ->
catch clear_db(Config),
clear_db(Config),
delete_bridge(Config),
snabbkaffe:start_trace(),
Config.
end_per_testcase(_Testcase, Config) ->
catch clear_db(Config),
clear_db(Config),
delete_bridge(Config),
snabbkaffe:stop(),
ok.
@ -175,19 +190,19 @@ mongo_config(MongoHost, MongoPort0, rs = Type, Config) ->
Name = atom_to_binary(?MODULE),
ConfigString =
io_lib:format(
"bridges.mongodb_rs.~s {\n"
" enable = true\n"
" collection = mycol\n"
" replica_set_name = rs0\n"
" servers = [~p]\n"
" w_mode = safe\n"
" use_legacy_protocol = auto\n"
" database = mqtt\n"
" resource_opts = {\n"
" query_mode = ~s\n"
" worker_pool_size = 1\n"
" }\n"
"}",
"bridges.mongodb_rs.~s {"
"\n enable = true"
"\n collection = mycol"
"\n replica_set_name = rs0"
"\n servers = [~p]"
"\n w_mode = safe"
"\n use_legacy_protocol = auto"
"\n database = mqtt"
"\n resource_opts = {"
"\n query_mode = ~s"
"\n worker_pool_size = 1"
"\n }"
"\n }",
[
Name,
Servers,
@ -202,18 +217,18 @@ mongo_config(MongoHost, MongoPort0, sharded = Type, Config) ->
Name = atom_to_binary(?MODULE),
ConfigString =
io_lib:format(
"bridges.mongodb_sharded.~s {\n"
" enable = true\n"
" collection = mycol\n"
" servers = [~p]\n"
" w_mode = safe\n"
" use_legacy_protocol = auto\n"
" database = mqtt\n"
" resource_opts = {\n"
" query_mode = ~s\n"
" worker_pool_size = 1\n"
" }\n"
"}",
"bridges.mongodb_sharded.~s {"
"\n enable = true"
"\n collection = mycol"
"\n servers = [~p]"
"\n w_mode = safe"
"\n use_legacy_protocol = auto"
"\n database = mqtt"
"\n resource_opts = {"
"\n query_mode = ~s"
"\n worker_pool_size = 1"
"\n }"
"\n }",
[
Name,
Servers,
@ -228,21 +243,27 @@ mongo_config(MongoHost, MongoPort0, single = Type, Config) ->
Name = atom_to_binary(?MODULE),
ConfigString =
io_lib:format(
"bridges.mongodb_single.~s {\n"
" enable = true\n"
" collection = mycol\n"
" server = ~p\n"
" w_mode = safe\n"
" use_legacy_protocol = auto\n"
" database = mqtt\n"
" resource_opts = {\n"
" query_mode = ~s\n"
" worker_pool_size = 1\n"
" }\n"
"}",
"bridges.mongodb_single.~s {"
"\n enable = true"
"\n collection = mycol"
"\n server = ~p"
"\n w_mode = safe"
"\n use_legacy_protocol = auto"
"\n database = mqtt"
"\n auth_source = ~s"
"\n username = ~s"
"\n password = \"file://~s\""
"\n resource_opts = {"
"\n query_mode = ~s"
"\n worker_pool_size = 1"
"\n }"
"\n }",
[
Name,
Server,
?config(mongo_authsource, Config),
?config(mongo_username, Config),
?config(mongo_passfile, Config),
QueryMode
]
),
@ -284,8 +305,24 @@ clear_db(Config) ->
Host = ?config(mongo_host, Config),
Port = ?config(mongo_port, Config),
Server = Host ++ ":" ++ integer_to_list(Port),
#{<<"database">> := Db, <<"collection">> := Collection} = ?config(mongo_config, Config),
{ok, Client} = mongo_api:connect(Type, [Server], [], [{database, Db}, {w_mode, unsafe}]),
#{
<<"database">> := Db,
<<"collection">> := Collection
} = ?config(mongo_config, Config),
WorkerOpts = [
{database, Db},
{w_mode, unsafe}
| lists:flatmap(
fun
({mongo_authsource, AS}) -> [{auth_source, AS}];
({mongo_username, User}) -> [{login, User}];
({mongo_password, Pass}) -> [{password, Pass}];
(_) -> []
end,
Config
)
],
{ok, Client} = mongo_api:connect(Type, [Server], [], WorkerOpts),
{true, _} = mongo_api:delete(Client, Collection, _Selector = #{}),
mongo_api:disconnect(Client).
@ -386,13 +423,21 @@ t_setup_via_config_and_publish(Config) ->
ok.
t_setup_via_http_api_and_publish(Config) ->
Type = mongo_type_bin(?config(mongo_type, Config)),
Type = ?config(mongo_type, Config),
Name = ?config(mongo_name, Config),
MongoConfig0 = ?config(mongo_config, Config),
MongoConfig = MongoConfig0#{
MongoConfig1 = MongoConfig0#{
<<"name">> => Name,
<<"type">> => Type
<<"type">> => mongo_type_bin(Type)
},
MongoConfig =
case Type of
single ->
%% NOTE: using literal password with HTTP API requests.
MongoConfig1#{<<"password">> => ?config(mongo_password, Config)};
_ ->
MongoConfig1
end,
?assertMatch(
{ok, _},
create_bridge_http(MongoConfig)

2
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge_mqtt, [
{description, "EMQX MQTT Broker Bridge"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, []},
{applications, [
kernel,

12
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl
View File

@ -96,7 +96,7 @@ choose_ingress_pool_size(
#{remote := #{topic := RemoteTopic}, pool_size := PoolSize}
) ->
case emqx_topic:parse(RemoteTopic) of
{_Filter, #{share := _Name}} ->
{#share{} = _Filter, _SubOpts} ->
% NOTE: this is shared subscription, many workers may subscribe
PoolSize;
{_Filter, #{}} when PoolSize > 1 ->
@ -326,7 +326,7 @@ mk_client_opts(
],
Config
),
Options#{
mk_client_opt_password(Options#{
hosts => [HostPort],
clientid => clientid(ResourceId, ClientScope, Config),
connect_timeout => 30,
@ -334,7 +334,13 @@ mk_client_opts(
force_ping => true,
ssl => EnableSsl,
ssl_opts => maps:to_list(maps:remove(enable, Ssl))
}.
}).
mk_client_opt_password(Options = #{password := Secret}) ->
%% TODO: Teach `emqtt` to accept 0-arity closures as passwords.
Options#{password := emqx_secret:unwrap(Secret)};
mk_client_opt_password(Options) ->
Options.
ms_to_s(Ms) ->
erlang:ceil(Ms / 1000).

8
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl
View File

@ -99,13 +99,9 @@ fields("server_configs") ->
}
)},
{password,
mk(
binary(),
emqx_schema_secret:mk(
#{
format => <<"password">>,
sensitive => true,
desc => ?DESC("password"),
converter => fun emqx_schema:password_converter/2
desc => ?DESC("password")
}
)},
{clean_start,

185
apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl
View File

@ -21,13 +21,15 @@
-import(emqx_dashboard_api_test_helpers, [request/4, uri/1]).
-include("emqx/include/emqx.hrl").
-include("emqx/include/emqx_hooks.hrl").
-include("emqx/include/asserts.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% output functions
-export([inspect/3]).
-define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
-define(TYPE_MQTT, <<"mqtt">>).
-define(BRIDGE_NAME_INGRESS, <<"ingress_mqtt_bridge">>).
-define(BRIDGE_NAME_EGRESS, <<"egress_mqtt_bridge">>).
@ -38,14 +40,18 @@
-define(EGRESS_REMOTE_TOPIC, "egress_remote_topic").
-define(EGRESS_LOCAL_TOPIC, "egress_local_topic").
-define(SERVER_CONF(Username), #{
-define(SERVER_CONF, #{
<<"type">> => ?TYPE_MQTT,
<<"server">> => <<"127.0.0.1:1883">>,
<<"username">> => Username,
<<"password">> => <<"">>,
<<"proto_ver">> => <<"v4">>,
<<"ssl">> => #{<<"enable">> => false}
}).
-define(SERVER_CONF(Username, Password), (?SERVER_CONF)#{
<<"username">> => Username,
<<"password">> => Password
}).
-define(INGRESS_CONF, #{
<<"remote">> => #{
<<"topic">> => <<?INGRESS_REMOTE_TOPIC, "/#">>,
@ -129,43 +135,32 @@ suite() ->
[{timetrap, {seconds, 30}}].
init_per_suite(Config) ->
_ = application:load(emqx_conf),
ok = emqx_common_test_helpers:start_apps(
Apps = emqx_cth_suite:start(
[
emqx_conf,
emqx_bridge,
emqx_rule_engine,
emqx_bridge,
emqx_bridge_mqtt,
emqx_dashboard
{emqx_dashboard,
"dashboard {"
"\n listeners.http { bind = 18083 }"
"\n default_username = connector_admin"
"\n default_password = public"
"\n }"}
],
fun set_special_configs/1
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
ok = emqx_common_test_helpers:load_config(
emqx_rule_engine_schema,
<<"rule_engine {rules {}}">>
),
ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
Config.
[{suite_apps, Apps} | Config].
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([
emqx_dashboard,
emqx_bridge_mqtt,
emqx_bridge,
emqx_rule_engine
]),
ok.
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config(<<"connector_admin">>);
set_special_configs(_) ->
ok.
end_per_suite(Config) ->
emqx_cth_suite:stop(?config(suite_apps, Config)).
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
ok = snabbkaffe:start_trace(),
Config.
end_per_testcase(_, _Config) ->
ok = unhook_authenticate(),
clear_resources(),
snabbkaffe:stop(),
ok.
@ -187,14 +182,86 @@ clear_resources() ->
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_conf_bridge_authn_anonymous(_) ->
ok = hook_authenticate(),
{ok, 201, _Bridge} = request(
post,
uri(["bridges"]),
?SERVER_CONF#{
<<"name">> => <<"t_conf_bridge_anonymous">>,
<<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1}
}
),
?assertReceive(
{authenticate, #{username := undefined, password := undefined}}
).
t_conf_bridge_authn_password(_) ->
Username1 = <<"user1">>,
Password1 = <<"from-here">>,
ok = hook_authenticate(),
{ok, 201, _Bridge1} = request(
post,
uri(["bridges"]),
?SERVER_CONF(Username1, Password1)#{
<<"name">> => <<"t_conf_bridge_authn_password">>,
<<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1}
}
),
?assertReceive(
{authenticate, #{username := Username1, password := Password1}}
).
t_conf_bridge_authn_passfile(Config) ->
DataDir = ?config(data_dir, Config),
Username2 = <<"user2">>,
PasswordFilename = filename:join(DataDir, "password"),
Password2 = <<"from-there">>,
ok = hook_authenticate(),
{ok, 201, _Bridge2} = request(
post,
uri(["bridges"]),
?SERVER_CONF(Username2, iolist_to_binary(["file://", PasswordFilename]))#{
<<"name">> => <<"t_conf_bridge_authn_passfile">>,
<<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1}
}
),
?assertReceive(
{authenticate, #{username := Username2, password := Password2}}
),
?assertMatch(
{ok, 201, #{
<<"status">> := <<"disconnected">>,
<<"status_reason">> := <<"#{msg => failed_to_read_secret_file", _/bytes>>
}},
request_json(
post,
uri(["bridges"]),
?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{
<<"name">> => <<"t_conf_bridge_authn_no_passfile">>
}
)
).
hook_authenticate() ->
emqx_hooks:add('client.authenticate', {?MODULE, authenticate, [self()]}, ?HP_HIGHEST).
unhook_authenticate() ->
emqx_hooks:del('client.authenticate', {?MODULE, authenticate}).
authenticate(Credential, _, TestRunnerPid) ->
_ = TestRunnerPid ! {authenticate, Credential},
ignore.
%%------------------------------------------------------------------------------
t_mqtt_conn_bridge_ingress(_) ->
User1 = <<"user1">>,
%% create an MQTT bridge, using POST
{ok, 201, Bridge} = request(
post,
uri(["bridges"]),
ServerConf = ?SERVER_CONF(User1)#{
<<"type">> => ?TYPE_MQTT,
ServerConf = ?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_INGRESS,
<<"ingress">> => ?INGRESS_CONF
}
@ -249,7 +316,6 @@ t_mqtt_conn_bridge_ingress(_) ->
ok.
t_mqtt_conn_bridge_ingress_full_context(_Config) ->
User1 = <<"user1">>,
IngressConf =
emqx_utils_maps:deep_merge(
?INGRESS_CONF,
@ -258,8 +324,7 @@ t_mqtt_conn_bridge_ingress_full_context(_Config) ->
{ok, 201, _Bridge} = request(
post,
uri(["bridges"]),
?SERVER_CONF(User1)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_INGRESS,
<<"ingress">> => IngressConf
}
@ -297,8 +362,7 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) ->
Ns = lists:seq(1, 10),
BridgeName = atom_to_binary(?FUNCTION_NAME),
BridgeID = create_bridge(
?SERVER_CONF(<<>>)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => BridgeName,
<<"ingress">> => #{
<<"pool_size">> => PoolSize,
@ -337,8 +401,7 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) ->
t_mqtt_egress_bridge_ignores_clean_start(_) ->
BridgeName = atom_to_binary(?FUNCTION_NAME),
BridgeID = create_bridge(
?SERVER_CONF(<<"user1">>)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => BridgeName,
<<"egress">> => ?EGRESS_CONF,
<<"clean_start">> => false
@ -366,8 +429,7 @@ t_mqtt_egress_bridge_ignores_clean_start(_) ->
t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) ->
BridgeName = atom_to_binary(?FUNCTION_NAME),
BridgeID = create_bridge(
?SERVER_CONF(<<"user1">>)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => BridgeName,
<<"ingress">> => emqx_utils_maps:deep_merge(
?INGRESS_CONF,
@ -392,9 +454,8 @@ t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) ->
ok.
t_mqtt_conn_bridge_ingress_no_payload_template(_) ->
User1 = <<"user1">>,
BridgeIDIngress = create_bridge(
?SERVER_CONF(User1)#{
?SERVER_CONF#{
<<"type">> => ?TYPE_MQTT,
<<"name">> => ?BRIDGE_NAME_INGRESS,
<<"ingress">> => ?INGRESS_CONF_NO_PAYLOAD_TEMPLATE
@ -428,10 +489,8 @@ t_mqtt_conn_bridge_ingress_no_payload_template(_) ->
t_mqtt_conn_bridge_egress(_) ->
%% then we add a mqtt connector, using POST
User1 = <<"user1">>,
BridgeIDEgress = create_bridge(
?SERVER_CONF(User1)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_EGRESS,
<<"egress">> => ?EGRESS_CONF
}
@ -473,11 +532,8 @@ t_mqtt_conn_bridge_egress(_) ->
t_mqtt_conn_bridge_egress_no_payload_template(_) ->
%% then we add a mqtt connector, using POST
User1 = <<"user1">>,
BridgeIDEgress = create_bridge(
?SERVER_CONF(User1)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_EGRESS,
<<"egress">> => ?EGRESS_CONF_NO_PAYLOAD_TEMPLATE
}
@ -520,11 +576,9 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) ->
ok.
t_egress_custom_clientid_prefix(_Config) ->
User1 = <<"user1">>,
BridgeIDEgress = create_bridge(
?SERVER_CONF(User1)#{
?SERVER_CONF#{
<<"clientid_prefix">> => <<"my-custom-prefix">>,
<<"type">> => ?TYPE_MQTT,
<<"name">> => ?BRIDGE_NAME_EGRESS,
<<"egress">> => ?EGRESS_CONF
}
@ -545,17 +599,14 @@ t_egress_custom_clientid_prefix(_Config) ->
ok.
t_mqtt_conn_bridge_ingress_and_egress(_) ->
User1 = <<"user1">>,
BridgeIDIngress = create_bridge(
?SERVER_CONF(User1)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_INGRESS,
<<"ingress">> => ?INGRESS_CONF
}
),
BridgeIDEgress = create_bridge(
?SERVER_CONF(User1)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_EGRESS,
<<"egress">> => ?EGRESS_CONF
}
@ -627,8 +678,7 @@ t_mqtt_conn_bridge_ingress_and_egress(_) ->
t_ingress_mqtt_bridge_with_rules(_) ->
BridgeIDIngress = create_bridge(
?SERVER_CONF(<<"user1">>)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_INGRESS,
<<"ingress">> => ?INGRESS_CONF
}
@ -712,8 +762,7 @@ t_ingress_mqtt_bridge_with_rules(_) ->
t_egress_mqtt_bridge_with_rules(_) ->
BridgeIDEgress = create_bridge(
?SERVER_CONF(<<"user1">>)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_EGRESS,
<<"egress">> => ?EGRESS_CONF
}
@ -789,10 +838,8 @@ t_egress_mqtt_bridge_with_rules(_) ->
t_mqtt_conn_bridge_egress_reconnect(_) ->
%% then we add a mqtt connector, using POST
User1 = <<"user1">>,
BridgeIDEgress = create_bridge(
?SERVER_CONF(User1)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_EGRESS,
<<"egress">> => ?EGRESS_CONF,
<<"resource_opts">> => #{
@ -897,10 +944,8 @@ t_mqtt_conn_bridge_egress_reconnect(_) ->
ok.
t_mqtt_conn_bridge_egress_async_reconnect(_) ->
User1 = <<"user1">>,
BridgeIDEgress = create_bridge(
?SERVER_CONF(User1)#{
<<"type">> => ?TYPE_MQTT,
?SERVER_CONF#{
<<"name">> => ?BRIDGE_NAME_EGRESS,
<<"egress">> => ?EGRESS_CONF,
<<"resource_opts">> => #{
@ -1018,5 +1063,9 @@ request_bridge_metrics(BridgeID) ->
{ok, 200, BridgeMetrics} = request(get, uri(["bridges", BridgeID, "metrics"]), []),
emqx_utils_json:decode(BridgeMetrics).
request_json(Method, Url, Body) ->
{ok, Code, Response} = request(Method, Url, Body),
{ok, Code, emqx_utils_json:decode(Response)}.
request(Method, Url, Body) ->
request(<<"connector_admin">>, Method, Url, Body).

1
apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE_data/password Normal file
View File

@ -0,0 +1 @@
from-there

24
apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl
View File

@ -21,7 +21,6 @@
"DEFAULT CHARSET=utf8MB4;"
).
-define(SQL_DROP_TABLE, "DROP TABLE mqtt_test").
-define(SQL_DELETE, "DELETE from mqtt_test").
-define(SQL_SELECT, "SELECT payload FROM mqtt_test").
% DB defaults
@ -112,8 +111,8 @@ end_per_suite(_Config) ->
ok.
init_per_testcase(_Testcase, Config) ->
connect_and_drop_table(Config),
connect_and_create_table(Config),
connect_and_clear_table(Config),
delete_bridge(Config),
snabbkaffe:start_trace(),
Config.
@ -122,9 +121,7 @@ end_per_testcase(_Testcase, Config) ->
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
connect_and_clear_table(Config),
ok = snabbkaffe:stop(),
delete_bridge(Config),
emqx_common_test_helpers:call_janitor(),
ok.
@ -323,9 +320,6 @@ connect_and_create_table(Config) ->
connect_and_drop_table(Config) ->
query_direct_mysql(Config, ?SQL_DROP_TABLE).
connect_and_clear_table(Config) ->
query_direct_mysql(Config, ?SQL_DELETE).
connect_and_get_payload(Config) ->
query_direct_mysql(Config, ?SQL_SELECT).
@ -777,8 +771,6 @@ t_table_removed(Config) ->
Name = ?config(mysql_name, Config),
BridgeType = ?config(mysql_bridge_type, Config),
ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
?check_trace(
begin
connect_and_create_table(Config),
?assertMatch({ok, _}, create_bridge(Config)),
?retry(
@ -792,14 +784,9 @@ t_table_removed(Config) ->
Timeout = 1000,
?assertMatch(
{error,
{unrecoverable_error,
{1146, <<"42S02">>, <<"Table 'mqtt.mqtt_test' doesn't exist">>}}},
{unrecoverable_error, {1146, <<"42S02">>, <<"Table 'mqtt.mqtt_test' doesn't exist">>}}},
sync_query_resource(Config, {send_message, SentData, [], Timeout})
),
ok
end,
[]
),
ok.
t_nested_payload_template(Config) ->
@ -807,9 +794,6 @@ t_nested_payload_template(Config) ->
BridgeType = ?config(mysql_bridge_type, Config),
ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
Value = integer_to_binary(erlang:unique_integer()),
?check_trace(
begin
connect_and_create_table(Config),
{ok, _} = create_bridge(
Config,
#{
@ -837,8 +821,4 @@ t_nested_payload_template(Config) ->
{ok, [<<"payload">>], [[Value]]},
connect_and_get_payload(Config)
),
ok
end,
[]
),
ok.

4
apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl
View File

@ -16,7 +16,6 @@
-define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_oracle, emqx_bridge_oracle]).
-define(SID, "XE").
-define(RULE_TOPIC, "mqtt/rule").
% -define(RULE_TOPIC_BIN, <<?RULE_TOPIC>>).
%%------------------------------------------------------------------------------
%% CT boilerplate
@ -33,9 +32,6 @@ groups() ->
{plain, AllTCs}
].
only_once_tests() ->
[t_create_via_http].
init_per_suite(Config) ->
Config.

46
apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl
View File

@ -183,31 +183,33 @@ pgsql_config(BridgeType, Config) ->
end,
QueryMode = ?config(query_mode, Config),
TlsEnabled = ?config(enable_tls, Config),
%% NOTE: supplying password through a file here, to verify that it works.
Password = create_passfile(BridgeType, Config),
ConfigString =
io_lib:format(
"bridges.~s.~s {\n"
" enable = true\n"
" server = ~p\n"
" database = ~p\n"
" username = ~p\n"
" password = ~p\n"
" sql = ~p\n"
" resource_opts = {\n"
" request_ttl = 500ms\n"
" batch_size = ~b\n"
" query_mode = ~s\n"
" }\n"
" ssl = {\n"
" enable = ~w\n"
" }\n"
"}",
"bridges.~s.~s {"
"\n enable = true"
"\n server = ~p"
"\n database = ~p"
"\n username = ~p"
"\n password = ~p"
"\n sql = ~p"
"\n resource_opts = {"
"\n request_ttl = 500ms"
"\n batch_size = ~b"
"\n query_mode = ~s"
"\n }"
"\n ssl = {"
"\n enable = ~w"
"\n }"
"\n }",
[
BridgeType,
Name,
Server,
?PGSQL_DATABASE,
?PGSQL_USERNAME,
?PGSQL_PASSWORD,
Password,
?SQL_BRIDGE,
BatchSize,
QueryMode,
@ -216,6 +218,12 @@ pgsql_config(BridgeType, Config) ->
),
{Name, parse_and_check(ConfigString, BridgeType, Name)}.
create_passfile(BridgeType, Config) ->
Filename = binary_to_list(BridgeType) ++ ".passfile",
Filepath = filename:join(?config(priv_dir, Config), Filename),
ok = file:write_file(Filepath, ?PGSQL_PASSWORD),
"file://" ++ Filepath.
parse_and_check(ConfigString, BridgeType, Name) ->
{ok, RawConf} = hocon:binary(ConfigString, #{format => map}),
hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}),
@ -379,7 +387,9 @@ t_setup_via_http_api_and_publish(Config) ->
QueryMode = ?config(query_mode, Config),
PgsqlConfig = PgsqlConfig0#{
<<"name">> => Name,
<<"type">> => BridgeType
<<"type">> => BridgeType,
%% NOTE: using literal passwords with HTTP API requests.
<<"password">> => <<?PGSQL_PASSWORD>>
},
?assertMatch(
{ok, _},

2
apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_pulsar, [
{description, "EMQX Pulsar Bridge"},
{vsn, "0.1.7"},
{vsn, "0.1.8"},
{registered, []},
{applications, [
kernel,

12
apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl
View File

@ -170,21 +170,17 @@ fields(auth_basic) ->
[
{username, mk(binary(), #{required => true, desc => ?DESC("auth_basic_username")})},
{password,
mk(binary(), #{
emqx_schema_secret:mk(#{
required => true,
desc => ?DESC("auth_basic_password"),
sensitive => true,
converter => fun emqx_schema:password_converter/2
desc => ?DESC("auth_basic_password")
})}
];
fields(auth_token) ->
[
{jwt,
mk(binary(), #{
emqx_schema_secret:mk(#{
required => true,
desc => ?DESC("auth_token_jwt"),
sensitive => true,
converter => fun emqx_schema:password_converter/2
desc => ?DESC("auth_token_jwt")
})}
];
fields("get_" ++ Type) ->

7
apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
View File

@ -78,7 +78,6 @@ query_mode(_Config) ->
-spec on_start(resource_id(), config()) -> {ok, state()}.
on_start(InstanceId, Config) ->
#{
authentication := _Auth,
bridge_name := BridgeName,
servers := Servers0,
ssl := SSL
@ -263,12 +262,14 @@ conn_opts(#{authentication := none}) ->
#{};
conn_opts(#{authentication := #{username := Username, password := Password}}) ->
#{
auth_data => iolist_to_binary([Username, <<":">>, Password]),
%% TODO: teach `pulsar` to accept 0-arity closures as passwords.
auth_data => iolist_to_binary([Username, <<":">>, emqx_secret:unwrap(Password)]),
auth_method_name => <<"basic">>
};
conn_opts(#{authentication := #{jwt := JWT}}) ->
#{
auth_data => JWT,
%% TODO: teach `pulsar` to accept 0-arity closures as passwords.
auth_data => emqx_secret:unwrap(JWT),
auth_method_name => <<"token">>
}.

11
apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl
View File

@ -74,7 +74,7 @@ fields(config) ->
desc => ?DESC("username")
}
)},
{password, fun emqx_connector_schema_lib:password_required/1},
{password, emqx_connector_schema_lib:password_field(#{required => true})},
{pool_size,
hoconsc:mk(
typerefl:pos_integer(),
@ -196,7 +196,6 @@ on_start(
#{
pool_size := PoolSize,
payload_template := PayloadTemplate,
password := Password,
delivery_mode := InitialDeliveryMode
} = InitialConfig
) ->
@ -206,7 +205,6 @@ on_start(
persistent -> 2
end,
Config = InitialConfig#{
password => emqx_secret:wrap(Password),
delivery_mode => DeliveryMode
},
?SLOG(info, #{
@ -242,13 +240,11 @@ on_start(
ok ->
{ok, State};
{error, Reason} ->
LogMessage =
#{
?SLOG(info, #{
msg => "rabbitmq_connector_start_failed",
error_reason => Reason,
config => emqx_utils:redact(Config)
},
?SLOG(info, LogMessage),
}),
{error, Reason}
end.
@ -321,6 +317,7 @@ create_rabbitmq_connection_and_channel(Config) ->
heartbeat := Heartbeat,
wait_for_publish_confirmations := WaitForPublishConfirmations
} = Config,
%% TODO: teach `amqp` to accept 0-arity closures as passwords.
Password = emqx_secret:unwrap(WrappedPassword),
SSLOptions =
case maps:get(ssl, Config, #{}) of

72
apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl
View File

@ -10,6 +10,7 @@
-include("emqx_connector.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("stdlib/include/assert.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("amqp_client/include/amqp_client.hrl").
%% This test SUITE requires a running RabbitMQ instance. If you don't want to
@ -26,6 +27,9 @@ rabbit_mq_host() ->
rabbit_mq_port() ->
5672.
rabbit_mq_password() ->
<<"guest">>.
rabbit_mq_exchange() ->
<<"test_exchange">>.
@ -45,12 +49,12 @@ init_per_suite(Config) ->
)
of
true ->
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
{ok, _} = application:ensure_all_started(emqx_connector),
{ok, _} = application:ensure_all_started(amqp_client),
Apps = emqx_cth_suite:start(
[emqx_conf, emqx_connector, emqx_bridge_rabbitmq],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
ChannelConnection = setup_rabbit_mq_exchange_and_queue(),
[{channel_connection, ChannelConnection} | Config];
[{channel_connection, ChannelConnection}, {suite_apps, Apps} | Config];
false ->
case os:getenv("IS_CI") of
"yes" ->
@ -106,13 +110,11 @@ end_per_suite(Config) ->
connection := Connection,
channel := Channel
} = get_channel_connection(Config),
ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
_ = application:stop(emqx_connector),
%% Close the channel
ok = amqp_channel:close(Channel),
%% Close the connection
ok = amqp_connection:close(Connection).
ok = amqp_connection:close(Connection),
ok = emqx_cth_suite:stop(?config(suite_apps, Config)).
% %%------------------------------------------------------------------------------
% %% Testcases
@ -125,23 +127,31 @@ t_lifecycle(Config) ->
Config
).
t_start_passfile(Config) ->
ResourceID = atom_to_binary(?FUNCTION_NAME),
PasswordFilename = filename:join(?config(priv_dir, Config), "passfile"),
ok = file:write_file(PasswordFilename, rabbit_mq_password()),
InitialConfig = rabbitmq_config(#{
password => iolist_to_binary(["file://", PasswordFilename])
}),
?assertMatch(
#{status := connected},
create_local_resource(ResourceID, check_config(InitialConfig))
),
?assertEqual(
ok,
emqx_resource:remove_local(ResourceID)
).
perform_lifecycle_check(ResourceID, InitialConfig, TestConfig) ->
#{
channel := Channel
} = get_channel_connection(TestConfig),
{ok, #{config := CheckedConfig}} =
emqx_resource:check_config(emqx_bridge_rabbitmq_connector, InitialConfig),
{ok, #{
CheckedConfig = check_config(InitialConfig),
#{
state := #{poolname := PoolName} = State,
status := InitialStatus
}} =
emqx_resource:create_local(
ResourceID,
?CONNECTOR_RESOURCE_GROUP,
emqx_bridge_rabbitmq_connector,
CheckedConfig,
#{}
),
} = create_local_resource(ResourceID, CheckedConfig),
?assertEqual(InitialStatus, connected),
%% Instance should match the state and status of the just started resource
{ok, ?CONNECTOR_RESOURCE_GROUP, #{
@ -184,6 +194,21 @@ perform_lifecycle_check(ResourceID, InitialConfig, TestConfig) ->
% %% Helpers
% %%------------------------------------------------------------------------------
check_config(Config) ->
{ok, #{config := CheckedConfig}} =
emqx_resource:check_config(emqx_bridge_rabbitmq_connector, Config),
CheckedConfig.
create_local_resource(ResourceID, CheckedConfig) ->
{ok, Bridge} = emqx_resource:create_local(
ResourceID,
?CONNECTOR_RESOURCE_GROUP,
emqx_bridge_rabbitmq_connector,
CheckedConfig,
#{}
),
Bridge.
perform_query(PoolName, Channel) ->
%% Send message to queue:
ok = emqx_resource:query(PoolName, {query, test_data()}),
@ -216,16 +241,19 @@ receive_simple_test_message(Channel) ->
end.
rabbitmq_config() ->
rabbitmq_config(#{}).
rabbitmq_config(Overrides) ->
Config =
#{
server => rabbit_mq_host(),
port => 5672,
username => <<"guest">>,
password => <<"guest">>,
password => rabbit_mq_password(),
exchange => rabbit_mq_exchange(),
routing_key => rabbit_mq_routing_key()
},
#{<<"config">> => Config}.
#{<<"config">> => maps:merge(Config, Overrides)}.
test_data() ->
#{<<"msg_field">> => <<"Hello">>}.

2
apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_bridge_rocketmq, [
{description, "EMQX Enterprise RocketMQ Bridge"},
{vsn, "0.1.3"},
{vsn, "0.1.4"},
{registered, []},
{applications, [kernel, stdlib, emqx_resource, rocketmq]},
{env, []},

33
apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl
View File

@ -48,13 +48,8 @@ fields(config) ->
binary(),
#{default => <<>>, desc => ?DESC("access_key")}
)},
{secret_key,
mk(
binary(),
#{default => <<>>, desc => ?DESC("secret_key"), sensitive => true}
)},
{security_token,
mk(binary(), #{default => <<>>, desc => ?DESC(security_token), sensitive => true})},
{secret_key, emqx_schema_secret:mk(#{default => <<>>, desc => ?DESC("secret_key")})},
{security_token, emqx_schema_secret:mk(#{default => <<>>, desc => ?DESC(security_token)})},
{sync_timeout,
mk(
emqx_schema:timeout_duration(),
@ -294,21 +289,19 @@ make_producer_opts(
acl_info => emqx_secret:wrap(ACLInfo)
}.
acl_info(<<>>, <<>>, <<>>) ->
acl_info(<<>>, _, _) ->
#{};
acl_info(AccessKey, SecretKey, <<>>) when is_binary(AccessKey), is_binary(SecretKey) ->
#{
acl_info(AccessKey, SecretKey, SecurityToken) when is_binary(AccessKey) ->
Info = #{
access_key => AccessKey,
secret_key => SecretKey
};
acl_info(AccessKey, SecretKey, SecurityToken) when
is_binary(AccessKey), is_binary(SecretKey), is_binary(SecurityToken)
->
#{
access_key => AccessKey,
secret_key => SecretKey,
security_token => SecurityToken
};
secret_key => emqx_maybe:define(emqx_secret:unwrap(SecretKey), <<>>)
},
case emqx_maybe:define(emqx_secret:unwrap(SecurityToken), <<>>) of
<<>> ->
Info;
Token ->
Info#{security_token => Token}
end;
acl_info(_, _, _) ->
#{}.

Some files were not shown because too many files have changed in this diff Show More