Merge pull request #6608 from emqx/merge-5.0-beta.3-to-master

Merge 5.0 beta.3 to master
This commit is contained in:
Zaiming (Stone) Shi 2022-01-04 22:47:27 +01:00 committed by GitHub
commit b5022e5cd6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
141 changed files with 3088 additions and 1717 deletions

View File

@ -61,7 +61,7 @@ jobs:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
repository: emqx/emqx-fvt repository: emqx/emqx-fvt
ref: 1.0.2-dev1 ref: 1.0.3-dev1
path: . path: .
- uses: actions/setup-java@v1 - uses: actions/setup-java@v1
with: with:
@ -93,7 +93,7 @@ jobs:
run: | run: |
/opt/jmeter/bin/jmeter.sh \ /opt/jmeter/bin/jmeter.sh \
-Jjmeter.save.saveservice.output_format=xml -n \ -Jjmeter.save.saveservice.output_format=xml -n \
-t .ci/api-test-suite/${{ matrix.script_name }}.jmx \ -t api-test-suite/${{ matrix.script_name }}.jmx \
-Demqx_ip="127.0.0.1" \ -Demqx_ip="127.0.0.1" \
-l jmeter_logs/${{ matrix.script_name }}.jtl \ -l jmeter_logs/${{ matrix.script_name }}.jtl \
-j jmeter_logs/logs/${{ matrix.script_name }}.log -j jmeter_logs/logs/${{ matrix.script_name }}.log

View File

@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3
export EMQX_DEFAULT_RUNNER = alpine:3.14 export EMQX_DEFAULT_RUNNER = alpine:3.14
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v0.10.0 export EMQX_DASHBOARD_VERSION ?= v0.14.0
export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE := deploy/docker/Dockerfile
export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
ifeq ($(OS),Windows_NT) ifeq ($(OS),Windows_NT)

View File

@ -59,16 +59,33 @@
%% structured logging %% structured logging
-define(SLOG(Level, Data), -define(SLOG(Level, Data),
%% check 'allow' here, only evaluate Data when necessary ?SLOG(Level, Data, #{})).
%% structured logging, meta is for handler's filter.
-define(SLOG(Level, Data, Meta),
%% check 'allow' here, only evaluate Data and Meta when necessary
case logger:allow(Level, ?MODULE) of case logger:allow(Level, ?MODULE) of
true -> true ->
logger:log(Level, (Data), #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} logger:log(Level, (Data), (Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
, line => ?LINE , line => ?LINE
}); }));
false -> false ->
ok ok
end). end).
-define(TRACE_FILTER, emqx_trace_filter).
%% Only evaluate when necessary
-define(TRACE(Event, Msg, Meta),
begin
case persistent_term:get(?TRACE_FILTER, undefined) of
undefined -> ok;
[] -> ok;
List ->
emqx_trace:log(List, Event, Msg, Meta)
end
end).
%% print to 'user' group leader %% print to 'user' group leader
-define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).
-define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)). -define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)).

View File

@ -11,7 +11,7 @@
{deps, {deps,
[ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.2"}}} [ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.2"}}}
, {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
, {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.6"}}}
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}

View File

@ -187,7 +187,7 @@ convert_certs(CertsDir, Config) ->
{ok, SSL} -> {ok, SSL} ->
new_ssl_config(Config, SSL); new_ssl_config(Config, SSL);
{error, Reason} -> {error, Reason} ->
?SLOG(error, Reason#{msg => bad_ssl_config}), ?SLOG(error, Reason#{msg => "bad_ssl_config"}),
throw({bad_ssl_config, Reason}) throw({bad_ssl_config, Reason})
end. end.
@ -199,7 +199,7 @@ convert_certs(CertsDir, NewConfig, OldConfig) ->
ok = emqx_tls_lib:delete_ssl_files(CertsDir, NewSSL1, OldSSL), ok = emqx_tls_lib:delete_ssl_files(CertsDir, NewSSL1, OldSSL),
new_ssl_config(NewConfig, NewSSL1); new_ssl_config(NewConfig, NewSSL1);
{error, Reason} -> {error, Reason} ->
?SLOG(error, Reason#{msg => bad_ssl_config}), ?SLOG(error, Reason#{msg => "bad_ssl_config"}),
throw({bad_ssl_config, Reason}) throw({bad_ssl_config, Reason})
end. end.

View File

@ -37,7 +37,6 @@
, info/1 , info/1
, format/1 , format/1
, parse/1 , parse/1
, to_timestamp/1
]). ]).
%% gen_server callbacks %% gen_server callbacks
@ -53,6 +52,11 @@
-define(BANNED_TAB, ?MODULE). -define(BANNED_TAB, ?MODULE).
-ifdef(TEST).
-compile(export_all).
-compile(nowarn_export_all).
-endif.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Mnesia bootstrap %% Mnesia bootstrap
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -106,32 +110,36 @@ format(#banned{who = Who0,
}. }.
parse(Params) -> parse(Params) ->
Who = pares_who(Params), case pares_who(Params) of
{error, Reason} -> {error, Reason};
Who ->
By = maps:get(<<"by">>, Params, <<"mgmt_api">>), By = maps:get(<<"by">>, Params, <<"mgmt_api">>),
Reason = maps:get(<<"reason">>, Params, <<"">>), Reason = maps:get(<<"reason">>, Params, <<"">>),
At = parse_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)), At = maps:get(<<"at">>, Params, erlang:system_time(second)),
Until = parse_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60), Until = maps:get(<<"until">>, Params, At + 5 * 60),
case Until > erlang:system_time(second) of
true ->
#banned{ #banned{
who = Who, who = Who,
by = By, by = By,
reason = Reason, reason = Reason,
at = At, at = At,
until = Until until = Until
}. };
false ->
{error, "already_expired"}
end
end.
pares_who(#{as := As, who := Who}) -> pares_who(#{as := As, who := Who}) ->
pares_who(#{<<"as">> => As, <<"who">> => Who}); pares_who(#{<<"as">> => As, <<"who">> => Who});
pares_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) -> pares_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) ->
{ok, Peerhost} = inet:parse_address(binary_to_list(Peerhost0)), case inet:parse_address(binary_to_list(Peerhost0)) of
{peerhost, Peerhost}; {ok, Peerhost} -> {peerhost, Peerhost};
{error, einval} -> {error, "bad peerhost"}
end;
pares_who(#{<<"as">> := As, <<"who">> := Who}) -> pares_who(#{<<"as">> := As, <<"who">> := Who}) ->
{As, Who}. {As, Who}.
parse_time(undefined, Default) ->
Default;
parse_time(Rfc3339, _Default) ->
to_timestamp(Rfc3339).
maybe_format_host({peerhost, Host}) -> maybe_format_host({peerhost, Host}) ->
AddrBinary = list_to_binary(inet:ntoa(Host)), AddrBinary = list_to_binary(inet:ntoa(Host)),
{peerhost, AddrBinary}; {peerhost, AddrBinary};
@ -141,11 +149,6 @@ maybe_format_host({As, Who}) ->
to_rfc3339(Timestamp) -> to_rfc3339(Timestamp) ->
list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])).
to_timestamp(Rfc3339) when is_binary(Rfc3339) ->
to_timestamp(binary_to_list(Rfc3339));
to_timestamp(Rfc3339) ->
calendar:rfc3339_to_system_time(Rfc3339, [{unit, second}]).
-spec(create(emqx_types:banned() | map()) -> -spec(create(emqx_types:banned() | map()) ->
{ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}). {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}).
create(#{who := Who, create(#{who := Who,
@ -168,10 +171,11 @@ create(Banned = #banned{who = Who}) ->
mria:dirty_write(?BANNED_TAB, Banned), mria:dirty_write(?BANNED_TAB, Banned),
{ok, Banned}; {ok, Banned};
[OldBanned = #banned{until = Until}] -> [OldBanned = #banned{until = Until}] ->
case Until < erlang:system_time(second) of %% Don't support shorten or extend the until time by overwrite.
true -> %% We don't support update api yet, user must delete then create new one.
{error, {already_exist, OldBanned}}; case Until > erlang:system_time(second) of
false -> true -> {error, {already_exist, OldBanned}};
false -> %% overwrite expired one is ok.
mria:dirty_write(?BANNED_TAB, Banned), mria:dirty_write(?BANNED_TAB, Banned),
{ok, Banned} {ok, Banned}
end end

View File

@ -204,9 +204,9 @@ publish(Msg) when is_record(Msg, message) ->
_ = emqx_trace:publish(Msg), _ = emqx_trace:publish(Msg),
emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'),
case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of
#message{headers = #{allow_publish := false}} -> #message{headers = #{allow_publish := false}, topic = Topic} ->
?SLOG(debug, #{msg => "message_not_published", ?TRACE("MQTT", "msg_publish_not_allowed", #{message => emqx_message:to_log_map(Msg),
payload => emqx_message:to_log_map(Msg)}), topic => Topic}),
[]; [];
Msg1 = #message{topic = Topic} -> Msg1 = #message{topic = Topic} ->
emqx_persistent_session:persist_message(Msg1), emqx_persistent_session:persist_message(Msg1),
@ -226,7 +226,9 @@ safe_publish(Msg) when is_record(Msg, message) ->
reason => Reason, reason => Reason,
payload => emqx_message:to_log_map(Msg), payload => emqx_message:to_log_map(Msg),
stacktrace => Stk stacktrace => Stk
}), },
#{topic => Msg#message.topic}
),
[] []
end. end.
@ -280,7 +282,7 @@ forward(Node, To, Delivery, async) ->
msg => "async_forward_msg_to_node_failed", msg => "async_forward_msg_to_node_failed",
node => Node, node => Node,
reason => Reason reason => Reason
}), }, #{topic => To}),
{error, badrpc} {error, badrpc}
end; end;
@ -291,7 +293,7 @@ forward(Node, To, Delivery, sync) ->
msg => "sync_forward_msg_to_node_failed", msg => "sync_forward_msg_to_node_failed",
node => Node, node => Node,
reason => Reason reason => Reason
}), }, #{topic => To}),
{error, badrpc}; {error, badrpc};
Result -> Result ->
emqx_metrics:inc('messages.forward'), Result emqx_metrics:inc('messages.forward'), Result

View File

@ -292,7 +292,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
fun check_banned/2 fun check_banned/2
], ConnPkt, Channel#channel{conn_state = connecting}) of ], ConnPkt, Channel#channel{conn_state = connecting}) of
{ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} -> {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
?SLOG(debug, #{msg => "recv_packet", packet => emqx_packet:format(Packet)}), ?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
NChannel1 = NChannel#channel{ NChannel1 = NChannel#channel{
will_msg = emqx_packet:will_msg(NConnPkt), will_msg = emqx_packet:will_msg(NConnPkt),
alias_maximum = init_alias_maximum(NConnPkt, ClientInfo) alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
@ -550,9 +550,8 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
{error, Rc = ?RC_NOT_AUTHORIZED, NChannel} -> {error, Rc = ?RC_NOT_AUTHORIZED, NChannel} ->
?SLOG(warning, #{ ?SLOG(warning, #{
msg => "cannot_publish_to_topic", msg => "cannot_publish_to_topic",
topic => Topic,
reason => emqx_reason_codes:name(Rc) reason => emqx_reason_codes:name(Rc)
}), }, #{topic => Topic}),
case emqx:get_config([authorization, deny_action], ignore) of case emqx:get_config([authorization, deny_action], ignore) of
ignore -> ignore ->
case QoS of case QoS of
@ -568,9 +567,8 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
{error, Rc = ?RC_QUOTA_EXCEEDED, NChannel} -> {error, Rc = ?RC_QUOTA_EXCEEDED, NChannel} ->
?SLOG(warning, #{ ?SLOG(warning, #{
msg => "cannot_publish_to_topic", msg => "cannot_publish_to_topic",
topic => Topic,
reason => emqx_reason_codes:name(Rc) reason => emqx_reason_codes:name(Rc)
}), }, #{topic => Topic}),
case QoS of case QoS of
?QOS_0 -> ?QOS_0 ->
ok = emqx_metrics:inc('packets.publish.dropped'), ok = emqx_metrics:inc('packets.publish.dropped'),
@ -585,7 +583,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
msg => "cannot_publish_to_topic", msg => "cannot_publish_to_topic",
topic => Topic, topic => Topic,
reason => emqx_reason_codes:name(Rc) reason => emqx_reason_codes:name(Rc)
}), }, #{topic => Topic}),
handle_out(disconnect, Rc, NChannel) handle_out(disconnect, Rc, NChannel)
end. end.
@ -635,7 +633,7 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2},
msg => "dropped_qos2_packet", msg => "dropped_qos2_packet",
reason => emqx_reason_codes:name(RC), reason => emqx_reason_codes:name(RC),
packet_id => PacketId packet_id => PacketId
}), }, #{topic => Msg#message.topic}),
ok = emqx_metrics:inc('packets.publish.dropped'), ok = emqx_metrics:inc('packets.publish.dropped'),
handle_out(disconnect, RC, Channel) handle_out(disconnect, RC, Channel)
end. end.
@ -687,7 +685,7 @@ process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Ac
?SLOG(warning, #{ ?SLOG(warning, #{
msg => "cannot_subscribe_topic_filter", msg => "cannot_subscribe_topic_filter",
reason => emqx_reason_codes:name(ReasonCode) reason => emqx_reason_codes:name(ReasonCode)
}), }, #{topic => TopicFilter}),
process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc]) process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc])
end. end.
@ -703,7 +701,7 @@ do_subscribe(TopicFilter, SubOpts = #{qos := QoS}, Channel =
?SLOG(warning, #{ ?SLOG(warning, #{
msg => "cannot_subscribe_topic_filter", msg => "cannot_subscribe_topic_filter",
reason => emqx_reason_codes:text(RC) reason => emqx_reason_codes:text(RC)
}), }, #{topic => NTopicFilter}),
{RC, Channel} {RC, Channel}
end. end.

View File

@ -375,7 +375,7 @@ discard_session(ClientId) when is_binary(ClientId) ->
-spec kick_or_kill(kick | discard, module(), pid()) -> ok. -spec kick_or_kill(kick | discard, module(), pid()) -> ok.
kick_or_kill(Action, ConnMod, Pid) -> kick_or_kill(Action, ConnMod, Pid) ->
try try
%% this is essentailly a gen_server:call implemented in emqx_connection %% this is essentially a gen_server:call implemented in emqx_connection
%% and emqx_ws_connection. %% and emqx_ws_connection.
%% the handle_call is implemented in emqx_channel %% the handle_call is implemented in emqx_channel
ok = apply(ConnMod, call, [Pid, Action, ?T_KICK]) ok = apply(ConnMod, call, [Pid, Action, ?T_KICK])
@ -390,19 +390,12 @@ kick_or_kill(Action, ConnMod, Pid) ->
ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}); ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action});
_ : {timeout, {gen_server, call, _}} -> _ : {timeout, {gen_server, call, _}} ->
?tp(warning, "session_kick_timeout", ?tp(warning, "session_kick_timeout",
#{pid => Pid, #{pid => Pid, action => Action, stale_channel => stale_channel_info(Pid)}),
action => Action,
stale_channel => stale_channel_info(Pid)
}),
ok = force_kill(Pid); ok = force_kill(Pid);
_ : Error : St -> _ : Error : St ->
?tp(error, "session_kick_exception", ?tp(error, "session_kick_exception",
#{pid => Pid, #{pid => Pid, action => Action, reason => Error, stacktrace => St,
action => Action, stale_channel => stale_channel_info(Pid)}),
reason => Error,
stacktrace => St,
stale_channel => stale_channel_info(Pid)
}),
ok = force_kill(Pid) ok = force_kill(Pid)
end. end.
@ -448,20 +441,22 @@ kick_session(Action, ClientId, ChanPid) ->
, action => Action , action => Action
, error => Error , error => Error
, reason => Reason , reason => Reason
}) },
#{clientid => ClientId})
end. end.
kick_session(ClientId) -> kick_session(ClientId) ->
case lookup_channels(ClientId) of case lookup_channels(ClientId) of
[] -> [] ->
?SLOG(warning, #{msg => "kicked_an_unknown_session", ?SLOG(warning, #{msg => "kicked_an_unknown_session"},
clientid => ClientId}), #{clientid => ClientId}),
ok; ok;
ChanPids -> ChanPids ->
case length(ChanPids) > 1 of case length(ChanPids) > 1 of
true -> true ->
?SLOG(warning, #{msg => "more_than_one_channel_found", ?SLOG(warning, #{msg => "more_than_one_channel_found",
chan_pids => ChanPids}); chan_pids => ChanPids},
#{clientid => ClientId});
false -> ok false -> ok
end, end,
lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids) lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids)
@ -478,12 +473,12 @@ with_channel(ClientId, Fun) ->
Pids -> Fun(lists:last(Pids)) Pids -> Fun(lists:last(Pids))
end. end.
%% @doc Get all registed channel pids. Debugg/test interface %% @doc Get all registered channel pids. Debug/test interface
all_channels() -> all_channels() ->
Pat = [{{'_', '$1'}, [], ['$1']}], Pat = [{{'_', '$1'}, [], ['$1']}],
ets:select(?CHAN_TAB, Pat). ets:select(?CHAN_TAB, Pat).
%% @doc Get all registed clientIDs. Debugg/test interface %% @doc Get all registered clientIDs. Debug/test interface
all_client_ids() -> all_client_ids() ->
Pat = [{{'$1', '_'}, [], ['$1']}], Pat = [{{'$1', '_'}, [], ['$1']}],
ets:select(?CHAN_TAB, Pat). ets:select(?CHAN_TAB, Pat).
@ -511,7 +506,7 @@ lookup_channels(local, ClientId) ->
rpc_call(Node, Fun, Args, Timeout) -> rpc_call(Node, Fun, Args, Timeout) ->
case rpc:call(Node, ?MODULE, Fun, Args, 2 * Timeout) of case rpc:call(Node, ?MODULE, Fun, Args, 2 * Timeout) of
{badrpc, Reason} -> {badrpc, Reason} ->
%% since eqmx app 4.3.10, the 'kick' and 'discard' calls hanndler %% since emqx app 4.3.10, the 'kick' and 'discard' calls handler
%% should catch all exceptions and always return 'ok'. %% should catch all exceptions and always return 'ok'.
%% This leaves 'badrpc' only possible when there is problem %% This leaves 'badrpc' only possible when there is problem
%% calling the remote node. %% calling the remote node.

View File

@ -262,8 +262,9 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) ->
{ok, RawRichConf} -> {ok, RawRichConf} ->
init_load(SchemaMod, RawRichConf); init_load(SchemaMod, RawRichConf);
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => failed_to_load_hocon_conf, ?SLOG(error, #{msg => "failed_to_load_hocon_conf",
reason => Reason, reason => Reason,
pwd => file:get_cwd(),
include_dirs => IncDir include_dirs => IncDir
}), }),
error(failed_to_load_hocon_conf) error(failed_to_load_hocon_conf)
@ -396,7 +397,7 @@ save_to_override_conf(RawConf, Opts) ->
case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
ok -> ok; ok -> ok;
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => failed_to_write_override_file, ?SLOG(error, #{msg => "failed_to_write_override_file",
filename => FileName, filename => FileName,
reason => Reason}), reason => Reason}),
{error, Reason} {error, Reason}

View File

@ -449,14 +449,12 @@ handle_msg({'$gen_cast', Req}, State) ->
{ok, NewState}; {ok, NewState};
handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl ->
?SLOG(debug, #{msg => "RECV_data", data => Data, transport => Inet}),
Oct = iolist_size(Data), Oct = iolist_size(Data),
inc_counter(incoming_bytes, Oct), inc_counter(incoming_bytes, Oct),
ok = emqx_metrics:inc('bytes.received', Oct), ok = emqx_metrics:inc('bytes.received', Oct),
when_bytes_in(Oct, Data, State); when_bytes_in(Oct, Data, State);
handle_msg({quic, Data, _Sock, _, _, _}, State) -> handle_msg({quic, Data, _Sock, _, _, _}, State) ->
?SLOG(debug, #{msg => "RECV_data", data => Data, transport => quic}),
Oct = iolist_size(Data), Oct = iolist_size(Data),
inc_counter(incoming_bytes, Oct), inc_counter(incoming_bytes, Oct),
ok = emqx_metrics:inc('bytes.received', Oct), ok = emqx_metrics:inc('bytes.received', Oct),
@ -528,7 +526,7 @@ handle_msg({connack, ConnAck}, State) ->
handle_outgoing(ConnAck, State); handle_outgoing(ConnAck, State);
handle_msg({close, Reason}, State) -> handle_msg({close, Reason}, State) ->
?SLOG(debug, #{msg => "force_socket_close", reason => Reason}), ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}),
handle_info({sock_closed, Reason}, close_socket(State)); handle_info({sock_closed, Reason}, close_socket(State));
handle_msg({event, connected}, State = #state{channel = Channel}) -> handle_msg({event, connected}, State = #state{channel = Channel}) ->
@ -566,7 +564,8 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport,
Channel1 = emqx_channel:set_conn_state(disconnected, Channel), Channel1 = emqx_channel:set_conn_state(disconnected, Channel),
emqx_congestion:cancel_alarms(Socket, Transport, Channel1), emqx_congestion:cancel_alarms(Socket, Transport, Channel1),
emqx_channel:terminate(Reason, Channel1), emqx_channel:terminate(Reason, Channel1),
close_socket_ok(State) close_socket_ok(State),
?TRACE("SOCKET", "tcp_socket_terminated", #{reason => Reason})
catch catch
E : C : S -> E : C : S ->
?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S}) ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S})
@ -716,7 +715,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) -> handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) ->
ok = inc_incoming_stats(Packet), ok = inc_incoming_stats(Packet),
?SLOG(debug, #{msg => "RECV_packet", packet => emqx_packet:format(Packet)}), ?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
with_channel(handle_in, [Packet], State); with_channel(handle_in, [Packet], State);
handle_incoming(FrameError, State) -> handle_incoming(FrameError, State) ->
@ -755,15 +754,13 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
<<>> -> ?SLOG(warning, #{ <<>> -> ?SLOG(warning, #{
msg => "packet_is_discarded", msg => "packet_is_discarded",
reason => "frame_is_too_large", reason => "frame_is_too_large",
packet => emqx_packet:format(Packet) packet => emqx_packet:format(Packet, hidden)
}), }),
ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped.too_large'),
ok = emqx_metrics:inc('delivery.dropped'), ok = emqx_metrics:inc('delivery.dropped'),
<<>>; <<>>;
Data -> ?SLOG(debug, #{ Data ->
msg => "SEND_packet", ?TRACE("MQTT", "mqtt_packet_sent", #{packet => Packet}),
packet => emqx_packet:format(Packet)
}),
ok = inc_outgoing_stats(Packet), ok = inc_outgoing_stats(Packet),
Data Data
catch catch
@ -875,7 +872,7 @@ check_limiter(Needs,
{ok, Limiter2} -> {ok, Limiter2} ->
WhenOk(Data, Msgs, State#state{limiter = Limiter2}); WhenOk(Data, Msgs, State#state{limiter = Limiter2});
{pause, Time, Limiter2} -> {pause, Time, Limiter2} ->
?SLOG(warning, #{msg => "pause time dueto rate limit", ?SLOG(warning, #{msg => "pause_time_dueto_rate_limit",
needs => Needs, needs => Needs,
time_in_ms => Time}), time_in_ms => Time}),
@ -915,7 +912,7 @@ retry_limiter(#state{limiter = Limiter} = State) ->
, limiter_timer = undefined , limiter_timer = undefined
}); });
{pause, Time, Limiter2} -> {pause, Time, Limiter2} ->
?SLOG(warning, #{msg => "pause time dueto rate limit", ?SLOG(warning, #{msg => "pause_time_dueto_rate_limit",
types => Types, types => Types,
time_in_ms => Time}), time_in_ms => Time}),

View File

@ -118,11 +118,10 @@ handle_cast({detected, #flapping{clientid = ClientId,
true -> %% Flapping happened:( true -> %% Flapping happened:(
?SLOG(warning, #{ ?SLOG(warning, #{
msg => "flapping_detected", msg => "flapping_detected",
client_id => ClientId,
peer_host => fmt_host(PeerHost), peer_host => fmt_host(PeerHost),
detect_cnt => DetectCnt, detect_cnt => DetectCnt,
wind_time_in_ms => WindTime wind_time_in_ms => WindTime
}), }, #{clientid => ClientId}),
Now = erlang:system_time(second), Now = erlang:system_time(second),
Banned = #banned{who = {clientid, ClientId}, Banned = #banned{who = {clientid, ClientId},
by = <<"flapping detector">>, by = <<"flapping detector">>,
@ -134,11 +133,10 @@ handle_cast({detected, #flapping{clientid = ClientId,
false -> false ->
?SLOG(warning, #{ ?SLOG(warning, #{
msg => "client_disconnected", msg => "client_disconnected",
client_id => ClientId,
peer_host => fmt_host(PeerHost), peer_host => fmt_host(PeerHost),
detect_cnt => DetectCnt, detect_cnt => DetectCnt,
interval => Interval interval => Interval
}) }, #{clientid => ClientId})
end, end,
{noreply, State}; {noreply, State};

View File

@ -197,15 +197,7 @@ critical(Metadata, Format, Args) when is_map(Metadata) ->
set_metadata_clientid(<<>>) -> set_metadata_clientid(<<>>) ->
ok; ok;
set_metadata_clientid(ClientId) -> set_metadata_clientid(ClientId) ->
try set_proc_metadata(#{clientid => ClientId}).
%% try put string format client-id metadata so
%% so the log is not like <<"...">>
Id = unicode:characters_to_list(ClientId, utf8),
set_proc_metadata(#{clientid => Id})
catch
_: _->
ok
end.
-spec(set_metadata_peername(peername_str()) -> ok). -spec(set_metadata_peername(peername_str()) -> ok).
set_metadata_peername(Peername) -> set_metadata_peername(Peername) ->

View File

@ -18,22 +18,77 @@
-export([format/2]). -export([format/2]).
-export([check_config/1]). -export([check_config/1]).
-export([try_format_unicode/1]).
check_config(X) -> logger_formatter:check_config(X). check_config(X) -> logger_formatter:check_config(X).
format(#{msg := {report, Report}, meta := Meta} = Event, Config) when is_map(Report) -> format(#{msg := {report, Report0}, meta := Meta} = Event, Config) when is_map(Report0) ->
logger_formatter:format(Event#{msg := {report, enrich(Report, Meta)}}, Config); Report1 = enrich_report_mfa(Report0, Meta),
format(#{msg := Msg, meta := Meta} = Event, Config) -> Report2 = enrich_report_clientid(Report1, Meta),
NewMsg = enrich_fmt(Msg, Meta), Report3 = enrich_report_peername(Report2, Meta),
logger_formatter:format(Event#{msg := NewMsg}, Config). Report4 = enrich_report_topic(Report3, Meta),
logger_formatter:format(Event#{msg := {report, Report4}}, Config);
format(#{msg := {string, String}} = Event, Config) ->
format(Event#{msg => {"~ts ", String}}, Config);
format(#{msg := Msg0, meta := Meta} = Event, Config) ->
Msg1 = enrich_client_info(Msg0, Meta),
Msg2 = enrich_mfa(Msg1, Meta),
Msg3 = enrich_topic(Msg2, Meta),
logger_formatter:format(Event#{msg := Msg3}, Config).
enrich(Report, #{mfa := Mfa, line := Line}) -> try_format_unicode(Char) ->
List =
try
case unicode:characters_to_list(Char) of
{error, _, _} -> error;
{incomplete, _, _} -> error;
Binary -> Binary
end
catch _:_ ->
error
end,
case List of
error -> io_lib:format("~0p", [Char]);
_ -> List
end.
enrich_report_mfa(Report, #{mfa := Mfa, line := Line}) ->
Report#{mfa => mfa(Mfa), line => Line}; Report#{mfa => mfa(Mfa), line => Line};
enrich(Report, _) -> Report. enrich_report_mfa(Report, _) -> Report.
enrich_fmt({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) -> enrich_report_clientid(Report, #{clientid := ClientId}) ->
Report#{clientid => try_format_unicode(ClientId)};
enrich_report_clientid(Report, _) -> Report.
enrich_report_peername(Report, #{peername := Peername}) ->
Report#{peername => Peername};
enrich_report_peername(Report, _) -> Report.
%% clientid and peername always in emqx_conn's process metadata.
%% topic can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2
enrich_report_topic(Report, #{topic := Topic}) ->
Report#{topic => try_format_unicode(Topic)};
enrich_report_topic(Report = #{topic := Topic}, _) ->
Report#{topic => try_format_unicode(Topic)};
enrich_report_topic(Report, _) -> Report.
enrich_mfa({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) ->
{Fmt ++ " mfa: ~ts line: ~w", Args ++ [mfa(Mfa), Line]}; {Fmt ++ " mfa: ~ts line: ~w", Args ++ [mfa(Mfa), Line]};
enrich_fmt(Msg, _) -> enrich_mfa(Msg, _) ->
Msg.
enrich_client_info({Fmt, Args}, #{clientid := ClientId, peername := Peer}) when is_list(Fmt) ->
{" ~ts@~ts " ++ Fmt, [ClientId, Peer | Args] };
enrich_client_info({Fmt, Args}, #{clientid := ClientId}) when is_list(Fmt) ->
{" ~ts " ++ Fmt, [ClientId | Args]};
enrich_client_info({Fmt, Args}, #{peername := Peer}) when is_list(Fmt) ->
{" ~ts " ++ Fmt, [Peer | Args]};
enrich_client_info(Msg, _) ->
Msg.
enrich_topic({Fmt, Args}, #{topic := Topic}) when is_list(Fmt) ->
{" topic: ~ts" ++ Fmt, [Topic | Args]};
enrich_topic(Msg, _) ->
Msg. Msg.
mfa({M, F, A}) -> atom_to_list(M) ++ ":" ++ atom_to_list(F) ++ "/" ++ integer_to_list(A). mfa({M, F, A}) -> atom_to_list(M) ++ ":" ++ atom_to_list(F) ++ "/" ++ integer_to_list(A).

View File

@ -44,7 +44,11 @@
, will_msg/1 , will_msg/1
]). ]).
-export([format/1]). -export([ format/1
, format/2
]).
-export([encode_hex/1]).
-define(TYPE_NAMES, -define(TYPE_NAMES,
{ 'CONNECT' { 'CONNECT'
@ -435,25 +439,28 @@ will_msg(#mqtt_packet_connect{clientid = ClientId,
%% @doc Format packet %% @doc Format packet
-spec(format(emqx_types:packet()) -> iolist()). -spec(format(emqx_types:packet()) -> iolist()).
format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}) -> format(Packet) -> format(Packet, emqx_trace_handler:payload_encode()).
format_header(Header, format_variable(Variable, Payload)).
%% @doc Format packet
-spec(format(emqx_types:packet(), hex | text | hidden) -> iolist()).
format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, PayloadEncode) ->
HeaderIO = format_header(Header),
case format_variable(Variable, Payload, PayloadEncode) of
"" -> HeaderIO;
VarIO -> [HeaderIO,",", VarIO]
end.
format_header(#mqtt_packet_header{type = Type, format_header(#mqtt_packet_header{type = Type,
dup = Dup, dup = Dup,
qos = QoS, qos = QoS,
retain = Retain}, S) -> retain = Retain}) ->
S1 = case S == undefined of io_lib:format("~ts(Q~p, R~p, D~p)", [type_name(Type), QoS, i(Retain), i(Dup)]).
true -> <<>>;
false -> [", ", S]
end,
io_lib:format("~ts(Q~p, R~p, D~p~ts)", [type_name(Type), QoS, i(Retain), i(Dup), S1]).
format_variable(undefined, _) -> format_variable(undefined, _, _) -> "";
undefined; format_variable(Variable, undefined, PayloadEncode) ->
format_variable(Variable, undefined) -> format_variable(Variable, PayloadEncode);
format_variable(Variable); format_variable(Variable, Payload, PayloadEncode) ->
format_variable(Variable, Payload) -> [format_variable(Variable, PayloadEncode), format_payload(Payload, PayloadEncode)].
io_lib:format("~ts, Payload=~0p", [format_variable(Variable), Payload]).
format_variable(#mqtt_packet_connect{ format_variable(#mqtt_packet_connect{
proto_ver = ProtoVer, proto_ver = ProtoVer,
@ -467,57 +474,140 @@ format_variable(#mqtt_packet_connect{
will_topic = WillTopic, will_topic = WillTopic,
will_payload = WillPayload, will_payload = WillPayload,
username = Username, username = Username,
password = Password}) -> password = Password},
Format = "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts", PayloadEncode) ->
Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)], Base = io_lib:format(
{Format1, Args1} = if "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts",
WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~ts, Payload=~0p)", [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)]),
Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; case WillFlag of
true -> {Format, Args} true ->
end, [Base, io_lib:format(", Will(Q~p, R~p, Topic=~ts ",
io_lib:format(Format1, Args1); [WillQoS, i(WillRetain), WillTopic]),
format_payload(WillPayload, PayloadEncode), ")"];
false ->
Base
end;
format_variable(#mqtt_packet_disconnect format_variable(#mqtt_packet_disconnect
{reason_code = ReasonCode}) -> {reason_code = ReasonCode}, _) ->
io_lib:format("ReasonCode=~p", [ReasonCode]); io_lib:format("ReasonCode=~p", [ReasonCode]);
format_variable(#mqtt_packet_connack{ack_flags = AckFlags, format_variable(#mqtt_packet_connack{ack_flags = AckFlags,
reason_code = ReasonCode}) -> reason_code = ReasonCode}, _) ->
io_lib:format("AckFlags=~p, ReasonCode=~p", [AckFlags, ReasonCode]); io_lib:format("AckFlags=~p, ReasonCode=~p", [AckFlags, ReasonCode]);
format_variable(#mqtt_packet_publish{topic_name = TopicName, format_variable(#mqtt_packet_publish{topic_name = TopicName,
packet_id = PacketId}) -> packet_id = PacketId}, _) ->
io_lib:format("Topic=~ts, PacketId=~p", [TopicName, PacketId]); io_lib:format("Topic=~ts, PacketId=~p", [TopicName, PacketId]);
format_variable(#mqtt_packet_puback{packet_id = PacketId, format_variable(#mqtt_packet_puback{packet_id = PacketId,
reason_code = ReasonCode}) -> reason_code = ReasonCode}, _) ->
io_lib:format("PacketId=~p, ReasonCode=~p", [PacketId, ReasonCode]); io_lib:format("PacketId=~p, ReasonCode=~p", [PacketId, ReasonCode]);
format_variable(#mqtt_packet_subscribe{packet_id = PacketId, format_variable(#mqtt_packet_subscribe{packet_id = PacketId,
topic_filters = TopicFilters}) -> topic_filters = TopicFilters}, _) ->
io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, TopicFilters]); [io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=",
format_topic_filters(TopicFilters)];
format_variable(#mqtt_packet_unsubscribe{packet_id = PacketId, format_variable(#mqtt_packet_unsubscribe{packet_id = PacketId,
topic_filters = Topics}) -> topic_filters = Topics}, _) ->
io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, Topics]); [io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=",
format_topic_filters(Topics)];
format_variable(#mqtt_packet_suback{packet_id = PacketId, format_variable(#mqtt_packet_suback{packet_id = PacketId,
reason_codes = ReasonCodes}) -> reason_codes = ReasonCodes}, _) ->
io_lib:format("PacketId=~p, ReasonCodes=~p", [PacketId, ReasonCodes]); io_lib:format("PacketId=~p, ReasonCodes=~p", [PacketId, ReasonCodes]);
format_variable(#mqtt_packet_unsuback{packet_id = PacketId}) -> format_variable(#mqtt_packet_unsuback{packet_id = PacketId}, _) ->
io_lib:format("PacketId=~p", [PacketId]); io_lib:format("PacketId=~p", [PacketId]);
format_variable(#mqtt_packet_auth{reason_code = ReasonCode}) -> format_variable(#mqtt_packet_auth{reason_code = ReasonCode}, _) ->
io_lib:format("ReasonCode=~p", [ReasonCode]); io_lib:format("ReasonCode=~p", [ReasonCode]);
format_variable(PacketId) when is_integer(PacketId) -> format_variable(PacketId, _) when is_integer(PacketId) ->
io_lib:format("PacketId=~p", [PacketId]). io_lib:format("PacketId=~p", [PacketId]).
format_password(undefined) -> undefined; format_password(undefined) -> "undefined";
format_password(_Password) -> '******'. format_password(_Password) -> "******".
format_payload(Payload, text) -> ["Payload=", io_lib:format("~ts", [Payload])];
format_payload(Payload, hex) -> ["Payload(hex)=", encode_hex(Payload)];
format_payload(_, hidden) -> "Payload=******".
i(true) -> 1; i(true) -> 1;
i(false) -> 0; i(false) -> 0;
i(I) when is_integer(I) -> I. i(I) when is_integer(I) -> I.
format_topic_filters(Filters) ->
["[",
lists:join(",",
lists:map(
fun({TopicFilter, SubOpts}) ->
io_lib:format("~ts(~p)", [TopicFilter, SubOpts]);
(TopicFilter) ->
io_lib:format("~ts", [TopicFilter])
end, Filters)),
"]"].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Hex encoding functions
%% Copy from binary:encode_hex/1 (was only introduced in OTP24).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-define(HEX(X), (hex(X)):16).
-compile({inline,[hex/1]}).
-spec encode_hex(Bin) -> Bin2 when
Bin :: binary(),
Bin2 :: <<_:_*16>>.
encode_hex(Data) when byte_size(Data) rem 8 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G),?HEX(H)>> || <<A,B,C,D,E,F,G,H>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 7 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G)>> || <<A,B,C,D,E,F,G>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 6 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F)>> || <<A,B,C,D,E,F>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 5 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E)>> || <<A,B,C,D,E>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 4 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D)>> || <<A,B,C,D>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 3 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C)>> || <<A,B,C>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 2 =:= 0 ->
<< <<?HEX(A),?HEX(B)>> || <<A,B>> <= Data >>;
encode_hex(Data) when is_binary(Data) ->
<< <<?HEX(N)>> || <<N>> <= Data >>;
encode_hex(Bin) ->
erlang:error(badarg, [Bin]).
hex(X) ->
element(
X+1, {16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, 16#3041,
16#3042, 16#3043, 16#3044, 16#3045, 16#3046,
16#3130, 16#3131, 16#3132, 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3141,
16#3142, 16#3143, 16#3144, 16#3145, 16#3146,
16#3230, 16#3231, 16#3232, 16#3233, 16#3234, 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3241,
16#3242, 16#3243, 16#3244, 16#3245, 16#3246,
16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, 16#3337, 16#3338, 16#3339, 16#3341,
16#3342, 16#3343, 16#3344, 16#3345, 16#3346,
16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, 16#3439, 16#3441,
16#3442, 16#3443, 16#3444, 16#3445, 16#3446,
16#3530, 16#3531, 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3541,
16#3542, 16#3543, 16#3544, 16#3545, 16#3546,
16#3630, 16#3631, 16#3632, 16#3633, 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3641,
16#3642, 16#3643, 16#3644, 16#3645, 16#3646,
16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, 16#3736, 16#3737, 16#3738, 16#3739, 16#3741,
16#3742, 16#3743, 16#3744, 16#3745, 16#3746,
16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, 16#3838, 16#3839, 16#3841,
16#3842, 16#3843, 16#3844, 16#3845, 16#3846,
16#3930, 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, 16#3941,
16#3942, 16#3943, 16#3944, 16#3945, 16#3946,
16#4130, 16#4131, 16#4132, 16#4133, 16#4134, 16#4135, 16#4136, 16#4137, 16#4138, 16#4139, 16#4141,
16#4142, 16#4143, 16#4144, 16#4145, 16#4146,
16#4230, 16#4231, 16#4232, 16#4233, 16#4234, 16#4235, 16#4236, 16#4237, 16#4238, 16#4239, 16#4241,
16#4242, 16#4243, 16#4244, 16#4245, 16#4246,
16#4330, 16#4331, 16#4332, 16#4333, 16#4334, 16#4335, 16#4336, 16#4337, 16#4338, 16#4339, 16#4341,
16#4342, 16#4343, 16#4344, 16#4345, 16#4346,
16#4430, 16#4431, 16#4432, 16#4433, 16#4434, 16#4435, 16#4436, 16#4437, 16#4438, 16#4439, 16#4441,
16#4442, 16#4443, 16#4444, 16#4445, 16#4446,
16#4530, 16#4531, 16#4532, 16#4533, 16#4534, 16#4535, 16#4536, 16#4537, 16#4538, 16#4539, 16#4541,
16#4542, 16#4543, 16#4544, 16#4545, 16#4546,
16#4630, 16#4631, 16#4632, 16#4633, 16#4634, 16#4635, 16#4636, 16#4637, 16#4638, 16#4639, 16#4641,
16#4642, 16#4643, 16#4644, 16#4645, 16#4646}).

View File

@ -38,7 +38,6 @@
-type ip_port() :: tuple(). -type ip_port() :: tuple().
-type cipher() :: map(). -type cipher() :: map().
-type rfc3339_system_time() :: integer(). -type rfc3339_system_time() :: integer().
-type unicode_binary() :: binary().
-typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration/0, emqx_schema, to_duration}).
-typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}).
@ -52,7 +51,6 @@
-typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}).
-typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}).
-typerefl_from_string({rfc3339_system_time/0, emqx_schema, rfc3339_to_system_time}). -typerefl_from_string({rfc3339_system_time/0, emqx_schema, rfc3339_to_system_time}).
-typerefl_from_string({unicode_binary/0, emqx_schema, to_unicode_binary}).
-export([ validate_heap_size/1 -export([ validate_heap_size/1
, parse_user_lookup_fun/1 , parse_user_lookup_fun/1
@ -66,8 +64,7 @@
to_bar_separated_list/1, to_ip_port/1, to_bar_separated_list/1, to_ip_port/1,
to_erl_cipher_suite/1, to_erl_cipher_suite/1,
to_comma_separated_atoms/1, to_comma_separated_atoms/1,
rfc3339_to_system_time/1, rfc3339_to_system_time/1]).
to_unicode_binary/1]).
-behaviour(hocon_schema). -behaviour(hocon_schema).
@ -76,8 +73,7 @@
comma_separated_list/0, bar_separated_list/0, ip_port/0, comma_separated_list/0, bar_separated_list/0, ip_port/0,
cipher/0, cipher/0,
comma_separated_atoms/0, comma_separated_atoms/0,
rfc3339_system_time/0, rfc3339_system_time/0]).
unicode_binary/0]).
-export([namespace/0, roots/0, roots/1, fields/1]). -export([namespace/0, roots/0, roots/1, fields/1]).
-export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]).
@ -184,6 +180,12 @@ roots(low) ->
, {"latency_stats", , {"latency_stats",
sc(ref("latency_stats"), sc(ref("latency_stats"),
#{})} #{})}
, {"trace",
sc(ref("trace"),
#{desc => """
Real-time filtering logs for the ClientID or Topic or IP for debugging.
"""
})}
]. ].
fields("persistent_session_store") -> fields("persistent_session_store") ->
@ -1044,6 +1046,17 @@ when deactivated, but after the retention time.
fields("latency_stats") -> fields("latency_stats") ->
[ {"samples", sc(integer(), #{default => 10, [ {"samples", sc(integer(), #{default => 10,
desc => "the number of smaples for calculate the average latency of delivery"})} desc => "the number of smaples for calculate the average latency of delivery"})}
];
fields("trace") ->
[ {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{
default => text,
desc => """
Determine the format of the payload format in the trace file.<br>
`text`: Text-based protocol or plain text protocol. It is recommended when payload is json encode.<br>
`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.<br>
`hidden`: payload is obfuscated as `******`
"""
})}
]. ].
mqtt_listener() -> mqtt_listener() ->
@ -1453,9 +1466,6 @@ rfc3339_to_system_time(DateTime) ->
{error, bad_rfc3339_timestamp} {error, bad_rfc3339_timestamp}
end. end.
to_unicode_binary(Str) ->
{ok, unicode:characters_to_binary(Str)}.
to_bar_separated_list(Str) -> to_bar_separated_list(Str) ->
{ok, string:tokens(Str, "| ")}. {ok, string:tokens(Str, "| ")}.

View File

@ -535,16 +535,20 @@ enqueue(Msg, Session = #session{mqueue = Q}) when is_record(Msg, message) ->
(Dropped =/= undefined) andalso log_dropped(Dropped, Session), (Dropped =/= undefined) andalso log_dropped(Dropped, Session),
Session#session{mqueue = NewQ}. Session#session{mqueue = NewQ}.
log_dropped(Msg = #message{qos = QoS}, #session{mqueue = Q}) -> log_dropped(Msg = #message{qos = QoS, topic = Topic}, #session{mqueue = Q}) ->
case (QoS == ?QOS_0) andalso (not emqx_mqueue:info(store_qos0, Q)) of Payload = emqx_message:to_log_map(Msg),
#{store_qos0 := StoreQos0} = QueueInfo = emqx_mqueue:info(Q),
case (QoS == ?QOS_0) andalso (not StoreQos0) of
true -> true ->
ok = emqx_metrics:inc('delivery.dropped.qos0_msg'), ok = emqx_metrics:inc('delivery.dropped.qos0_msg'),
?SLOG(warning, #{msg => "dropped_qos0_msg", ?SLOG(warning, #{msg => "dropped_qos0_msg",
payload => emqx_message:to_log_map(Msg)}); queue => QueueInfo,
payload => Payload}, #{topic => Topic});
false -> false ->
ok = emqx_metrics:inc('delivery.dropped.queue_full'), ok = emqx_metrics:inc('delivery.dropped.queue_full'),
?SLOG(warning, #{msg => "dropped_msg_due_to_mqueue_is_full", ?SLOG(warning, #{msg => "dropped_msg_due_to_mqueue_is_full",
payload => emqx_message:to_log_map(Msg)}) queue => QueueInfo,
payload => Payload}, #{topic => Topic})
end. end.
enrich_fun(Session = #session{subscriptions = Subs}) -> enrich_fun(Session = #session{subscriptions = Subs}) ->

View File

@ -260,7 +260,7 @@ code_change(_OldVsn, State, _Extra) ->
init_resume_worker(RemotePid, SessionID, #{ pmon := Pmon } = State) -> init_resume_worker(RemotePid, SessionID, #{ pmon := Pmon } = State) ->
case emqx_session_router_worker_sup:start_worker(SessionID, RemotePid) of case emqx_session_router_worker_sup:start_worker(SessionID, RemotePid) of
{error, What} -> {error, What} ->
?SLOG(error, #{msg => "Could not start resume worker", reason => What}), ?SLOG(error, #{msg => "failed_to_start_resume_worker", reason => What}),
error; error;
{ok, Pid} -> {ok, Pid} ->
Pmon1 = emqx_pmon:monitor(Pid, Pmon), Pmon1 = emqx_pmon:monitor(Pid, Pmon),

View File

@ -26,6 +26,7 @@
-export([ publish/1 -export([ publish/1
, subscribe/3 , subscribe/3
, unsubscribe/2 , unsubscribe/2
, log/4
]). ]).
-export([ start_link/0 -export([ start_link/0
@ -36,6 +37,7 @@
, delete/1 , delete/1
, clear/0 , clear/0
, update/2 , update/2
, check/0
]). ]).
-export([ format/1 -export([ format/1
@ -50,6 +52,7 @@
-define(TRACE, ?MODULE). -define(TRACE, ?MODULE).
-define(MAX_SIZE, 30). -define(MAX_SIZE, 30).
-define(OWN_KEYS, [level, filters, filter_default, handlers]).
-ifdef(TEST). -ifdef(TEST).
-export([ log_file/2 -export([ log_file/2
@ -80,27 +83,53 @@ mnesia(boot) ->
publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore;
publish(#message{from = From, topic = Topic, payload = Payload}) when publish(#message{from = From, topic = Topic, payload = Payload}) when
is_binary(From); is_atom(From) -> is_binary(From); is_atom(From) ->
emqx_logger:info( ?TRACE("PUBLISH", "publish_to", #{topic => Topic, payload => Payload}).
#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
"PUBLISH to ~s: ~0p",
[Topic, Payload]
).
subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore; subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore;
subscribe(Topic, SubId, SubOpts) -> subscribe(Topic, SubId, SubOpts) ->
emqx_logger:info( ?TRACE("SUBSCRIBE", "subscribe", #{topic => Topic, sub_opts => SubOpts, sub_id => SubId}).
#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
"~ts SUBSCRIBE ~ts: Options: ~0p",
[SubId, Topic, SubOpts]
).
unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore; unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore;
unsubscribe(Topic, SubOpts) -> unsubscribe(Topic, SubOpts) ->
emqx_logger:info( ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}).
#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
"~ts UNSUBSCRIBE ~ts: Options: ~0p", log(List, Event, Msg, Meta0) ->
[maps:get(subid, SubOpts, ""), Topic, SubOpts] Meta =
). case logger:get_process_metadata() of
undefined -> Meta0;
ProcMeta -> maps:merge(ProcMeta, Meta0)
end,
Log = #{level => trace, event => Event, meta => Meta, msg => Msg},
log_filter(List, Log).
log_filter([], _Log) -> ok;
log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
case FilterFun(Log0, {Filter, Name}) of
stop -> stop;
ignore -> ignore;
Log ->
case logger_config:get(ets:whereis(logger), Id) of
{ok, #{module := Module} = HandlerConfig0} ->
HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
try Module:log(Log, HandlerConfig)
catch C:R:S ->
case logger:remove_handler(Id) of
ok ->
logger:internal_log(error, {removed_failing_handler, Id, C, R, S});
{error,{not_found,_}} ->
%% Probably already removed by other client
%% Don't report again
ok;
{error,Reason} ->
logger:internal_log(error,
{removed_handler_failed, Id, Reason, C, R, S})
end
end;
{error, {not_found, Id}} -> ok;
{error, Reason} -> logger:internal_log(error, {find_handle_id_failed, Id, Reason})
end
end,
log_filter(Rest, Log0).
-spec(start_link() -> emqx_types:startlink_ret()). -spec(start_link() -> emqx_types:startlink_ret()).
start_link() -> start_link() ->
@ -161,6 +190,9 @@ update(Name, Enable) ->
end, end,
transaction(Tran). transaction(Tran).
check() ->
gen_server:call(?MODULE, check).
-spec get_trace_filename(Name :: binary()) -> -spec get_trace_filename(Name :: binary()) ->
{ok, FileName :: string()} | {error, not_found}. {ok, FileName :: string()} | {error, not_found}.
get_trace_filename(Name) -> get_trace_filename(Name) ->
@ -196,15 +228,17 @@ format(Traces) ->
init([]) -> init([]) ->
ok = mria:wait_for_tables([?TRACE]), ok = mria:wait_for_tables([?TRACE]),
erlang:process_flag(trap_exit, true), erlang:process_flag(trap_exit, true),
OriginLogLevel = emqx_logger:get_primary_log_level(),
ok = filelib:ensure_dir(trace_dir()), ok = filelib:ensure_dir(trace_dir()),
ok = filelib:ensure_dir(zip_dir()), ok = filelib:ensure_dir(zip_dir()),
{ok, _} = mnesia:subscribe({table, ?TRACE, simple}), {ok, _} = mnesia:subscribe({table, ?TRACE, simple}),
Traces = get_enable_trace(), Traces = get_enable_trace(),
ok = update_log_primary_level(Traces, OriginLogLevel),
TRef = update_trace(Traces), TRef = update_trace(Traces),
{ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}. update_trace_handler(),
{ok, #{timer => TRef, monitors => #{}}}.
handle_call(check, _From, State) ->
{_, NewState} = handle_info({mnesia_table_event, check}, State),
{reply, ok, NewState};
handle_call(Req, _From, State) -> handle_call(Req, _From, State) ->
?SLOG(error, #{unexpected_call => Req}), ?SLOG(error, #{unexpected_call => Req}),
{reply, ok, State}. {reply, ok, State}.
@ -223,11 +257,10 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor
lists:foreach(fun file:delete/1, Files), lists:foreach(fun file:delete/1, Files),
{noreply, State#{monitors => NewMonitors}} {noreply, State#{monitors => NewMonitors}}
end; end;
handle_info({timeout, TRef, update_trace}, handle_info({timeout, TRef, update_trace}, #{timer := TRef} = State) ->
#{timer := TRef, primary_log_level := OriginLogLevel} = State) ->
Traces = get_enable_trace(), Traces = get_enable_trace(),
ok = update_log_primary_level(Traces, OriginLogLevel),
NextTRef = update_trace(Traces), NextTRef = update_trace(Traces),
update_trace_handler(),
{noreply, State#{timer => NextTRef}}; {noreply, State#{timer => NextTRef}};
handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) ->
@ -238,11 +271,11 @@ handle_info(Info, State) ->
?SLOG(error, #{unexpected_info => Info}), ?SLOG(error, #{unexpected_info => Info}),
{noreply, State}. {noreply, State}.
terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) -> terminate(_Reason, #{timer := TRef}) ->
ok = set_log_primary_level(OriginLogLevel),
_ = mnesia:unsubscribe({table, ?TRACE, simple}), _ = mnesia:unsubscribe({table, ?TRACE, simple}),
emqx_misc:cancel_timer(TRef), emqx_misc:cancel_timer(TRef),
stop_all_trace_handler(), stop_all_trace_handler(),
update_trace_handler(),
_ = file:del_dir_r(zip_dir()), _ = file:del_dir_r(zip_dir()),
ok. ok.
@ -270,7 +303,7 @@ update_trace(Traces) ->
disable_finished(Finished), disable_finished(Finished),
Started = emqx_trace_handler:running(), Started = emqx_trace_handler:running(),
{NeedRunning, AllStarted} = start_trace(Running, Started), {NeedRunning, AllStarted} = start_trace(Running, Started),
NeedStop = AllStarted -- NeedRunning, NeedStop = filter_cli_handler(AllStarted) -- NeedRunning,
ok = stop_trace(NeedStop, Started), ok = stop_trace(NeedStop, Started),
clean_stale_trace_files(), clean_stale_trace_files(),
NextTime = find_closest_time(Traces, Now), NextTime = find_closest_time(Traces, Now),
@ -308,10 +341,10 @@ disable_finished(Traces) ->
start_trace(Traces, Started0) -> start_trace(Traces, Started0) ->
Started = lists:map(fun(#{name := Name}) -> Name end, Started0), Started = lists:map(fun(#{name := Name}) -> Name end, Started0),
lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) -> lists:foldl(fun(#?TRACE{name = Name} = Trace,
{Running, StartedAcc}) ->
case lists:member(Name, StartedAcc) of case lists:member(Name, StartedAcc) of
true -> true -> {[Name | Running], StartedAcc};
{[Name | Running], StartedAcc};
false -> false ->
case start_trace(Trace) of case start_trace(Trace) of
ok -> {[Name | Running], [Name | StartedAcc]}; ok -> {[Name | Running], [Name | StartedAcc]};
@ -330,9 +363,11 @@ start_trace(Trace) ->
emqx_trace_handler:install(Who, debug, log_file(Name, Start)). emqx_trace_handler:install(Who, debug, log_file(Name, Start)).
stop_trace(Finished, Started) -> stop_trace(Finished, Started) ->
lists:foreach(fun(#{name := Name, type := Type}) -> lists:foreach(fun(#{name := Name, type := Type, filter := Filter}) ->
case lists:member(Name, Finished) of case lists:member(Name, Finished) of
true -> emqx_trace_handler:uninstall(Type, Name); true ->
?TRACE("API", "trace_stopping", #{Type => Filter}),
emqx_trace_handler:uninstall(Type, Name);
false -> ok false -> ok
end end
end, Started). end, Started).
@ -419,7 +454,7 @@ to_trace(#{type := ip_address, ip_address := Filter} = Trace, Rec) ->
case validate_ip_address(Filter) of case validate_ip_address(Filter) of
ok -> ok ->
Trace0 = maps:without([type, ip_address], Trace), Trace0 = maps:without([type, ip_address], Trace),
to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = Filter}); to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = binary_to_list(Filter)});
Error -> Error Error -> Error
end; end;
to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])}; to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])};
@ -481,11 +516,20 @@ transaction(Tran) ->
{aborted, Reason} -> {error, Reason} {aborted, Reason} -> {error, Reason}
end. end.
update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel); update_trace_handler() ->
update_log_primary_level(_, _) -> set_log_primary_level(debug). case emqx_trace_handler:running() of
[] -> persistent_term:erase(?TRACE_FILTER);
set_log_primary_level(NewLevel) -> Running ->
case NewLevel =/= emqx_logger:get_primary_log_level() of List = lists:map(fun(#{id := Id, filter_fun := FilterFun,
true -> emqx_logger:set_primary_log_level(NewLevel); filter := Filter, name := Name}) ->
{Id, FilterFun, Filter, Name} end, Running),
case List =/= persistent_term:get(?TRACE_FILTER, undefined) of
true -> persistent_term:put(?TRACE_FILTER, List);
false -> ok false -> ok
end
end. end.
filter_cli_handler(Names) ->
lists:filter(fun(Name) ->
nomatch =:= re:run(Name, "^CLI-+.", [])
end, Names).

View File

@ -0,0 +1,62 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_trace_formatter).
-export([format/2]).
%%%-----------------------------------------------------------------
%%% API
-spec format(LogEvent, Config) -> unicode:chardata() when
LogEvent :: logger:log_event(),
Config :: logger:config().
format(#{level := trace, event := Event, meta := Meta, msg := Msg},
#{payload_encode := PEncode}) ->
Time = calendar:system_time_to_rfc3339(erlang:system_time(second)),
ClientId = to_iolist(maps:get(clientid, Meta, "")),
Peername = maps:get(peername, Meta, ""),
MetaBin = format_meta(Meta, PEncode),
[Time, " [", Event, "] ", ClientId, "@", Peername, " msg: ", Msg, MetaBin, "\n"];
format(Event, Config) ->
emqx_logger_textfmt:format(Event, Config).
format_meta(Meta0, Encode) ->
Packet = format_packet(maps:get(packet, Meta0, undefined), Encode),
Payload = format_payload(maps:get(payload, Meta0, undefined), Encode),
Meta1 = maps:without([msg, clientid, peername, packet, payload], Meta0),
case Meta1 =:= #{} of
true -> [Packet, Payload];
false -> [Packet, ", ", map_to_iolist(Meta1), Payload]
end.
format_packet(undefined, _) -> "";
format_packet(Packet, Encode) -> [", packet: ", emqx_packet:format(Packet, Encode)].
format_payload(undefined, _) -> "";
format_payload(Payload, text) -> [", payload: ", io_lib:format("~ts", [Payload])];
format_payload(Payload, hex) -> [", payload(hex): ", emqx_packet:encode_hex(Payload)];
format_payload(_, hidden) -> ", payload=******".
to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom);
to_iolist(Int) when is_integer(Int) -> integer_to_list(Int);
to_iolist(Float) when is_float(Float) -> float_to_list(Float, [{decimals, 2}]);
to_iolist(SubMap) when is_map(SubMap) -> ["[", map_to_iolist(SubMap), "]"];
to_iolist(Char) -> emqx_logger_textfmt:try_format_unicode(Char).
map_to_iolist(Map) ->
lists:join(",",
lists:map(fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end,
maps:to_list(Map))).

View File

@ -25,6 +25,7 @@
-export([ running/0 -export([ running/0
, install/3 , install/3
, install/4 , install/4
, install/5
, uninstall/1 , uninstall/1
, uninstall/2 , uninstall/2
]). ]).
@ -36,6 +37,7 @@
]). ]).
-export([handler_id/2]). -export([handler_id/2]).
-export([payload_encode/0]).
-type tracer() :: #{ -type tracer() :: #{
name := binary(), name := binary(),
@ -77,22 +79,18 @@ install(Type, Filter, Level, LogFile) ->
-spec install(tracer(), logger:level() | all, string()) -> ok | {error, term()}. -spec install(tracer(), logger:level() | all, string()) -> ok | {error, term()}.
install(Who, all, LogFile) -> install(Who, all, LogFile) ->
install(Who, debug, LogFile); install(Who, debug, LogFile);
install(Who, Level, LogFile) -> install(Who = #{name := Name, type := Type}, Level, LogFile) ->
PrimaryLevel = emqx_logger:get_primary_log_level(), HandlerId = handler_id(Name, Type),
try logger:compare_levels(Level, PrimaryLevel) of Config = #{
lt -> level => Level,
{error, formatter => formatter(Who),
io_lib:format( filter_default => stop,
"Cannot trace at a log level (~s) " filters => filters(Who),
"lower than the primary log level (~s)", config => ?CONFIG(LogFile)
[Level, PrimaryLevel] },
)}; Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
_GtOrEq -> show_prompts(Res, Who, "start_trace"),
install_handler(Who, Level, LogFile) Res.
catch
error:badarg ->
{error, {invalid_log_level, Level}}
end.
-spec uninstall(Type :: clientid | topic | ip_address, -spec uninstall(Type :: clientid | topic | ip_address,
Name :: binary() | list()) -> ok | {error, term()}. Name :: binary() | list()) -> ok | {error, term()}.
@ -121,83 +119,59 @@ uninstall(HandlerId) ->
running() -> running() ->
lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)). lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)).
-spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. -spec filter_clientid(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop.
filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log; filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log;
filter_clientid(_Log, _ExpectId) -> ignore. filter_clientid(_Log, _ExpectId) -> stop.
-spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. -spec filter_topic(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop.
filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) -> filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) ->
case emqx_topic:match(Topic, TopicFilter) of case emqx_topic:match(Topic, TopicFilter) of
true -> Log; true -> Log;
false -> ignore false -> stop
end; end;
filter_topic(_Log, _ExpectId) -> ignore. filter_topic(_Log, _ExpectId) -> stop.
-spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore. -spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop.
filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) -> filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) ->
case lists:prefix(IP, Peername) of case lists:prefix(IP, Peername) of
true -> Log; true -> Log;
false -> ignore false -> stop
end; end;
filter_ip_address(_Log, _ExpectId) -> ignore. filter_ip_address(_Log, _ExpectId) -> stop.
install_handler(Who = #{name := Name, type := Type}, Level, LogFile) ->
HandlerId = handler_id(Name, Type),
Config = #{
level => Level,
formatter => formatter(Who),
filter_default => stop,
filters => filters(Who),
config => ?CONFIG(LogFile)
},
Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
show_prompts(Res, Who, "start_trace"),
Res.
filters(#{type := clientid, filter := Filter, name := Name}) -> filters(#{type := clientid, filter := Filter, name := Name}) ->
[{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}]; [{clientid, {fun ?MODULE:filter_clientid/2, {Filter, Name}}}];
filters(#{type := topic, filter := Filter, name := Name}) -> filters(#{type := topic, filter := Filter, name := Name}) ->
[{topic, {fun ?MODULE:filter_topic/2, {ensure_bin(Filter), Name}}}]; [{topic, {fun ?MODULE:filter_topic/2, {ensure_bin(Filter), Name}}}];
filters(#{type := ip_address, filter := Filter, name := Name}) -> filters(#{type := ip_address, filter := Filter, name := Name}) ->
[{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}].
formatter(#{type := Type}) -> formatter(#{type := _Type}) ->
{logger_formatter, {emqx_trace_formatter,
#{ #{
template => template(Type), %% template is for ?SLOG message not ?TRACE.
single_line => false, template => [time," [",level,"] ", msg,"\n"],
single_line => true,
max_size => unlimited, max_size => unlimited,
depth => unlimited depth => unlimited,
payload_encode => payload_encode()
} }
}. }.
%% Don't log clientid since clientid only supports exact match, all client ids are the same.
%% if clientid is not latin characters. the logger_formatter restricts the output must be `~tp`
%% (actually should use `~ts`), the utf8 characters clientid will become very difficult to read.
template(clientid) ->
[time, " [", level, "] ", {peername, [peername, " "], []}, msg, "\n"];
%% TODO better format when clientid is utf8.
template(_) ->
[time, " [", level, "] ",
{clientid,
[{peername, [clientid, "@", peername, " "], [clientid, " "]}],
[{peername, [peername, " "], []}]
},
msg, "\n"
].
filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) ->
Init = #{id => Id, level => Level, dst => Dst}, Init = #{id => Id, level => Level, dst => Dst},
case Filters of case Filters of
[{Type, {_FilterFun, {Filter, Name}}}] when [{Type, {FilterFun, {Filter, Name}}}] when
Type =:= topic orelse Type =:= topic orelse
Type =:= clientid orelse Type =:= clientid orelse
Type =:= ip_address -> Type =:= ip_address ->
[Init#{type => Type, filter => Filter, name => Name} | Acc]; [Init#{type => Type, filter => Filter, name => Name, filter_fun => FilterFun} | Acc];
_ -> _ ->
Acc Acc
end. end.
payload_encode() -> emqx_config:get([trace, payload_encode], text).
handler_id(Name, Type) -> handler_id(Name, Type) ->
try try
do_handler_id(Name, Type) do_handler_id(Name, Type)

View File

@ -347,7 +347,6 @@ websocket_handle({binary, Data}, State) when is_list(Data) ->
websocket_handle({binary, iolist_to_binary(Data)}, State); websocket_handle({binary, iolist_to_binary(Data)}, State);
websocket_handle({binary, Data}, State) -> websocket_handle({binary, Data}, State) ->
?SLOG(debug, #{msg => "RECV_data", data => Data, transport => websocket}),
State2 = ensure_stats_timer(State), State2 = ensure_stats_timer(State),
{Packets, State3} = parse_incoming(Data, [], State2), {Packets, State3} = parse_incoming(Data, [], State2),
LenMsg = erlang:length(Packets), LenMsg = erlang:length(Packets),
@ -432,11 +431,11 @@ websocket_info(Info, State) ->
websocket_close({_, ReasonCode, _Payload}, State) when is_integer(ReasonCode) -> websocket_close({_, ReasonCode, _Payload}, State) when is_integer(ReasonCode) ->
websocket_close(ReasonCode, State); websocket_close(ReasonCode, State);
websocket_close(Reason, State) -> websocket_close(Reason, State) ->
?SLOG(debug, #{msg => "websocket_closed", reason => Reason}), ?TRACE("SOCKET", "websocket_closed", #{reason => Reason}),
handle_info({sock_closed, Reason}, State). handle_info({sock_closed, Reason}, State).
terminate(Reason, _Req, #state{channel = Channel}) -> terminate(Reason, _Req, #state{channel = Channel}) ->
?SLOG(debug, #{msg => "terminated", reason => Reason}), ?TRACE("SOCKET", "websocket_terminated", #{reason => Reason}),
emqx_channel:terminate(Reason, Channel); emqx_channel:terminate(Reason, Channel);
terminate(_Reason, _Req, _UnExpectedState) -> terminate(_Reason, _Req, _UnExpectedState) ->
@ -480,7 +479,7 @@ handle_info({connack, ConnAck}, State) ->
return(enqueue(ConnAck, State)); return(enqueue(ConnAck, State));
handle_info({close, Reason}, State) -> handle_info({close, Reason}, State) ->
?SLOG(debug, #{msg => "force_socket_close", reason => Reason}), ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}),
return(enqueue({close, Reason}, State)); return(enqueue({close, Reason}, State));
handle_info({event, connected}, State = #state{channel = Channel}) -> handle_info({event, connected}, State = #state{channel = Channel}) ->
@ -550,7 +549,7 @@ check_limiter(Needs,
{ok, Limiter2} -> {ok, Limiter2} ->
WhenOk(Data, Msgs, State#state{limiter = Limiter2}); WhenOk(Data, Msgs, State#state{limiter = Limiter2});
{pause, Time, Limiter2} -> {pause, Time, Limiter2} ->
?SLOG(warning, #{msg => "pause time dueto rate limit", ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit",
needs => Needs, needs => Needs,
time_in_ms => Time}), time_in_ms => Time}),
@ -586,7 +585,7 @@ retry_limiter(#state{limiter = Limiter} = State) ->
, limiter_timer = undefined , limiter_timer = undefined
}); });
{pause, Time, Limiter2} -> {pause, Time, Limiter2} ->
?SLOG(warning, #{msg => "pause time dueto rate limit", ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit",
types => Types, types => Types,
time_in_ms => Time}), time_in_ms => Time}),
@ -663,7 +662,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
handle_incoming(Packet, State = #state{listener = {Type, Listener}}) handle_incoming(Packet, State = #state{listener = {Type, Listener}})
when is_record(Packet, mqtt_packet) -> when is_record(Packet, mqtt_packet) ->
?SLOG(debug, #{msg => "RECV", packet => emqx_packet:format(Packet)}), ?TRACE("WS-MQTT", "mqtt_packet_received", #{packet => Packet}),
ok = inc_incoming_stats(Packet), ok = inc_incoming_stats(Packet),
NState = case emqx_pd:get_counter(incoming_pubs) > NState = case emqx_pd:get_counter(incoming_pubs) >
get_active_n(Type, Listener) of get_active_n(Type, Listener) of
@ -727,7 +726,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped.too_large'),
ok = emqx_metrics:inc('delivery.dropped'), ok = emqx_metrics:inc('delivery.dropped'),
<<>>; <<>>;
Data -> ?SLOG(debug, #{msg => "SEND", packet => Packet}), Data -> ?TRACE("WS-MQTT", "mqtt_packet_sent", #{packet => Packet}),
ok = inc_outgoing_stats(Packet), ok = inc_outgoing_stats(Packet),
Data Data
catch catch

View File

@ -39,9 +39,13 @@ t_add_delete(_) ->
by = <<"banned suite">>, by = <<"banned suite">>,
reason = <<"test">>, reason = <<"test">>,
at = erlang:system_time(second), at = erlang:system_time(second),
until = erlang:system_time(second) + 1000 until = erlang:system_time(second) + 1
}, },
{ok, _} = emqx_banned:create(Banned), {ok, _} = emqx_banned:create(Banned),
{error, {already_exist, Banned}} = emqx_banned:create(Banned),
?assertEqual(1, emqx_banned:info(size)),
{error, {already_exist, Banned}} =
emqx_banned:create(Banned#banned{until = erlang:system_time(second) + 100}),
?assertEqual(1, emqx_banned:info(size)), ?assertEqual(1, emqx_banned:info(size)),
ok = emqx_banned:delete({clientid, <<"TestClient">>}), ok = emqx_banned:delete({clientid, <<"TestClient">>}),
@ -68,10 +72,14 @@ t_check(_) ->
username => <<"user">>, username => <<"user">>,
peerhost => {127,0,0,1} peerhost => {127,0,0,1}
}, },
ClientInfo5 = #{},
ClientInfo6 = #{clientid => <<"client1">>},
?assert(emqx_banned:check(ClientInfo1)), ?assert(emqx_banned:check(ClientInfo1)),
?assert(emqx_banned:check(ClientInfo2)), ?assert(emqx_banned:check(ClientInfo2)),
?assert(emqx_banned:check(ClientInfo3)), ?assert(emqx_banned:check(ClientInfo3)),
?assertNot(emqx_banned:check(ClientInfo4)), ?assertNot(emqx_banned:check(ClientInfo4)),
?assertNot(emqx_banned:check(ClientInfo5)),
?assertNot(emqx_banned:check(ClientInfo6)),
ok = emqx_banned:delete({clientid, <<"BannedClient">>}), ok = emqx_banned:delete({clientid, <<"BannedClient">>}),
ok = emqx_banned:delete({username, <<"BannedUser">>}), ok = emqx_banned:delete({username, <<"BannedUser">>}),
ok = emqx_banned:delete({peerhost, {192,168,0,1}}), ok = emqx_banned:delete({peerhost, {192,168,0,1}}),
@ -83,8 +91,10 @@ t_check(_) ->
t_unused(_) -> t_unused(_) ->
{ok, Banned} = emqx_banned:start_link(), {ok, Banned} = emqx_banned:start_link(),
{ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient">>}, {ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient1">>},
until = erlang:system_time(second)}), until = erlang:system_time(second)}),
{ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient2">>},
until = erlang:system_time(second) - 1}),
?assertEqual(ignored, gen_server:call(Banned, unexpected_req)), ?assertEqual(ignored, gen_server:call(Banned, unexpected_req)),
?assertEqual(ok, gen_server:cast(Banned, unexpected_msg)), ?assertEqual(ok, gen_server:cast(Banned, unexpected_msg)),
?assertEqual(ok, Banned ! ok), ?assertEqual(ok, Banned ! ok),

View File

@ -39,32 +39,29 @@ end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]). emqx_common_test_helpers:stop_apps([]).
init_per_testcase(t_trace_clientid, Config) -> init_per_testcase(t_trace_clientid, Config) ->
init(),
Config; Config;
init_per_testcase(_Case, Config) -> init_per_testcase(_Case, Config) ->
ok = emqx_logger:set_log_level(debug),
_ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()], _ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()],
init(),
Config. Config.
end_per_testcase(_Case, _Config) -> end_per_testcase(_Case, _Config) ->
ok = emqx_logger:set_log_level(warning), terminate(),
ok. ok.
t_trace_clientid(_Config) -> t_trace_clientid(_Config) ->
%% Start tracing %% Start tracing
emqx_logger:set_log_level(error),
{error, _} = emqx_trace_handler:install(clientid, <<"client">>, debug, "tmp/client.log"),
emqx_logger:set_log_level(debug),
%% add list clientid %% add list clientid
ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"), ok = emqx_trace_handler:install("CLI-client1", clientid, "client", debug, "tmp/client.log"),
ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"), ok = emqx_trace_handler:install("CLI-client2", clientid, <<"client2">>, all, "tmp/client2.log"),
ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"), ok = emqx_trace_handler:install("CLI-client3", clientid, <<"client3">>, all, "tmp/client3.log"),
{error, {invalid_log_level, bad_level}} =
emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"),
{error, {handler_not_added, {file_error, ".", eisdir}}} = {error, {handler_not_added, {file_error, ".", eisdir}}} =
emqx_trace_handler:install(clientid, <<"client5">>, debug, "."), emqx_trace_handler:install(clientid, <<"client5">>, debug, "."),
ok = filesync(<<"client">>, clientid), emqx_trace:check(),
ok = filesync(<<"client2">>, clientid), ok = filesync(<<"CLI-client1">>, clientid),
ok = filesync(<<"client3">>, clientid), ok = filesync(<<"CLI-client2">>, clientid),
ok = filesync(<<"CLI-client3">>, clientid),
%% Verify the tracing file exits %% Verify the tracing file exits
?assert(filelib:is_regular("tmp/client.log")), ?assert(filelib:is_regular("tmp/client.log")),
@ -72,11 +69,11 @@ t_trace_clientid(_Config) ->
?assert(filelib:is_regular("tmp/client3.log")), ?assert(filelib:is_regular("tmp/client3.log")),
%% Get current traces %% Get current traces
?assertMatch([#{type := clientid, filter := "client", name := <<"client">>, ?assertMatch([#{type := clientid, filter := <<"client">>, name := <<"CLI-client1">>,
level := debug, dst := "tmp/client.log"}, level := debug, dst := "tmp/client.log"},
#{type := clientid, filter := "client2", name := <<"client2">> #{type := clientid, filter := <<"client2">>, name := <<"CLI-client2">>
, level := debug, dst := "tmp/client2.log"}, , level := debug, dst := "tmp/client2.log"},
#{type := clientid, filter := "client3", name := <<"client3">>, #{type := clientid, filter := <<"client3">>, name := <<"CLI-client3">>,
level := debug, dst := "tmp/client3.log"} level := debug, dst := "tmp/client3.log"}
], emqx_trace_handler:running()), ], emqx_trace_handler:running()),
@ -85,9 +82,9 @@ t_trace_clientid(_Config) ->
emqtt:connect(T), emqtt:connect(T),
emqtt:publish(T, <<"a/b/c">>, <<"hi">>), emqtt:publish(T, <<"a/b/c">>, <<"hi">>),
emqtt:ping(T), emqtt:ping(T),
ok = filesync(<<"client">>, clientid), ok = filesync(<<"CLI-client1">>, clientid),
ok = filesync(<<"client2">>, clientid), ok = filesync(<<"CLI-client2">>, clientid),
ok = filesync(<<"client3">>, clientid), ok = filesync(<<"CLI-client3">>, clientid),
%% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log".
{ok, Bin} = file:read_file("tmp/client.log"), {ok, Bin} = file:read_file("tmp/client.log"),
@ -98,25 +95,24 @@ t_trace_clientid(_Config) ->
?assert(filelib:file_size("tmp/client2.log") == 0), ?assert(filelib:file_size("tmp/client2.log") == 0),
%% Stop tracing %% Stop tracing
ok = emqx_trace_handler:uninstall(clientid, <<"client">>), ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client1">>),
ok = emqx_trace_handler:uninstall(clientid, <<"client2">>), ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client2">>),
ok = emqx_trace_handler:uninstall(clientid, <<"client3">>), ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client3">>),
emqtt:disconnect(T), emqtt:disconnect(T),
?assertEqual([], emqx_trace_handler:running()). ?assertEqual([], emqx_trace_handler:running()).
t_trace_clientid_utf8(_) -> t_trace_clientid_utf8(_) ->
emqx_logger:set_log_level(debug),
Utf8Id = <<"client 漢字編碼"/utf8>>, Utf8Id = <<"client 漢字編碼"/utf8>>,
ok = emqx_trace_handler:install(clientid, Utf8Id, debug, "tmp/client-utf8.log"), ok = emqx_trace_handler:install("CLI-UTF8", clientid, Utf8Id, debug, "tmp/client-utf8.log"),
emqx_trace:check(),
{ok, T} = emqtt:start_link([{clientid, Utf8Id}]), {ok, T} = emqtt:start_link([{clientid, Utf8Id}]),
emqtt:connect(T), emqtt:connect(T),
[begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)], [begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)],
emqtt:ping(T), emqtt:ping(T),
ok = filesync(Utf8Id, clientid), ok = filesync("CLI-UTF8", clientid),
ok = emqx_trace_handler:uninstall(clientid, Utf8Id), ok = emqx_trace_handler:uninstall(clientid, "CLI-UTF8"),
emqtt:disconnect(T), emqtt:disconnect(T),
?assertEqual([], emqx_trace_handler:running()), ?assertEqual([], emqx_trace_handler:running()),
ok. ok.
@ -126,11 +122,11 @@ t_trace_topic(_Config) ->
emqtt:connect(T), emqtt:connect(T),
%% Start tracing %% Start tracing
emqx_logger:set_log_level(debug), ok = emqx_trace_handler:install("CLI-TOPIC-1", topic, <<"x/#">>, all, "tmp/topic_trace_x.log"),
ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), ok = emqx_trace_handler:install("CLI-TOPIC-2", topic, <<"y/#">>, all, "tmp/topic_trace_y.log"),
ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), emqx_trace:check(),
ok = filesync(<<"x/#">>, topic), ok = filesync("CLI-TOPIC-1", topic),
ok = filesync(<<"y/#">>, topic), ok = filesync("CLI-TOPIC-2", topic),
%% Verify the tracing file exits %% Verify the tracing file exits
?assert(filelib:is_regular("tmp/topic_trace_x.log")), ?assert(filelib:is_regular("tmp/topic_trace_x.log")),
@ -138,9 +134,9 @@ t_trace_topic(_Config) ->
%% Get current traces %% Get current traces
?assertMatch([#{type := topic, filter := <<"x/#">>, ?assertMatch([#{type := topic, filter := <<"x/#">>,
level := debug, dst := "tmp/topic_trace_x.log", name := <<"x/#">>}, level := debug, dst := "tmp/topic_trace_x.log", name := <<"CLI-TOPIC-1">>},
#{type := topic, filter := <<"y/#">>, #{type := topic, filter := <<"y/#">>,
name := <<"y/#">>, level := debug, dst := "tmp/topic_trace_y.log"} name := <<"CLI-TOPIC-2">>, level := debug, dst := "tmp/topic_trace_y.log"}
], ],
emqx_trace_handler:running()), emqx_trace_handler:running()),
@ -149,8 +145,8 @@ t_trace_topic(_Config) ->
emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
emqtt:subscribe(T, <<"x/y/z">>), emqtt:subscribe(T, <<"x/y/z">>),
emqtt:unsubscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>),
ok = filesync(<<"x/#">>, topic), ok = filesync("CLI-TOPIC-1", topic),
ok = filesync(<<"y/#">>, topic), ok = filesync("CLI-TOPIC-2", topic),
{ok, Bin} = file:read_file("tmp/topic_trace_x.log"), {ok, Bin} = file:read_file("tmp/topic_trace_x.log"),
?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@ -161,8 +157,8 @@ t_trace_topic(_Config) ->
?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0), ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0),
%% Stop tracing %% Stop tracing
ok = emqx_trace_handler:uninstall(topic, <<"x/#">>), ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-1">>),
ok = emqx_trace_handler:uninstall(topic, <<"y/#">>), ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-2">>),
{error, _Reason} = emqx_trace_handler:uninstall(topic, <<"z/#">>), {error, _Reason} = emqx_trace_handler:uninstall(topic, <<"z/#">>),
?assertEqual([], emqx_trace_handler:running()), ?assertEqual([], emqx_trace_handler:running()),
emqtt:disconnect(T). emqtt:disconnect(T).
@ -172,10 +168,12 @@ t_trace_ip_address(_Config) ->
emqtt:connect(T), emqtt:connect(T),
%% Start tracing %% Start tracing
ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), ok = emqx_trace_handler:install("CLI-IP-1", ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"),
ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"), ok = emqx_trace_handler:install("CLI-IP-2", ip_address,
ok = filesync(<<"127.0.0.1">>, ip_address), "192.168.1.1", all, "tmp/ip_trace_y.log"),
ok = filesync(<<"192.168.1.1">>, ip_address), emqx_trace:check(),
ok = filesync(<<"CLI-IP-1">>, ip_address),
ok = filesync(<<"CLI-IP-2">>, ip_address),
%% Verify the tracing file exits %% Verify the tracing file exits
?assert(filelib:is_regular("tmp/ip_trace_x.log")), ?assert(filelib:is_regular("tmp/ip_trace_x.log")),
@ -183,10 +181,10 @@ t_trace_ip_address(_Config) ->
%% Get current traces %% Get current traces
?assertMatch([#{type := ip_address, filter := "127.0.0.1", ?assertMatch([#{type := ip_address, filter := "127.0.0.1",
name := <<"127.0.0.1">>, name := <<"CLI-IP-1">>,
level := debug, dst := "tmp/ip_trace_x.log"}, level := debug, dst := "tmp/ip_trace_x.log"},
#{type := ip_address, filter := "192.168.1.1", #{type := ip_address, filter := "192.168.1.1",
name := <<"192.168.1.1">>, name := <<"CLI-IP-2">>,
level := debug, dst := "tmp/ip_trace_y.log"} level := debug, dst := "tmp/ip_trace_y.log"}
], ],
emqx_trace_handler:running()), emqx_trace_handler:running()),
@ -196,8 +194,8 @@ t_trace_ip_address(_Config) ->
emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
emqtt:subscribe(T, <<"x/y/z">>), emqtt:subscribe(T, <<"x/y/z">>),
emqtt:unsubscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>),
ok = filesync(<<"127.0.0.1">>, ip_address), ok = filesync(<<"CLI-IP-1">>, ip_address),
ok = filesync(<<"192.168.1.1">>, ip_address), ok = filesync(<<"CLI-IP-2">>, ip_address),
{ok, Bin} = file:read_file("tmp/ip_trace_x.log"), {ok, Bin} = file:read_file("tmp/ip_trace_x.log"),
?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@ -208,8 +206,8 @@ t_trace_ip_address(_Config) ->
?assert(filelib:file_size("tmp/ip_trace_y.log") =:= 0), ?assert(filelib:file_size("tmp/ip_trace_y.log") =:= 0),
%% Stop tracing %% Stop tracing
ok = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.1">>), ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-1">>),
ok = emqx_trace_handler:uninstall(ip_address, <<"192.168.1.1">>), ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-2">>),
{error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>), {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>),
emqtt:disconnect(T), emqtt:disconnect(T),
?assertEqual([], emqx_trace_handler:running()). ?assertEqual([], emqx_trace_handler:running()).
@ -221,7 +219,12 @@ filesync(Name, Type) ->
%% sometime the handler process is not started yet. %% sometime the handler process is not started yet.
filesync(_Name, _Type, 0) -> ok; filesync(_Name, _Type, 0) -> ok;
filesync(Name, Type, Retry) -> filesync(Name0, Type, Retry) ->
Name =
case is_binary(Name0) of
true -> Name0;
false -> list_to_binary(Name0)
end,
try try
Handler = binary_to_atom(<<"trace_", Handler = binary_to_atom(<<"trace_",
(atom_to_binary(Type))/binary, "_", Name/binary>>), (atom_to_binary(Type))/binary, "_", Name/binary>>),
@ -231,3 +234,9 @@ filesync(Name, Type, Retry) ->
ct:sleep(100), ct:sleep(100),
filesync(Name, Type, Retry - 1) filesync(Name, Type, Retry - 1)
end. end.
init() ->
emqx_trace:start_link().
terminate() ->
catch ok = gen_server:stop(emqx_trace, normal, 5000).

View File

@ -872,7 +872,7 @@ fill_defaults(Configs) when is_list(Configs) ->
fill_defaults(Config) -> fill_defaults(Config) ->
emqx_authn:check_config(Config, #{only_fill_defaults => true}). emqx_authn:check_config(Config, #{only_fill_defaults => true}).
convert_certs(#{ssl := SSLOpts} = Config) -> convert_certs(#{ssl := #{enable := true} = SSLOpts} = Config) ->
NSSLOpts = lists:foldl(fun(K, Acc) -> NSSLOpts = lists:foldl(fun(K, Acc) ->
case maps:get(K, Acc, undefined) of case maps:get(K, Acc, undefined) of
undefined -> Acc; undefined -> Acc;
@ -979,7 +979,7 @@ authenticator_examples() ->
mechanism => <<"password-based">>, mechanism => <<"password-based">>,
backend => <<"http">>, backend => <<"http">>,
method => <<"post">>, method => <<"post">>,
url => <<"http://127.0.0.2:8080">>, url => <<"http://127.0.0.1:18083">>,
headers => #{ headers => #{
<<"content-type">> => <<"application/json">> <<"content-type">> => <<"application/json">>
}, },

View File

@ -106,7 +106,7 @@ authenticate(#{password := Password} = Credential,
resource_id := ResourceId, resource_id := ResourceId,
password_hash_algorithm := Algorithm}) -> password_hash_algorithm := Algorithm}) ->
Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential),
case emqx_resource:query(ResourceId, {sql, Query, Params}) of case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Query, Params}) of
{ok, _Columns, []} -> ignore; {ok, _Columns, []} -> ignore;
{ok, Columns, [Row | _]} -> {ok, Columns, [Row | _]} ->
NColumns = [Name || #column{name = Name} <- Columns], NColumns = [Name || #column{name = Name} <- Columns],

View File

@ -67,7 +67,7 @@ init_per_suite(Config) ->
Config. Config.
end_per_suite(_Config) -> end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx_authn, emqx_dashboard]), emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authn]),
ok. ok.
set_special_configs(emqx_dashboard) -> set_special_configs(emqx_dashboard) ->

View File

@ -153,9 +153,8 @@ t_destroy(_Config) ->
?GLOBAL), ?GLOBAL),
% Authenticator should not be usable anymore % Authenticator should not be usable anymore
?assertException( ?assertMatch(
error, ignore,
_,
emqx_authn_http:authenticate( emqx_authn_http:authenticate(
Credentials, Credentials,
State)). State)).

View File

@ -146,9 +146,8 @@ t_destroy(_Config) ->
?GLOBAL), ?GLOBAL),
% Authenticator should not be usable anymore % Authenticator should not be usable anymore
?assertException( ?assertMatch(
error, ignore,
_,
emqx_authn_mongodb:authenticate( emqx_authn_mongodb:authenticate(
#{username => <<"plain">>, #{username => <<"plain">>,
password => <<"plain">> password => <<"plain">>

View File

@ -91,7 +91,7 @@ t_create_invalid_server_name(_Config) ->
create_mongo_auth_with_ssl_opts( create_mongo_auth_with_ssl_opts(
#{<<"server_name_indication">> => <<"authn-server-unknown-host">>, #{<<"server_name_indication">> => <<"authn-server-unknown-host">>,
<<"verify">> => <<"verify_peer">>}), <<"verify">> => <<"verify_peer">>}),
fun({ok, _}, Trace) -> fun({error, _}, Trace) ->
?assertEqual( ?assertEqual(
[failed], [failed],
?projection( ?projection(
@ -109,7 +109,7 @@ t_create_invalid_version(_Config) ->
#{<<"server_name_indication">> => <<"authn-server">>, #{<<"server_name_indication">> => <<"authn-server">>,
<<"verify">> => <<"verify_peer">>, <<"verify">> => <<"verify_peer">>,
<<"versions">> => [<<"tlsv1.1">>]}), <<"versions">> => [<<"tlsv1.1">>]}),
fun({ok, _}, Trace) -> fun({error, _}, Trace) ->
?assertEqual( ?assertEqual(
[failed], [failed],
?projection( ?projection(
@ -128,7 +128,7 @@ t_invalid_ciphers(_Config) ->
<<"verify">> => <<"verify_peer">>, <<"verify">> => <<"verify_peer">>,
<<"versions">> => [<<"tlsv1.2">>], <<"versions">> => [<<"tlsv1.2">>],
<<"ciphers">> => [<<"DHE-RSA-AES256-GCM-SHA384">>]}), <<"ciphers">> => [<<"DHE-RSA-AES256-GCM-SHA384">>]}),
fun({ok, _}, Trace) -> fun({error, _}, Trace) ->
?assertEqual( ?assertEqual(
[failed], [failed],
?projection( ?projection(

View File

@ -157,9 +157,8 @@ t_destroy(_Config) ->
?GLOBAL), ?GLOBAL),
% Authenticator should not be usable anymore % Authenticator should not be usable anymore
?assertException( ?assertMatch(
error, ignore,
_,
emqx_authn_mysql:authenticate( emqx_authn_mysql:authenticate(
#{username => <<"plain">>, #{username => <<"plain">>,
password => <<"plain">> password => <<"plain">>

View File

@ -158,9 +158,8 @@ t_destroy(_Config) ->
?GLOBAL), ?GLOBAL),
% Authenticator should not be usable anymore % Authenticator should not be usable anymore
?assertException( ?assertMatch(
error, ignore,
_,
emqx_authn_pgsql:authenticate( emqx_authn_pgsql:authenticate(
#{username => <<"plain">>, #{username => <<"plain">>,
password => <<"plain">> password => <<"plain">>
@ -440,12 +439,12 @@ create_user(Values) ->
q(Sql) -> q(Sql) ->
emqx_resource:query( emqx_resource:query(
?PGSQL_RESOURCE, ?PGSQL_RESOURCE,
{sql, Sql}). {query, Sql}).
q(Sql, Params) -> q(Sql, Params) ->
emqx_resource:query( emqx_resource:query(
?PGSQL_RESOURCE, ?PGSQL_RESOURCE,
{sql, Sql, Params}). {query, Sql, Params}).
drop_seeds() -> drop_seeds() ->
{ok, _, _} = q("DROP TABLE IF EXISTS users"), {ok, _, _} = q("DROP TABLE IF EXISTS users"),

View File

@ -164,9 +164,8 @@ t_destroy(_Config) ->
?GLOBAL), ?GLOBAL),
% Authenticator should not be usable anymore % Authenticator should not be usable anymore
?assertException( ?assertMatch(
error, ignore,
_,
emqx_authn_redis:authenticate( emqx_authn_redis:authenticate(
#{username => <<"plain">>, #{username => <<"plain">>,
password => <<"plain">> password => <<"plain">>

View File

@ -31,9 +31,7 @@
, lookup/0 , lookup/0
, lookup/1 , lookup/1
, move/2 , move/2
, move/3
, update/2 , update/2
, update/3
, authorize/5 , authorize/5
]). ]).
@ -112,28 +110,19 @@ lookup(Type) ->
{Source, _Front, _Rear} = take(Type), {Source, _Front, _Rear} = take(Type),
Source. Source.
move(Type, Cmd) -> move(Type, #{<<"before">> := Before}) ->
move(Type, Cmd, #{}). emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))});
move(Type, #{<<"after">> := After}) ->
move(Type, #{<<"before">> := Before}, Opts) -> emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))});
emqx:update_config( ?CONF_KEY_PATH move(Type, Position) ->
, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts); emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}).
move(Type, #{<<"after">> := After}, Opts) ->
emqx:update_config( ?CONF_KEY_PATH
, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts);
move(Type, Position, Opts) ->
emqx:update_config( ?CONF_KEY_PATH
, {?CMD_MOVE, type(Type), Position}, Opts).
update({?CMD_REPLACE, Type}, Sources) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources});
update({?CMD_DELETE, Type}, Sources) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_DELETE, type(Type)}, Sources});
update(Cmd, Sources) -> update(Cmd, Sources) ->
update(Cmd, Sources, #{}). emqx_authz_utils:update_config(?CONF_KEY_PATH, {Cmd, Sources}).
update({?CMD_REPLACE, Type}, Sources, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources}, Opts);
update({?CMD_DELETE, Type}, Sources, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {{?CMD_DELETE, type(Type)}, Sources}, Opts);
update(Cmd, Sources, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}, Opts).
do_update({?CMD_MOVE, Type, ?CMD_MOVE_TOP}, Conf) when is_list(Conf) -> do_update({?CMD_MOVE, Type, ?CMD_MOVE_TOP}, Conf) when is_list(Conf) ->
{Source, Front, Rear} = take(Type, Conf), {Source, Front, Rear} = take(Type, Conf),
@ -167,7 +156,8 @@ do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := Enable} = Source}, Conf)
NConf; NConf;
{error, _} = Error -> Error {error, _} = Error -> Error
end; end;
do_update({{?CMD_REPLACE, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> do_update({{?CMD_REPLACE, Type}, Source}, Conf)
when is_map(Source), is_list(Conf) ->
{_Old, Front, Rear} = take(Type, Conf), {_Old, Front, Rear} = take(Type, Conf),
NConf = Front ++ [Source | Rear], NConf = Front ++ [Source | Rear],
ok = check_dup_types(NConf), ok = check_dup_types(NConf),

View File

@ -182,8 +182,7 @@ definitions() ->
mongo_type => #{type => string, mongo_type => #{type => string,
enum => [<<"rs">>], enum => [<<"rs">>],
example => <<"rs">>}, example => <<"rs">>},
servers => #{type => array, servers => #{type => string, example => <<"127.0.0.1:27017, 127.0.0.2:27017">>},
items => #{type => string,example => <<"127.0.0.1:27017">>}},
replica_set_name => #{type => string}, replica_set_name => #{type => string},
pool_size => #{type => integer}, pool_size => #{type => integer},
username => #{type => string}, username => #{type => string},
@ -240,8 +239,7 @@ definitions() ->
mongo_type => #{type => string, mongo_type => #{type => string,
enum => [<<"sharded">>], enum => [<<"sharded">>],
example => <<"sharded">>}, example => <<"sharded">>},
servers => #{type => array, servers => #{type => string,example => <<"127.0.0.1:27017, 127.0.0.2:27017">>},
items => #{type => string,example => <<"127.0.0.1:27017">>}},
pool_size => #{type => integer}, pool_size => #{type => integer},
username => #{type => string}, username => #{type => string},
password => #{type => string}, password => #{type => string},
@ -401,8 +399,7 @@ definitions() ->
type => string, type => string,
example => <<"HGETALL mqtt_authz">> example => <<"HGETALL mqtt_authz">>
}, },
servers => #{type => array, servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>},
items => #{type => string,example => <<"127.0.0.1:3306">>}},
redis_type => #{type => string, redis_type => #{type => string,
enum => [<<"sentinel">>], enum => [<<"sentinel">>],
example => <<"sentinel">>}, example => <<"sentinel">>},
@ -438,8 +435,7 @@ definitions() ->
type => string, type => string,
example => <<"HGETALL mqtt_authz">> example => <<"HGETALL mqtt_authz">>
}, },
servers => #{type => array, servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>},
items => #{type => string, example => <<"127.0.0.1:3306">>}},
redis_type => #{type => string, redis_type => #{type => string,
enum => [<<"cluster">>], enum => [<<"cluster">>],
example => <<"cluster">>}, example => <<"cluster">>},

View File

@ -54,8 +54,9 @@ settings(get, _Params) ->
settings(put, #{body := #{<<"no_match">> := NoMatch, settings(put, #{body := #{<<"no_match">> := NoMatch,
<<"deny_action">> := DenyAction, <<"deny_action">> := DenyAction,
<<"cache">> := Cache}}) -> <<"cache">> := Cache}}) ->
{ok, _} = emqx:update_config([authorization, no_match], NoMatch), {ok, _} = emqx_authz_utils:update_config([authorization, no_match], NoMatch),
{ok, _} = emqx:update_config([authorization, deny_action], DenyAction), {ok, _} = emqx_authz_utils:update_config(
{ok, _} = emqx:update_config([authorization, cache], Cache), [authorization, deny_action], DenyAction),
{ok, _} = emqx_authz_utils:update_config([authorization, cache], Cache),
ok = emqx_authz_cache:drain_cache(), ok = emqx_authz_cache:drain_cache(),
{200, authorization_settings()}. {200, authorization_settings()}.

View File

@ -46,10 +46,10 @@ init(Source) ->
end. end.
destroy(#{annotations := #{id := Id}}) -> destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id). ok = emqx_resource:remove_local(Id).
dry_run(Source) -> dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_http, Source). emqx_resource:create_dry_run_local(emqx_connector_http, Source).
authorize(Client, PubSub, Topic, authorize(Client, PubSub, Topic,
#{type := http, #{type := http,

View File

@ -46,10 +46,10 @@ init(Source) ->
end. end.
dry_run(Source) -> dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_mongo, Source). emqx_resource:create_dry_run_local(emqx_connector_mongo, Source).
destroy(#{annotations := #{id := Id}}) -> destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id). ok = emqx_resource:remove_local(Id).
authorize(Client, PubSub, Topic, authorize(Client, PubSub, Topic,
#{collection := Collection, #{collection := Collection,

View File

@ -48,10 +48,10 @@ init(#{query := SQL} = Source) ->
end. end.
dry_run(Source) -> dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_mysql, Source). emqx_resource:create_dry_run_local(emqx_connector_mysql, Source).
destroy(#{annotations := #{id := Id}}) -> destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id). ok = emqx_resource:remove_local(Id).
authorize(Client, PubSub, Topic, authorize(Client, PubSub, Topic,
#{annotations := #{id := ResourceID, #{annotations := #{id := ResourceID,

View File

@ -48,10 +48,10 @@ init(#{query := SQL} = Source) ->
end. end.
destroy(#{annotations := #{id := Id}}) -> destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id). ok = emqx_resource:remove_local(Id).
dry_run(Source) -> dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_pgsql, Source). emqx_resource:create_dry_run_local(emqx_connector_pgsql, Source).
parse_query(Sql) -> parse_query(Sql) ->
case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of
@ -73,7 +73,7 @@ authorize(Client, PubSub, Topic,
query := {Query, Params} query := {Query, Params}
} }
}) -> }) ->
case emqx_resource:query(ResourceID, {sql, Query, replvar(Params, Client)}) of case emqx_resource:query(ResourceID, {prepared_query, ResourceID, Query, replvar(Params, Client)}) of
{ok, _Columns, []} -> nomatch; {ok, _Columns, []} -> nomatch;
{ok, Columns, Rows} -> {ok, Columns, Rows} ->
do_authorize(Client, PubSub, Topic, Columns, Rows); do_authorize(Client, PubSub, Topic, Columns, Rows);

View File

@ -46,10 +46,10 @@ init(Source) ->
end. end.
destroy(#{annotations := #{id := Id}}) -> destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id). ok = emqx_resource:remove_local(Id).
dry_run(Source) -> dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_redis, Source). emqx_resource:create_dry_run_local(emqx_connector_redis, Source).
authorize(Client, PubSub, Topic, authorize(Client, PubSub, Topic,
#{cmd := CMD, #{cmd := CMD,

View File

@ -18,9 +18,11 @@
-include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl").
-export([cleanup_resources/0, -export([ cleanup_resources/0
make_resource_id/1, , make_resource_id/1
create_resource/2]). , create_resource/2
, update_config/2
]).
-define(RESOURCE_GROUP, <<"emqx_authz">>). -define(RESOURCE_GROUP, <<"emqx_authz">>).
@ -30,7 +32,7 @@
create_resource(Module, Config) -> create_resource(Module, Config) ->
ResourceID = make_resource_id(Module), ResourceID = make_resource_id(Module),
case emqx_resource:create(ResourceID, Module, Config) of case emqx_resource:create_local(ResourceID, Module, Config) of
{ok, already_created} -> {ok, ResourceID}; {ok, already_created} -> {ok, ResourceID};
{ok, _} -> {ok, ResourceID}; {ok, _} -> {ok, ResourceID};
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
@ -38,13 +40,17 @@ create_resource(Module, Config) ->
cleanup_resources() -> cleanup_resources() ->
lists:foreach( lists:foreach(
fun emqx_resource:remove/1, fun emqx_resource:remove_local/1,
emqx_resource:list_group_instances(?RESOURCE_GROUP)). emqx_resource:list_group_instances(?RESOURCE_GROUP)).
make_resource_id(Name) -> make_resource_id(Name) ->
NameBin = bin(Name), NameBin = bin(Name),
emqx_resource:generate_id(?RESOURCE_GROUP, NameBin). emqx_resource:generate_id(?RESOURCE_GROUP, NameBin).
update_config(Path, ConfigRequest) ->
emqx_conf:update(Path, ConfigRequest, #{rawconf_with_defaults => true,
override_to => cluster}).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Internal functions %% Internal functions
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------

View File

@ -31,10 +31,9 @@ groups() ->
init_per_suite(Config) -> init_per_suite(Config) ->
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end), meck:expect(emqx_resource, remove_local, fun(_) -> ok end),
meck:expect(emqx_resource, remove, fun(_) -> ok end), meck:expect(emqx_resource, create_dry_run_local, fun(_, _) -> ok end),
meck:expect(emqx_resource, create_dry_run, fun(_, _) -> ok end),
ok = emqx_common_test_helpers:start_apps( ok = emqx_common_test_helpers:start_apps(
[emqx_connector, emqx_conf, emqx_authz], [emqx_connector, emqx_conf, emqx_authz],
@ -105,6 +104,7 @@ set_special_configs(_App) ->
<<"query">> => <<"abcb">> <<"query">> => <<"abcb">>
}). }).
-define(SOURCE5, #{<<"type">> => <<"redis">>, -define(SOURCE5, #{<<"type">> => <<"redis">>,
<<"redis_type">> => <<"single">>,
<<"enable">> => true, <<"enable">> => true,
<<"server">> => <<"127.0.0.1:27017">>, <<"server">> => <<"127.0.0.1:27017">>,
<<"pool_size">> => 1, <<"pool_size">> => 1,

View File

@ -26,6 +26,7 @@
-define(HOST, "http://127.0.0.1:18083/"). -define(HOST, "http://127.0.0.1:18083/").
-define(API_VERSION, "v5"). -define(API_VERSION, "v5").
-define(BASE_PATH, "api"). -define(BASE_PATH, "api").
-define(MONGO_SINGLE_HOST, "mongo:27017").
-define(SOURCE1, #{<<"type">> => <<"http">>, -define(SOURCE1, #{<<"type">> => <<"http">>,
<<"enable">> => true, <<"enable">> => true,
@ -38,8 +39,8 @@
}). }).
-define(SOURCE2, #{<<"type">> => <<"mongodb">>, -define(SOURCE2, #{<<"type">> => <<"mongodb">>,
<<"enable">> => true, <<"enable">> => true,
<<"mongo_type">> => <<"sharded">>, <<"mongo_type">> => <<"single">>,
<<"servers">> => <<"127.0.0.1:27017,192.168.0.1:27017">>, <<"server">> => <<?MONGO_SINGLE_HOST>>,
<<"pool_size">> => 1, <<"pool_size">> => 1,
<<"database">> => <<"mqtt">>, <<"database">> => <<"mqtt">>,
<<"ssl">> => #{<<"enable">> => false}, <<"ssl">> => #{<<"enable">> => false},
@ -48,7 +49,7 @@
}). }).
-define(SOURCE3, #{<<"type">> => <<"mysql">>, -define(SOURCE3, #{<<"type">> => <<"mysql">>,
<<"enable">> => true, <<"enable">> => true,
<<"server">> => <<"127.0.0.1:3306">>, <<"server">> => <<"mysql:3306">>,
<<"pool_size">> => 1, <<"pool_size">> => 1,
<<"database">> => <<"mqtt">>, <<"database">> => <<"mqtt">>,
<<"username">> => <<"xx">>, <<"username">> => <<"xx">>,
@ -59,7 +60,7 @@
}). }).
-define(SOURCE4, #{<<"type">> => <<"postgresql">>, -define(SOURCE4, #{<<"type">> => <<"postgresql">>,
<<"enable">> => true, <<"enable">> => true,
<<"server">> => <<"127.0.0.1:5432">>, <<"server">> => <<"pgsql:5432">>,
<<"pool_size">> => 1, <<"pool_size">> => 1,
<<"database">> => <<"mqtt">>, <<"database">> => <<"mqtt">>,
<<"username">> => <<"xx">>, <<"username">> => <<"xx">>,
@ -70,9 +71,7 @@
}). }).
-define(SOURCE5, #{<<"type">> => <<"redis">>, -define(SOURCE5, #{<<"type">> => <<"redis">>,
<<"enable">> => true, <<"enable">> => true,
<<"servers">> => [<<"127.0.0.1:6379">>, <<"servers">> => <<"redis:6379,127.0.0.1:6380">>,
<<"127.0.0.1:6380">>
],
<<"pool_size">> => 1, <<"pool_size">> => 1,
<<"database">> => 0, <<"database">> => 0,
<<"password">> => <<"ee">>, <<"password">> => <<"ee">>,
@ -98,14 +97,14 @@ groups() ->
init_per_suite(Config) -> init_per_suite(Config) ->
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, create_dry_run, meck:expect(emqx_resource, create_dry_run_local,
fun(emqx_connector_mysql, _) -> ok; fun(emqx_connector_mysql, _) -> ok;
(emqx_connector_mongo, _) -> ok;
(T, C) -> meck:passthrough([T, C]) (T, C) -> meck:passthrough([T, C])
end), end),
meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end), meck:expect(emqx_resource, health_check, fun(St) -> {ok, St} end),
meck:expect(emqx_resource, health_check, fun(_) -> ok end), meck:expect(emqx_resource, remove_local, fun(_) -> ok end ),
meck:expect(emqx_resource, remove, fun(_) -> ok end ),
ok = emqx_common_test_helpers:start_apps( ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authz, emqx_dashboard], [emqx_conf, emqx_authz, emqx_dashboard],

View File

@ -343,17 +343,16 @@ t_create_replace(_Config) ->
listener => {tcp, default} listener => {tcp, default}
}, },
%% Bad URL %% Create with valid URL
ok = setup_handler_and_config( ok = setup_handler_and_config(
fun(Req0, State) -> fun(Req0, State) ->
Req = cowboy_req:reply(200, Req0), Req = cowboy_req:reply(200, Req0),
{ok, Req, State} {ok, Req, State}
end, end,
#{<<"base_url">> => <<"http://127.0.0.1:33331/authz">>}), #{<<"base_url">> => <<"http://127.0.0.1:33333/authz">>}),
?assertEqual( ?assertEqual(
deny, allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)), emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
%% Changing to other bad config does not work %% Changing to other bad config does not work
@ -366,7 +365,7 @@ t_create_replace(_Config) ->
emqx_authz:update({?CMD_REPLACE, http}, BadConfig)), emqx_authz:update({?CMD_REPLACE, http}, BadConfig)),
?assertEqual( ?assertEqual(
deny, allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)), emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
%% Changing to valid config %% Changing to valid config

View File

@ -228,12 +228,12 @@ raw_pgsql_authz_config() ->
q(Sql) -> q(Sql) ->
emqx_resource:query( emqx_resource:query(
?PGSQL_RESOURCE, ?PGSQL_RESOURCE,
{sql, Sql}). {query, Sql}).
insert(Sql, Params) -> insert(Sql, Params) ->
{ok, _} = emqx_resource:query( {ok, _} = emqx_resource:query(
?PGSQL_RESOURCE, ?PGSQL_RESOURCE,
{sql, Sql, Params}), {query, Sql, Params}),
ok. ok.
init_table() -> init_table() ->

View File

@ -80,8 +80,15 @@ format(Rule = #{topic := Topic}) when is_map(Rule) ->
}. }.
update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE -> update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE ->
{ok, _} = emqx:update_config([auto_subscribe, topics], Topics), case emqx_conf:update([auto_subscribe, topics],
update_hook(); Topics,
#{rawconf_with_defaults => true, override_to => cluster}) of
{ok, #{raw_config := NewTopics}} ->
ok = update_hook(),
{ok, NewTopics};
{error, Reason} ->
{error, Reason}
end;
update_(_Topics) -> update_(_Topics) ->
{error, quota_exceeded}. {error, quota_exceeded}.

View File

@ -22,6 +22,7 @@
-export([auto_subscribe/2]). -export([auto_subscribe/2]).
-define(INTERNAL_ERROR, 'INTERNAL_ERROR').
-define(EXCEED_LIMIT, 'EXCEED_LIMIT'). -define(EXCEED_LIMIT, 'EXCEED_LIMIT').
-define(BAD_REQUEST, 'BAD_REQUEST'). -define(BAD_REQUEST, 'BAD_REQUEST').
@ -90,6 +91,9 @@ auto_subscribe(put, #{body := Params}) ->
Message = list_to_binary(io_lib:format("Max auto subscribe topic count is ~p", Message = list_to_binary(io_lib:format("Max auto subscribe topic count is ~p",
[emqx_auto_subscribe:max_limit()])), [emqx_auto_subscribe:max_limit()])),
{409, #{code => ?EXCEED_LIMIT, message => Message}}; {409, #{code => ?EXCEED_LIMIT, message => Message}};
ok -> {error, Reason} ->
{200, emqx_auto_subscribe:list()} Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])),
{500, #{code => ?INTERNAL_ERROR, message => Message}};
{ok, NewTopics} ->
{200, NewTopics}
end. end.

View File

@ -85,7 +85,7 @@ init_per_suite(Config) ->
} }
] ]
}">>), }">>),
emqx_common_test_helpers:start_apps([emqx_dashboard, ?APP], fun set_special_configs/1), emqx_common_test_helpers:start_apps([emqx_dashboard, emqx_conf, ?APP], fun set_special_configs/1),
Config. Config.
set_special_configs(emqx_dashboard) -> set_special_configs(emqx_dashboard) ->
@ -113,15 +113,17 @@ topic_config(T) ->
end_per_suite(_) -> end_per_suite(_) ->
application:unload(emqx_management), application:unload(emqx_management),
application:unload(emqx_conf),
application:unload(?APP), application:unload(?APP),
meck:unload(emqx_resource), meck:unload(emqx_resource),
meck:unload(emqx_schema), meck:unload(emqx_schema),
emqx_common_test_helpers:stop_apps([emqx_dashboard, ?APP]). emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_conf, ?APP]).
t_auto_subscribe(_) -> t_auto_subscribe(_) ->
emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]),
{ok, Client} = emqtt:start_link(#{username => ?CLIENT_USERNAME, clientid => ?CLIENT_ID}), {ok, Client} = emqtt:start_link(#{username => ?CLIENT_USERNAME, clientid => ?CLIENT_ID}),
{ok, _} = emqtt:connect(Client), {ok, _} = emqtt:connect(Client),
timer:sleep(100), timer:sleep(200),
?assertEqual(check_subs(length(?TOPICS)), ok), ?assertEqual(check_subs(length(?TOPICS)), ok),
emqtt:disconnect(Client), emqtt:disconnect(Client),
ok. ok.
@ -148,6 +150,7 @@ t_update(_) ->
check_subs(Count) -> check_subs(Count) ->
Subs = ets:tab2list(emqx_suboption), Subs = ets:tab2list(emqx_suboption),
ct:pal("---> ~p ~p ~n", [Subs, Count]),
?assert(length(Subs) >= Count), ?assert(length(Subs) >= Count),
check_subs((Subs), ?ENSURE_TOPICS). check_subs((Subs), ?ENSURE_TOPICS).

View File

@ -34,8 +34,8 @@
# direction = egress # direction = egress
# ## NOTE: we cannot use placehodler variables in the `scheme://host:port` part of the url # ## NOTE: we cannot use placehodler variables in the `scheme://host:port` part of the url
# url = "http://localhost:9901/messages/${topic}" # url = "http://localhost:9901/messages/${topic}"
# request_timeout = "30s" # request_timeout = "15s"
# connect_timeout = "30s" # connect_timeout = "15s"
# max_retries = 3 # max_retries = 3
# retry_interval = "10s" # retry_interval = "10s"
# pool_type = "random" # pool_type = "random"

View File

@ -35,15 +35,19 @@
]). ]).
-export([ load/0 -export([ load/0
, lookup/1
, lookup/2 , lookup/2
, lookup/3 , lookup/3
, list/0 , list/0
, list_bridges_by_connector/1 , list_bridges_by_connector/1
, create/2
, create/3 , create/3
, recreate/2 , recreate/2
, recreate/3 , recreate/3
, create_dry_run/2 , create_dry_run/2
, remove/1
, remove/3 , remove/3
, update/2
, update/3 , update/3
, start/2 , start/2
, stop/2 , stop/2
@ -80,17 +84,36 @@ unload_hook() ->
on_message_publish(Message = #message{topic = Topic, flags = Flags}) -> on_message_publish(Message = #message{topic = Topic, flags = Flags}) ->
case maps:get(sys, Flags, false) of case maps:get(sys, Flags, false) of
false -> false ->
lists:foreach(fun (Id) -> Msg = emqx_rule_events:eventmsg_publish(Message),
send_message(Id, emqx_rule_events:eventmsg_publish(Message)) send_to_matched_egress_bridges(Topic, Msg);
end, get_matched_bridges(Topic));
true -> ok true -> ok
end, end,
{ok, Message}. {ok, Message}.
send_to_matched_egress_bridges(Topic, Msg) ->
lists:foreach(fun (Id) ->
try send_message(Id, Msg) of
ok -> ok;
Error -> ?SLOG(error, #{msg => "send_message_to_bridge_failed",
bridge => Id, error => Error})
catch Err:Reason:ST ->
?SLOG(error, #{msg => "send_message_to_bridge_crash",
bridge => Id, error => Err, reason => Reason,
stacktrace => ST})
end
end, get_matched_bridges(Topic)).
send_message(BridgeId, Message) -> send_message(BridgeId, Message) ->
{BridgeType, BridgeName} = parse_bridge_id(BridgeId), {BridgeType, BridgeName} = parse_bridge_id(BridgeId),
ResId = emqx_bridge:resource_id(BridgeType, BridgeName), ResId = emqx_bridge:resource_id(BridgeType, BridgeName),
emqx_resource:query(ResId, {send_message, Message}). case emqx:get_config([bridges, BridgeType, BridgeName], not_found) of
not_found ->
{error, {bridge_not_found, BridgeId}};
#{enable := true} ->
emqx_resource:query(ResId, {send_message, Message});
#{enable := false} ->
{error, {bridge_stopped, BridgeId}}
end.
config_key_path() -> config_key_path() ->
[bridges]. [bridges].
@ -169,6 +192,10 @@ list_bridges_by_connector(ConnectorId) ->
[B || B = #{raw_config := #{<<"connector">> := Id}} <- list(), [B || B = #{raw_config := #{<<"connector">> := Id}} <- list(),
ConnectorId =:= Id]. ConnectorId =:= Id].
lookup(Id) ->
{Type, Name} = parse_bridge_id(Id),
lookup(Type, Name).
lookup(Type, Name) -> lookup(Type, Name) ->
RawConf = emqx:get_raw_config([bridges, Type, Name], #{}), RawConf = emqx:get_raw_config([bridges, Type, Name], #{}),
lookup(Type, Name, RawConf). lookup(Type, Name, RawConf).
@ -188,16 +215,24 @@ stop(Type, Name) ->
restart(Type, Name) -> restart(Type, Name) ->
emqx_resource:restart(resource_id(Type, Name)). emqx_resource:restart(resource_id(Type, Name)).
create(BridgeId, Conf) ->
{BridgeType, BridgeName} = parse_bridge_id(BridgeId),
create(BridgeType, BridgeName, Conf).
create(Type, Name, Conf) -> create(Type, Name, Conf) ->
?SLOG(info, #{msg => "create bridge", type => Type, name => Name, ?SLOG(info, #{msg => "create bridge", type => Type, name => Name,
config => Conf}), config => Conf}),
case emqx_resource:create_local(resource_id(Type, Name), emqx_bridge:resource_type(Type), case emqx_resource:create_local(resource_id(Type, Name), emqx_bridge:resource_type(Type),
parse_confs(Type, Name, Conf), #{force_create => true}) of parse_confs(Type, Name, Conf), #{async_create => true}) of
{ok, already_created} -> maybe_disable_bridge(Type, Name, Conf); {ok, already_created} -> maybe_disable_bridge(Type, Name, Conf);
{ok, _} -> maybe_disable_bridge(Type, Name, Conf); {ok, _} -> maybe_disable_bridge(Type, Name, Conf);
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
end. end.
update(BridgeId, {OldConf, Conf}) ->
{BridgeType, BridgeName} = parse_bridge_id(BridgeId),
update(BridgeType, BridgeName, {OldConf, Conf}).
update(Type, Name, {OldConf, Conf}) -> update(Type, Name, {OldConf, Conf}) ->
%% TODO: sometimes its not necessary to restart the bridge connection. %% TODO: sometimes its not necessary to restart the bridge connection.
%% %%
@ -214,23 +249,27 @@ update(Type, Name, {OldConf, Conf}) ->
case recreate(Type, Name, Conf) of case recreate(Type, Name, Conf) of
{ok, _} -> maybe_disable_bridge(Type, Name, Conf); {ok, _} -> maybe_disable_bridge(Type, Name, Conf);
{error, not_found} -> {error, not_found} ->
?SLOG(warning, #{ msg => "updating a non-exist bridge, create a new one" ?SLOG(warning, #{ msg => "updating_a_non-exist_bridge_need_create_a_new_one"
, type => Type, name => Name, config => Conf}), , type => Type, name => Name, config => Conf}),
create(Type, Name, Conf); create(Type, Name, Conf);
{error, Reason} -> {update_bridge_failed, Reason} {error, Reason} -> {error, {update_bridge_failed, Reason}}
end; end;
true -> true ->
%% we don't need to recreate the bridge if this config change is only to %% we don't need to recreate the bridge if this config change is only to
%% toggole the config 'bridge.{type}.{name}.enable' %% toggole the config 'bridge.{type}.{name}.enable'
ok case maps:get(enable, Conf, true) of
false -> stop(Type, Name);
true -> start(Type, Name)
end
end. end.
recreate(Type, Name) -> recreate(Type, Name) ->
recreate(Type, Name, emqx:get_raw_config([bridges, Type, Name])). recreate(Type, Name, emqx:get_config([bridges, Type, Name])).
recreate(Type, Name, Conf) -> recreate(Type, Name, Conf) ->
emqx_resource:recreate_local(resource_id(Type, Name), emqx_resource:recreate_local(resource_id(Type, Name),
emqx_bridge:resource_type(Type), parse_confs(Type, Name, Conf), []). emqx_bridge:resource_type(Type), parse_confs(Type, Name, Conf),
#{async_create => true}).
create_dry_run(Type, Conf) -> create_dry_run(Type, Conf) ->
Conf0 = Conf#{<<"ingress">> => #{<<"remote_topic">> => <<"t">>}}, Conf0 = Conf#{<<"ingress">> => #{<<"remote_topic">> => <<"t">>}},
@ -241,8 +280,12 @@ create_dry_run(Type, Conf) ->
Error Error
end. end.
remove(BridgeId) ->
{BridgeType, BridgeName} = parse_bridge_id(BridgeId),
remove(BridgeType, BridgeName, #{}).
remove(Type, Name, _Conf) -> remove(Type, Name, _Conf) ->
?SLOG(info, #{msg => "remove bridge", type => Type, name => Name}), ?SLOG(info, #{msg => "remove_bridge", type => Type, name => Name}),
case emqx_resource:remove_local(resource_id(Type, Name)) of case emqx_resource:remove_local(resource_id(Type, Name)) of
ok -> ok; ok -> ok;
{error, not_found} -> ok; {error, not_found} -> ok;
@ -276,6 +319,8 @@ get_matched_bridges(Topic) ->
end, Acc0, Conf) end, Acc0, Conf)
end, [], Bridges). end, [], Bridges).
get_matched_bridge_id(#{enable := false}, _Topic, _BType, _BName, Acc) ->
Acc;
get_matched_bridge_id(#{local_topic := Filter}, Topic, BType, BName, Acc) -> get_matched_bridge_id(#{local_topic := Filter}, Topic, BType, BName, Acc) ->
case emqx_topic:match(Topic, Filter) of case emqx_topic:match(Topic, Filter) of
true -> [bridge_id(BType, BName) | Acc]; true -> [bridge_id(BType, BName) | Acc];
@ -306,21 +351,21 @@ parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf)
{Type, ConnName} -> {Type, ConnName} ->
ConnectorConfs = emqx:get_config([connectors, Type, ConnName]), ConnectorConfs = emqx:get_config([connectors, Type, ConnName]),
make_resource_confs(Direction, ConnectorConfs, make_resource_confs(Direction, ConnectorConfs,
maps:without([connector, direction], Conf), Name); maps:without([connector, direction], Conf), Type, Name);
{_ConnType, _ConnName} -> {_ConnType, _ConnName} ->
error({cannot_use_connector_with_different_type, ConnId}) error({cannot_use_connector_with_different_type, ConnId})
end; end;
parse_confs(_Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf) parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf)
when is_map(ConnectorConfs) -> when is_map(ConnectorConfs) ->
make_resource_confs(Direction, ConnectorConfs, make_resource_confs(Direction, ConnectorConfs,
maps:without([connector, direction], Conf), Name). maps:without([connector, direction], Conf), Type, Name).
make_resource_confs(ingress, ConnectorConfs, BridgeConf, Name) -> make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) ->
BName = bin(Name), BName = bridge_id(Type, Name),
ConnectorConfs#{ ConnectorConfs#{
ingress => BridgeConf#{hookpoint => <<"$bridges/", BName/binary>>} ingress => BridgeConf#{hookpoint => <<"$bridges/", BName/binary>>}
}; };
make_resource_confs(egress, ConnectorConfs, BridgeConf, _Name) -> make_resource_confs(egress, ConnectorConfs, BridgeConf, _Type, _Name) ->
ConnectorConfs#{ ConnectorConfs#{
egress => BridgeConf egress => BridgeConf
}. }.

View File

@ -158,8 +158,8 @@ method_example(_Type, _Direction, put) ->
info_example_basic(http, _) -> info_example_basic(http, _) ->
#{ #{
url => <<"http://localhost:9901/messages/${topic}">>, url => <<"http://localhost:9901/messages/${topic}">>,
request_timeout => <<"30s">>, request_timeout => <<"15s">>,
connect_timeout => <<"30s">>, connect_timeout => <<"15s">>,
max_retries => 3, max_retries => 3,
retry_interval => <<"10s">>, retry_interval => <<"10s">>,
pool_type => <<"random">>, pool_type => <<"random">>,
@ -276,7 +276,7 @@ schema("/bridges/:id/operation/:operation") ->
'/bridges'(post, #{body := #{<<"type">> := BridgeType} = Conf0}) -> '/bridges'(post, #{body := #{<<"type">> := BridgeType} = Conf0}) ->
Conf = filter_out_request_body(Conf0), Conf = filter_out_request_body(Conf0),
BridgeName = maps:get(<<"name">>, Conf, emqx_misc:gen_id()), BridgeName = emqx_misc:gen_id(),
case emqx_bridge:lookup(BridgeType, BridgeName) of case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, _} -> {ok, _} ->
{400, error_msg('ALREADY_EXISTS', <<"bridge already exists">>)}; {400, error_msg('ALREADY_EXISTS', <<"bridge already exists">>)};
@ -356,9 +356,8 @@ operation_to_conf_req(<<"restart">>) -> restart;
operation_to_conf_req(_) -> invalid. operation_to_conf_req(_) -> invalid.
ensure_bridge_created(BridgeType, BridgeName, Conf) -> ensure_bridge_created(BridgeType, BridgeName, Conf) ->
Conf1 = maps:without([<<"type">>, <<"name">>], Conf),
case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName],
Conf1, #{override_to => cluster}) of Conf, #{override_to => cluster}) of
{ok, _} -> ok; {ok, _} -> ok;
{error, Reason} -> {error, Reason} ->
{error, error_msg('BAD_ARG', Reason)} {error, error_msg('BAD_ARG', Reason)}
@ -411,12 +410,12 @@ aggregate_metrics(AllMetrics) ->
format_resp(#{id := Id, raw_config := RawConf, format_resp(#{id := Id, raw_config := RawConf,
resource_data := #{status := Status, metrics := Metrics}}) -> resource_data := #{status := Status, metrics := Metrics}}) ->
{Type, Name} = emqx_bridge:parse_bridge_id(Id), {Type, BridgeName} = emqx_bridge:parse_bridge_id(Id),
IsConnected = fun(started) -> connected; (_) -> disconnected end, IsConnected = fun(started) -> connected; (_) -> disconnected end,
RawConf#{ RawConf#{
id => Id, id => Id,
type => Type, type => Type,
name => Name, name => maps:get(<<"name">>, RawConf, BridgeName),
node => node(), node => node(),
status => IsConnected(Status), status => IsConnected(Status),
metrics => Metrics metrics => Metrics
@ -431,8 +430,8 @@ rpc_multicall(Func, Args) ->
end. end.
filter_out_request_body(Conf) -> filter_out_request_body(Conf) ->
ExtraConfs = [<<"id">>, <<"status">>, <<"node_status">>, <<"node_metrics">>, ExtraConfs = [<<"id">>, <<"type">>, <<"status">>, <<"node_status">>,
<<"metrics">>, <<"node">>], <<"node_metrics">>, <<"metrics">>, <<"node">>],
maps:without(ExtraConfs, Conf). maps:without(ExtraConfs, Conf).
rpc_call(Node, Fun, Args) -> rpc_call(Node, Fun, Args) ->

View File

@ -59,7 +59,7 @@ Template with variables is allowed.
""" """
})} })}
, {request_timeout, mk(emqx_schema:duration_ms(), , {request_timeout, mk(emqx_schema:duration_ms(),
#{ default => <<"30s">> #{ default => <<"15s">>
, desc =>""" , desc =>"""
How long will the HTTP request timeout. How long will the HTTP request timeout.
""" """
@ -68,7 +68,6 @@ How long will the HTTP request timeout.
fields("post") -> fields("post") ->
[ type_field() [ type_field()
, name_field()
] ++ fields("bridge"); ] ++ fields("bridge");
fields("put") -> fields("put") ->
@ -84,9 +83,14 @@ basic_config() ->
#{ desc => "Enable or disable this bridge" #{ desc => "Enable or disable this bridge"
, default => true , default => true
})} })}
, {name,
mk(binary(),
#{ desc => "Bridge name, used as a human-readable description of the bridge."
})}
, {direction, , {direction,
mk(egress, mk(egress,
#{ desc => "The direction of this bridge, MUST be egress" #{ desc => "The direction of this bridge, MUST be egress"
, default => egress
})} })}
] ]
++ proplists:delete(base_url, emqx_connector_http:fields(config)). ++ proplists:delete(base_url, emqx_connector_http:fields(config)).
@ -98,8 +102,5 @@ id_field() ->
type_field() -> type_field() ->
{type, mk(http, #{desc => "The Bridge Type"})}. {type, mk(http, #{desc => "The Bridge Type"})}.
name_field() ->
{name, mk(binary(), #{desc => "The Bridge Name"})}.
method() -> method() ->
enum([post, put, get, delete]). enum([post, put, get, delete]).

View File

@ -24,11 +24,9 @@ fields("egress") ->
fields("post_ingress") -> fields("post_ingress") ->
[ type_field() [ type_field()
, name_field()
] ++ proplists:delete(enable, fields("ingress")); ] ++ proplists:delete(enable, fields("ingress"));
fields("post_egress") -> fields("post_egress") ->
[ type_field() [ type_field()
, name_field()
] ++ proplists:delete(enable, fields("egress")); ] ++ proplists:delete(enable, fields("egress"));
fields("put_ingress") -> fields("put_ingress") ->
@ -49,9 +47,3 @@ id_field() ->
type_field() -> type_field() ->
{type, mk(mqtt, #{desc => "The Bridge Type"})}. {type, mk(mqtt, #{desc => "The Bridge Type"})}.
name_field() ->
{name, mk(binary(),
#{ desc => "The Bridge Name"
, example => "some_bridge_name"
})}.

View File

@ -46,6 +46,10 @@ common_bridge_fields() ->
#{ desc => "Enable or disable this bridge" #{ desc => "Enable or disable this bridge"
, default => true , default => true
})} })}
, {name,
mk(binary(),
#{ desc => "Bridge name, used as a human-readable description of the bridge."
})}
, {connector, , {connector,
mk(binary(), mk(binary(),
#{ nullable => false #{ nullable => false
@ -71,6 +75,7 @@ metrics_status_fields() ->
direction_field(Dir, Desc) -> direction_field(Dir, Desc) ->
{direction, mk(Dir, {direction, mk(Dir,
#{ nullable => false #{ nullable => false
, default => egress
, desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.<br>" , desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.<br>"
++ Desc ++ Desc
})}. })}.

View File

@ -23,12 +23,13 @@
-define(CONF_DEFAULT, <<"bridges: {}">>). -define(CONF_DEFAULT, <<"bridges: {}">>).
-define(BRIDGE_TYPE, <<"http">>). -define(BRIDGE_TYPE, <<"http">>).
-define(BRIDGE_NAME, <<"test_bridge">>). -define(BRIDGE_NAME, <<"test_bridge">>).
-define(BRIDGE_ID, <<"http:test_bridge">>).
-define(URL(PORT, PATH), list_to_binary( -define(URL(PORT, PATH), list_to_binary(
io_lib:format("http://localhost:~s/~s", io_lib:format("http://localhost:~s/~s",
[integer_to_list(PORT), PATH]))). [integer_to_list(PORT), PATH]))).
-define(HTTP_BRIDGE(URL), -define(HTTP_BRIDGE(URL, TYPE, NAME),
#{ #{
<<"type">> => TYPE,
<<"name">> => NAME,
<<"url">> => URL, <<"url">> => URL,
<<"local_topic">> => <<"emqx_http/#">>, <<"local_topic">> => <<"emqx_http/#">>,
<<"method">> => <<"post">>, <<"method">> => <<"post">>,
@ -47,7 +48,7 @@ groups() ->
[]. [].
suite() -> suite() ->
[{timetrap,{seconds,30}}]. [{timetrap,{seconds,60}}].
init_per_suite(Config) -> init_per_suite(Config) ->
ok = emqx_config:put([emqx_dashboard], #{ ok = emqx_config:put([emqx_dashboard], #{
@ -84,7 +85,7 @@ start_http_server(HandleFun) ->
spawn_link(fun() -> spawn_link(fun() ->
{Port, Sock} = listen_on_random_port(), {Port, Sock} = listen_on_random_port(),
Parent ! {port, Port}, Parent ! {port, Port},
loop(Sock, HandleFun) loop(Sock, HandleFun, Parent)
end), end),
receive receive
{port, Port} -> Port {port, Port} -> Port
@ -95,40 +96,49 @@ start_http_server(HandleFun) ->
listen_on_random_port() -> listen_on_random_port() ->
Min = 1024, Max = 65000, Min = 1024, Max = 65000,
Port = rand:uniform(Max - Min) + Min, Port = rand:uniform(Max - Min) + Min,
case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}]) of case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}, binary]) of
{ok, Sock} -> {Port, Sock}; {ok, Sock} -> {Port, Sock};
{error, eaddrinuse} -> listen_on_random_port() {error, eaddrinuse} -> listen_on_random_port()
end. end.
loop(Sock, HandleFun) -> loop(Sock, HandleFun, Parent) ->
{ok, Conn} = gen_tcp:accept(Sock), {ok, Conn} = gen_tcp:accept(Sock),
Handler = spawn(fun () -> HandleFun(Conn) end), Handler = spawn(fun () -> HandleFun(Conn, Parent) end),
gen_tcp:controlling_process(Conn, Handler), gen_tcp:controlling_process(Conn, Handler),
loop(Sock, HandleFun). loop(Sock, HandleFun, Parent).
make_response(CodeStr, Str) -> make_response(CodeStr, Str) ->
B = iolist_to_binary(Str), B = iolist_to_binary(Str),
iolist_to_binary( iolist_to_binary(
io_lib:fwrite( io_lib:fwrite(
"HTTP/1.0 ~s\nContent-Type: text/html\nContent-Length: ~p\n\n~s", "HTTP/1.0 ~s\r\nContent-Type: text/html\r\nContent-Length: ~p\r\n\r\n~s",
[CodeStr, size(B), B])). [CodeStr, size(B), B])).
handle_fun_200_ok(Conn) -> handle_fun_200_ok(Conn, Parent) ->
case gen_tcp:recv(Conn, 0) of case gen_tcp:recv(Conn, 0) of
{ok, Request} -> {ok, ReqStr} ->
ct:pal("the http handler got request: ~p", [ReqStr]),
Req = parse_http_request(ReqStr),
Parent ! {http_server, received, Req},
gen_tcp:send(Conn, make_response("200 OK", "Request OK")), gen_tcp:send(Conn, make_response("200 OK", "Request OK")),
self() ! {http_server, received, Request}, handle_fun_200_ok(Conn, Parent);
handle_fun_200_ok(Conn);
{error, closed} -> {error, closed} ->
gen_tcp:close(Conn) gen_tcp:close(Conn)
end. end.
parse_http_request(ReqStr0) ->
[Method, ReqStr1] = string:split(ReqStr0, " ", leading),
[Path, ReqStr2] = string:split(ReqStr1, " ", leading),
[_ProtoVsn, ReqStr3] = string:split(ReqStr2, "\r\n", leading),
[_HeaderStr, Body] = string:split(ReqStr3, "\r\n\r\n", leading),
#{method => Method, path => Path, body => Body}.
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Testcases %% Testcases
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
t_http_crud_apis(_) -> t_http_crud_apis(_) ->
Port = start_http_server(fun handle_fun_200_ok/1), Port = start_http_server(fun handle_fun_200_ok/2),
%% assert we there's no bridges at first %% assert we there's no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
@ -136,13 +146,10 @@ t_http_crud_apis(_) ->
%% POST /bridges/ will create a bridge %% POST /bridges/ will create a bridge
URL1 = ?URL(Port, "path1"), URL1 = ?URL(Port, "path1"),
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(post, uri(["bridges"]),
?HTTP_BRIDGE(URL1)#{ ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
<<"type">> => ?BRIDGE_TYPE,
<<"name">> => ?BRIDGE_NAME
}),
%ct:pal("---bridge: ~p", [Bridge]), %ct:pal("---bridge: ~p", [Bridge]),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID #{ <<"id">> := BridgeID
, <<"type">> := ?BRIDGE_TYPE , <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME , <<"name">> := ?BRIDGE_NAME
, <<"status">> := _ , <<"status">> := _
@ -150,24 +157,28 @@ t_http_crud_apis(_) ->
, <<"metrics">> := _ , <<"metrics">> := _
, <<"node_metrics">> := [_|_] , <<"node_metrics">> := [_|_]
, <<"url">> := URL1 , <<"url">> := URL1
}, jsx:decode(Bridge)), } = jsx:decode(Bridge),
%% create a again returns an error
{ok, 400, RetMsg} = request(post, uri(["bridges"]),
?HTTP_BRIDGE(URL1)#{
<<"type">> => ?BRIDGE_TYPE,
<<"name">> => ?BRIDGE_NAME
}),
?assertMatch(
#{ <<"code">> := _
, <<"message">> := <<"bridge already exists">>
}, jsx:decode(RetMsg)),
%% send an message to emqx and the message should be forwarded to the HTTP server
wait_for_resource_ready(BridgeID, 5),
Body = <<"my msg">>,
emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)),
?assert(
receive
{http_server, received, #{method := <<"POST">>, path := <<"/path1">>,
body := Body}} ->
true;
Msg ->
ct:pal("error: http got unexpected request: ~p", [Msg]),
false
after 100 ->
false
end),
%% update the request-path of the bridge %% update the request-path of the bridge
URL2 = ?URL(Port, "path2"), URL2 = ?URL(Port, "path2"),
{ok, 200, Bridge2} = request(put, uri(["bridges", ?BRIDGE_ID]), {ok, 200, Bridge2} = request(put, uri(["bridges", BridgeID]),
?HTTP_BRIDGE(URL2)), ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID ?assertMatch(#{ <<"id">> := BridgeID
, <<"type">> := ?BRIDGE_TYPE , <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME , <<"name">> := ?BRIDGE_NAME
, <<"status">> := _ , <<"status">> := _
@ -179,7 +190,7 @@ t_http_crud_apis(_) ->
%% list all bridges again, assert Bridge2 is in it %% list all bridges again, assert Bridge2 is in it
{ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []), {ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []),
?assertMatch([#{ <<"id">> := ?BRIDGE_ID ?assertMatch([#{ <<"id">> := BridgeID
, <<"type">> := ?BRIDGE_TYPE , <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME , <<"name">> := ?BRIDGE_NAME
, <<"status">> := _ , <<"status">> := _
@ -190,8 +201,8 @@ t_http_crud_apis(_) ->
}], jsx:decode(Bridge2Str)), }], jsx:decode(Bridge2Str)),
%% get the bridge by id %% get the bridge by id
{ok, 200, Bridge3Str} = request(get, uri(["bridges", ?BRIDGE_ID]), []), {ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID ?assertMatch(#{ <<"id">> := BridgeID
, <<"type">> := ?BRIDGE_TYPE , <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME , <<"name">> := ?BRIDGE_NAME
, <<"status">> := _ , <<"status">> := _
@ -201,13 +212,27 @@ t_http_crud_apis(_) ->
, <<"url">> := URL2 , <<"url">> := URL2
}, jsx:decode(Bridge3Str)), }, jsx:decode(Bridge3Str)),
%% send an message to emqx again, check the path has been changed
wait_for_resource_ready(BridgeID, 5),
emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)),
?assert(
receive
{http_server, received, #{path := <<"/path2">>}} ->
true;
Msg2 ->
ct:pal("error: http got unexpected request: ~p", [Msg2]),
false
after 100 ->
false
end),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% update a deleted bridge returns an error %% update a deleted bridge returns an error
{ok, 404, ErrMsg2} = request(put, uri(["bridges", ?BRIDGE_ID]), {ok, 404, ErrMsg2} = request(put, uri(["bridges", BridgeID]),
?HTTP_BRIDGE(URL2)), ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
?assertMatch( ?assertMatch(
#{ <<"code">> := _ #{ <<"code">> := _
, <<"message">> := <<"bridge not found">> , <<"message">> := <<"bridge not found">>
@ -215,16 +240,15 @@ t_http_crud_apis(_) ->
ok. ok.
t_start_stop_bridges(_) -> t_start_stop_bridges(_) ->
Port = start_http_server(fun handle_fun_200_ok/1), %% assert we there's no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
Port = start_http_server(fun handle_fun_200_ok/2),
URL1 = ?URL(Port, "abc"), URL1 = ?URL(Port, "abc"),
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(post, uri(["bridges"]),
?HTTP_BRIDGE(URL1)#{ ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
<<"type">> => ?BRIDGE_TYPE,
<<"name">> => ?BRIDGE_NAME
}),
%ct:pal("the bridge ==== ~p", [Bridge]), %ct:pal("the bridge ==== ~p", [Bridge]),
?assertMatch( #{ <<"id">> := BridgeID
#{ <<"id">> := ?BRIDGE_ID
, <<"type">> := ?BRIDGE_TYPE , <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME , <<"name">> := ?BRIDGE_NAME
, <<"status">> := _ , <<"status">> := _
@ -232,35 +256,35 @@ t_start_stop_bridges(_) ->
, <<"metrics">> := _ , <<"metrics">> := _
, <<"node_metrics">> := [_|_] , <<"node_metrics">> := [_|_]
, <<"url">> := URL1 , <<"url">> := URL1
}, jsx:decode(Bridge)), } = jsx:decode(Bridge),
%% stop it %% stop it
{ok, 200, <<>>} = request(post, operation_path(stop), <<"">>), {ok, 200, <<>>} = request(post, operation_path(stop, BridgeID), <<"">>),
{ok, 200, Bridge2} = request(get, uri(["bridges", ?BRIDGE_ID]), []), {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID ?assertMatch(#{ <<"id">> := BridgeID
, <<"status">> := <<"disconnected">> , <<"status">> := <<"disconnected">>
}, jsx:decode(Bridge2)), }, jsx:decode(Bridge2)),
%% start again %% start again
{ok, 200, <<>>} = request(post, operation_path(start), <<"">>), {ok, 200, <<>>} = request(post, operation_path(start, BridgeID), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []), {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID ?assertMatch(#{ <<"id">> := BridgeID
, <<"status">> := <<"connected">> , <<"status">> := <<"connected">>
}, jsx:decode(Bridge3)), }, jsx:decode(Bridge3)),
%% restart an already started bridge %% restart an already started bridge
{ok, 200, <<>>} = request(post, operation_path(restart), <<"">>), {ok, 200, <<>>} = request(post, operation_path(restart, BridgeID), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []), {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID ?assertMatch(#{ <<"id">> := BridgeID
, <<"status">> := <<"connected">> , <<"status">> := <<"connected">>
}, jsx:decode(Bridge3)), }, jsx:decode(Bridge3)),
%% stop it again %% stop it again
{ok, 200, <<>>} = request(post, operation_path(stop), <<"">>), {ok, 200, <<>>} = request(post, operation_path(stop, BridgeID), <<"">>),
%% restart a stopped bridge %% restart a stopped bridge
{ok, 200, <<>>} = request(post, operation_path(restart), <<"">>), {ok, 200, <<>>} = request(post, operation_path(restart, BridgeID), <<"">>),
{ok, 200, Bridge4} = request(get, uri(["bridges", ?BRIDGE_ID]), []), {ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID ?assertMatch(#{ <<"id">> := BridgeID
, <<"status">> := <<"connected">> , <<"status">> := <<"connected">>
}, jsx:decode(Bridge4)), }, jsx:decode(Bridge4)),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []).
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -296,5 +320,16 @@ auth_header_() ->
{ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
{"Authorization", "Bearer " ++ binary_to_list(Token)}. {"Authorization", "Bearer " ++ binary_to_list(Token)}.
operation_path(Oper) -> operation_path(Oper, BridgeID) ->
uri(["bridges", ?BRIDGE_ID, "operation", Oper]). uri(["bridges", BridgeID, "operation", Oper]).
wait_for_resource_ready(InstId, 0) ->
ct:pal("--- bridge ~p: ~p", [InstId, emqx_bridge:lookup(InstId)]),
ct:fail(wait_resource_timeout);
wait_for_resource_ready(InstId, Retry) ->
case emqx_bridge:lookup(InstId) of
{ok, #{resource_data := #{status := started}}} -> ok;
_ ->
timer:sleep(100),
wait_for_resource_ready(InstId, Retry-1)
end.

View File

@ -236,7 +236,7 @@ catch_up(#{node := Node, retry_interval := RetryMs} = State, SkipResult) ->
false -> RetryMs false -> RetryMs
end; end;
{aborted, Reason} -> {aborted, Reason} ->
?SLOG(error, #{msg => "read_next_mfa transaction failed", error => Reason}), ?SLOG(error, #{msg => "read_next_mfa_transaction_failed", error => Reason}),
RetryMs RetryMs
end. end.
@ -248,7 +248,7 @@ read_next_mfa(Node) ->
TnxId = max(LatestId - 1, 0), TnxId = max(LatestId - 1, 0),
commit(Node, TnxId), commit(Node, TnxId),
?SLOG(notice, #{ ?SLOG(notice, #{
msg => "New node first catch up and start commit.", msg => "new_node_first_catch_up_and_start_commit.",
node => Node, tnx_id => TnxId}), node => Node, tnx_id => TnxId}),
TnxId; TnxId;
[#cluster_rpc_commit{tnx_id = LastAppliedID}] -> LastAppliedID + 1 [#cluster_rpc_commit{tnx_id = LastAppliedID}] -> LastAppliedID + 1
@ -277,7 +277,7 @@ do_catch_up(ToTnxId, Node) ->
io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)", io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)",
[Node, LastAppliedId, ToTnxId])), [Node, LastAppliedId, ToTnxId])),
?SLOG(error, #{ ?SLOG(error, #{
msg => "catch up failed!", msg => "catch_up_failed!",
last_applied_id => LastAppliedId, last_applied_id => LastAppliedId,
to_tnx_id => ToTnxId to_tnx_id => ToTnxId
}), }),

View File

@ -144,7 +144,7 @@ multicall(M, F, Args) ->
{retry, TnxId, Res, Nodes} -> {retry, TnxId, Res, Nodes} ->
%% The init MFA return ok, but other nodes failed. %% The init MFA return ok, but other nodes failed.
%% We return ok and alert an alarm. %% We return ok and alert an alarm.
?SLOG(error, #{msg => "failed to update config in cluster", nodes => Nodes, ?SLOG(error, #{msg => "failed_to_update_config_in_cluster", nodes => Nodes,
tnx_id => TnxId, mfa => {M, F, Args}}), tnx_id => TnxId, mfa => {M, F, Args}}),
Res; Res;
{error, Error} -> %% all MFA return not ok or {ok, term()}. {error, Error} -> %% all MFA return not ok or {ok, term()}.

View File

@ -730,16 +730,7 @@ do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth) ->
}}; }};
do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth) -> do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth) ->
{emqx_logger_textfmt, {emqx_logger_textfmt,
#{template => #{template => [time," [",level,"] ", msg,"\n"],
[time," [",level,"] ",
{clientid,
[{peername,
[clientid,"@",peername," "],
[clientid, " "]}],
[{peername,
[peername," "],
[]}]},
msg,"\n"],
chars_limit => CharsLimit, chars_limit => CharsLimit,
single_line => SingleLine, single_line => SingleLine,
time_offset => TimeOffSet, time_offset => TimeOffSet,

View File

@ -74,9 +74,19 @@ t_base_test(_Config) ->
?assertEqual(node(), maps:get(initiator, Query)), ?assertEqual(node(), maps:get(initiator, Query)),
?assert(maps:is_key(created_at, Query)), ?assert(maps:is_key(created_at, Query)),
?assertEqual(ok, receive_msg(3, test)), ?assertEqual(ok, receive_msg(3, test)),
?assertEqual({ok, 2, ok}, emqx_cluster_rpc:multicall(M, F, A)),
{atomic, Status} = emqx_cluster_rpc:status(), {atomic, Status} = emqx_cluster_rpc:status(),
?assertEqual(3, length(Status)), case length(Status) =:= 3 of
?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 1 end, Status)), true -> ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status));
false ->
%% wait for mnesia to write in.
ct:sleep(42),
{atomic, Status1} = emqx_cluster_rpc:status(),
ct:pal("status: ~p", Status),
ct:pal("status1: ~p", Status1),
?assertEqual(3, length(Status1)),
?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status))
end,
ok. ok.
t_commit_fail_test(_Config) -> t_commit_fail_test(_Config) ->

View File

@ -7,7 +7,7 @@
{emqx, {path, "../emqx"}}, {emqx, {path, "../emqx"}},
{eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
{mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.6.0"}}}, {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.1"}}},
%% NOTE: mind poolboy version when updating mongodb-erlang version %% NOTE: mind poolboy version when updating mongodb-erlang version
{mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.11"}}}, {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.11"}}},
%% NOTE: mind poolboy version when updating eredis_cluster version %% NOTE: mind poolboy version when updating eredis_cluster version

View File

@ -37,31 +37,26 @@
config_key_path() -> config_key_path() ->
[connectors]. [connectors].
-dialyzer([{nowarn_function, [post_config_update/5]}, error_handling]).
post_config_update([connectors, Type, Name], '$remove', _, _OldConf, _AppEnvs) -> post_config_update([connectors, Type, Name], '$remove', _, _OldConf, _AppEnvs) ->
ConnId = connector_id(Type, Name), ConnId = connector_id(Type, Name),
LinkedBridgeIds = lists:foldl(fun try foreach_linked_bridges(ConnId, fun(#{id := BId}) ->
(#{id := BId, raw_config := #{<<"connector">> := ConnId0}}, Acc) throw({dependency_bridges_exist, BId})
when ConnId0 == ConnId -> end)
[BId | Acc]; catch throw:Error -> {error, Error}
(_, Acc) -> Acc
end, [], emqx_bridge:list()),
case LinkedBridgeIds of
[] -> ok;
_ -> {error, {dependency_bridges_exist, LinkedBridgeIds}}
end; end;
post_config_update([connectors, Type, Name], _Req, NewConf, _OldConf, _AppEnvs) -> post_config_update([connectors, Type, Name], _Req, NewConf, OldConf, _AppEnvs) ->
ConnId = connector_id(Type, Name), ConnId = connector_id(Type, Name),
lists:foreach(fun foreach_linked_bridges(ConnId,
(#{id := BId, raw_config := #{<<"connector">> := ConnId0}}) when ConnId0 == ConnId -> fun(#{id := BId}) ->
{BType, BName} = emqx_bridge:parse_bridge_id(BId), {BType, BName} = emqx_bridge:parse_bridge_id(BId),
BridgeConf = emqx:get_config([bridges, BType, BName]), BridgeConf = emqx:get_config([bridges, BType, BName]),
case emqx_bridge:recreate(BType, BName, BridgeConf#{connector => NewConf}) of case emqx_bridge:update(BType, BName, {BridgeConf#{connector => OldConf},
{ok, _} -> ok; BridgeConf#{connector => NewConf}}) of
ok -> ok;
{error, Reason} -> error({update_bridge_error, Reason}) {error, Reason} -> error({update_bridge_error, Reason})
end; end
(_) -> end).
ok
end, emqx_bridge:list()).
connector_id(Type0, Name0) -> connector_id(Type0, Name0) ->
Type = bin(Type0), Type = bin(Type0),
@ -112,3 +107,10 @@ delete(Type, Name) ->
bin(Bin) when is_binary(Bin) -> Bin; bin(Bin) when is_binary(Bin) -> Bin;
bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Str) when is_list(Str) -> list_to_binary(Str);
bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
foreach_linked_bridges(ConnId, Do) ->
lists:foreach(fun
(#{raw_config := #{<<"connector">> := ConnId0}} = Bridge) when ConnId0 == ConnId ->
Do(Bridge);
(_) -> ok
end, emqx_bridge:list()).

View File

@ -107,14 +107,14 @@ info_example_basic(mqtt) ->
#{ #{
mode => cluster_shareload, mode => cluster_shareload,
server => <<"127.0.0.1:1883">>, server => <<"127.0.0.1:1883">>,
reconnect_interval => <<"30s">>, reconnect_interval => <<"15s">>,
proto_ver => <<"v4">>, proto_ver => <<"v4">>,
username => <<"foo">>, username => <<"foo">>,
password => <<"bar">>, password => <<"bar">>,
clientid => <<"foo">>, clientid => <<"foo">>,
clean_start => true, clean_start => true,
keepalive => <<"300s">>, keepalive => <<"300s">>,
retry_interval => <<"30s">>, retry_interval => <<"15s">>,
max_inflight => 100, max_inflight => 100,
ssl => #{ ssl => #{
enable => false enable => false
@ -155,8 +155,7 @@ schema("/connectors") ->
}, },
post => #{ post => #{
tags => [<<"connectors">>], tags => [<<"connectors">>],
description => <<"Create a new connector by given Id <br>" description => <<"Create a new connector">>,
"The ID must be of format '{type}:{name}'">>,
summary => <<"Create connector">>, summary => <<"Create connector">>,
requestBody => post_request_body_schema(), requestBody => post_request_body_schema(),
responses => #{ responses => #{
@ -212,13 +211,13 @@ schema("/connectors/:id") ->
{200, [format_resp(Conn) || Conn <- emqx_connector:list()]}; {200, [format_resp(Conn) || Conn <- emqx_connector:list()]};
'/connectors'(post, #{body := #{<<"type">> := ConnType} = Params}) -> '/connectors'(post, #{body := #{<<"type">> := ConnType} = Params}) ->
ConnName = maps:get(<<"name">>, Params, emqx_misc:gen_id()), ConnName = emqx_misc:gen_id(),
case emqx_connector:lookup(ConnType, ConnName) of case emqx_connector:lookup(ConnType, ConnName) of
{ok, _} -> {ok, _} ->
{400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)}; {400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)};
{error, not_found} -> {error, not_found} ->
case emqx_connector:update(ConnType, ConnName, case emqx_connector:update(ConnType, ConnName,
maps:without([<<"type">>, <<"name">>], Params)) of filter_out_request_body(Params)) of
{ok, #{raw_config := RawConf}} -> {ok, #{raw_config := RawConf}} ->
Id = emqx_connector:connector_id(ConnType, ConnName), Id = emqx_connector:connector_id(ConnType, ConnName),
{201, format_resp(Id, RawConf)}; {201, format_resp(Id, RawConf)};
@ -254,6 +253,10 @@ schema("/connectors/:id") ->
{ok, _} -> {ok, _} ->
case emqx_connector:delete(ConnType, ConnName) of case emqx_connector:delete(ConnType, ConnName) of
{ok, _} -> {204}; {ok, _} -> {204};
{error, {post_config_update, _, {dependency_bridges_exist, BridgeID}}} ->
{403, error_msg('DEPENDENCY_EXISTS',
<<"Cannot remove the connector as it's in use by a bridge: ",
BridgeID/binary>>)};
{error, Error} -> {400, error_msg('BAD_ARG', Error)} {error, Error} -> {400, error_msg('BAD_ARG', Error)}
end; end;
{error, not_found} -> {error, not_found} ->
@ -270,16 +273,16 @@ format_resp(#{<<"id">> := Id} = RawConf) ->
format_resp(ConnId, RawConf) -> format_resp(ConnId, RawConf) ->
NumOfBridges = length(emqx_bridge:list_bridges_by_connector(ConnId)), NumOfBridges = length(emqx_bridge:list_bridges_by_connector(ConnId)),
{Type, Name} = emqx_connector:parse_connector_id(ConnId), {Type, ConnName} = emqx_connector:parse_connector_id(ConnId),
RawConf#{ RawConf#{
<<"id">> => ConnId, <<"id">> => ConnId,
<<"type">> => Type, <<"type">> => Type,
<<"name">> => Name, <<"name">> => maps:get(<<"name">>, RawConf, ConnName),
<<"num_of_bridges">> => NumOfBridges <<"num_of_bridges">> => NumOfBridges
}. }.
filter_out_request_body(Conf) -> filter_out_request_body(Conf) ->
ExtraConfs = [<<"num_of_bridges">>, <<"type">>, <<"name">>], ExtraConfs = [<<"clientid">>, <<"num_of_bridges">>, <<"type">>],
maps:without(ExtraConfs, Conf). maps:without(ExtraConfs, Conf).
bin(S) when is_list(S) -> bin(S) when is_list(S) ->

View File

@ -75,7 +75,7 @@ For example: http://localhost:9901/
})} })}
, {connect_timeout, , {connect_timeout,
sc(emqx_schema:duration_ms(), sc(emqx_schema:duration_ms(),
#{ default => "30s" #{ default => "15s"
, desc => "The timeout when connecting to the HTTP server" , desc => "The timeout when connecting to the HTTP server"
})} })}
, {max_retries, , {max_retries,
@ -143,7 +143,7 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
retry_interval := RetryInterval, retry_interval := RetryInterval,
pool_type := PoolType, pool_type := PoolType,
pool_size := PoolSize} = Config) -> pool_size := PoolSize} = Config) ->
?SLOG(info, #{msg => "starting http connector", ?SLOG(info, #{msg => "starting_http_connector",
connector => InstId, config => Config}), connector => InstId, config => Config}),
{Transport, TransportOpts} = case Scheme of {Transport, TransportOpts} = case Scheme of
http -> http ->
@ -181,13 +181,13 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
end. end.
on_stop(InstId, #{pool_name := PoolName}) -> on_stop(InstId, #{pool_name := PoolName}) ->
?SLOG(info, #{msg => "stopping http connector", ?SLOG(info, #{msg => "stopping_http_connector",
connector => InstId}), connector => InstId}),
ehttpc_sup:stop_pool(PoolName). ehttpc_sup:stop_pool(PoolName).
on_query(InstId, {send_message, Msg}, AfterQuery, State) -> on_query(InstId, {send_message, Msg}, AfterQuery, State) ->
case maps:get(request, State, undefined) of case maps:get(request, State, undefined) of
undefined -> ?SLOG(error, #{msg => "request not found", connector => InstId}); undefined -> ?SLOG(error, #{msg => "request_not_found", connector => InstId});
Request -> Request ->
#{method := Method, path := Path, body := Body, headers := Headers, #{method := Method, path := Path, body := Body, headers := Headers,
request_timeout := Timeout} = process_request(Request, Msg), request_timeout := Timeout} = process_request(Request, Msg),
@ -199,23 +199,32 @@ on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) ->
on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State); on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State);
on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery,
#{pool_name := PoolName, base_path := BasePath} = State) -> #{pool_name := PoolName, base_path := BasePath} = State) ->
?SLOG(debug, #{msg => "http connector received request", ?TRACE("QUERY", "http_connector_received",
request => Request, connector => InstId, #{request => Request, connector => InstId, state => State}),
state => State}), NRequest = formalize_request(Method, BasePath, Request),
NRequest = update_path(BasePath, Request), case Result = ehttpc:request(case KeyOrNum of
Name = case KeyOrNum of
undefined -> PoolName; undefined -> PoolName;
_ -> {PoolName, KeyOrNum} _ -> {PoolName, KeyOrNum}
end, end, Method, NRequest, Timeout) of
Result = ehttpc:request(Name, Method, NRequest, Timeout),
case Result of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "http connector do reqeust failed", ?SLOG(error, #{msg => "http_connector_do_reqeust_failed",
request => NRequest, reason => Reason, request => NRequest, reason => Reason,
connector => InstId}), connector => InstId}),
emqx_resource:query_failed(AfterQuery); emqx_resource:query_failed(AfterQuery);
_ -> {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 ->
emqx_resource:query_success(AfterQuery) emqx_resource:query_success(AfterQuery);
{ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 ->
emqx_resource:query_success(AfterQuery);
{ok, StatusCode, _} ->
?SLOG(error, #{msg => "http connector do reqeust, received error response",
request => NRequest, connector => InstId,
status_code => StatusCode}),
emqx_resource:query_failed(AfterQuery);
{ok, StatusCode, _, _} ->
?SLOG(error, #{msg => "http connector do reqeust, received error response",
request => NRequest, connector => InstId,
status_code => StatusCode}),
emqx_resource:query_failed(AfterQuery)
end, end,
Result. Result.
@ -268,11 +277,16 @@ process_request(#{
} = Conf, Msg) -> } = Conf, Msg) ->
Conf#{ method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg)) Conf#{ method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg))
, path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg) , path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg)
, body => emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg) , body => process_request_body(BodyTks, Msg)
, headers => maps:to_list(proc_headers(HeadersTks, Msg)) , headers => maps:to_list(proc_headers(HeadersTks, Msg))
, request_timeout => ReqTimeout , request_timeout => ReqTimeout
}. }.
process_request_body([], Msg) ->
emqx_json:encode(Msg);
process_request_body(BodyTks, Msg) ->
emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg).
proc_headers(HeaderTks, Msg) -> proc_headers(HeaderTks, Msg) ->
maps:fold(fun(K, V, Acc) -> maps:fold(fun(K, V, Acc) ->
Acc#{emqx_plugin_libs_rule:proc_tmpl(K, Msg) => Acc#{emqx_plugin_libs_rule:proc_tmpl(K, Msg) =>
@ -296,10 +310,14 @@ check_ssl_opts(URLFrom, Conf) ->
{_, _} -> false {_, _} -> false
end. end.
update_path(BasePath, {Path, Headers}) -> formalize_request(Method, BasePath, {Path, Headers, _Body})
{filename:join(BasePath, Path), Headers}; when Method =:= get; Method =:= delete ->
update_path(BasePath, {Path, Headers, Body}) -> formalize_request(Method, BasePath, {Path, Headers});
{filename:join(BasePath, Path), Headers, Body}. formalize_request(_Method, BasePath, {Path, Headers, Body}) ->
{filename:join(BasePath, Path), Headers, Body};
formalize_request(_Method, BasePath, {Path, Headers}) ->
{filename:join(BasePath, Path), Headers}.
bin(Bin) when is_binary(Bin) -> bin(Bin) when is_binary(Bin) ->
Bin; Bin;

View File

@ -55,7 +55,7 @@ on_start(InstId, #{servers := Servers0,
pool_size := PoolSize, pool_size := PoolSize,
auto_reconnect := AutoReconn, auto_reconnect := AutoReconn,
ssl := SSL} = Config) -> ssl := SSL} = Config) ->
?SLOG(info, #{msg => "starting ldap connector", ?SLOG(info, #{msg => "starting_ldap_connector",
connector => InstId, config => Config}), connector => InstId, config => Config}),
Servers = [begin proplists:get_value(host, S) end || S <- Servers0], Servers = [begin proplists:get_value(host, S) end || S <- Servers0],
SslOpts = case maps:get(enable, SSL) of SslOpts = case maps:get(enable, SSL) of
@ -81,23 +81,21 @@ on_start(InstId, #{servers := Servers0,
{ok, #{poolname => PoolName}}. {ok, #{poolname => PoolName}}.
on_stop(InstId, #{poolname := PoolName}) -> on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping ldap connector", ?SLOG(info, #{msg => "stopping_ldap_connector",
connector => InstId}), connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName). emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) -> on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) ->
Request = {Base, Filter, Attributes}, Request = {Base, Filter, Attributes},
?SLOG(debug, #{msg => "ldap connector received request", ?TRACE("QUERY", "ldap_connector_received",
request => Request, connector => InstId, #{request => Request, connector => InstId, state => State}),
state => State}),
case Result = ecpool:pick_and_do( case Result = ecpool:pick_and_do(
PoolName, PoolName,
{?MODULE, search, [Base, Filter, Attributes]}, {?MODULE, search, [Base, Filter, Attributes]},
no_handover) of no_handover) of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "ldap connector do request failed", ?SLOG(error, #{msg => "ldap_connector_do_request_failed",
request => Request, connector => InstId, request => Request, connector => InstId, reason => Reason}),
reason => Reason}),
emqx_resource:query_failed(AfterQuery); emqx_resource:query_failed(AfterQuery);
_ -> _ ->
emqx_resource:query_success(AfterQuery) emqx_resource:query_success(AfterQuery)

View File

@ -34,6 +34,8 @@
, on_jsonify/1 , on_jsonify/1
]). ]).
%% ecpool callback
-export([connect/1]). -export([connect/1]).
-export([roots/0, fields/1]). -export([roots/0, fields/1]).
@ -125,11 +127,11 @@ on_start(InstId, Config = #{mongo_type := Type,
{options, init_topology_options(maps:to_list(Topology), [])}, {options, init_topology_options(maps:to_list(Topology), [])},
{worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)}], {worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)}],
PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolName = emqx_plugin_libs_pool:pool_name(InstId),
_ = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts), ok = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts),
{ok, #{poolname => PoolName, type => Type}}. {ok, #{poolname => PoolName, type => Type}}.
on_stop(InstId, #{poolname := PoolName}) -> on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping mongodb connector", ?SLOG(info, #{msg => "stopping_mongodb_connector",
connector => InstId}), connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName). emqx_plugin_libs_pool:stop_pool(PoolName).
@ -138,14 +140,13 @@ on_query(InstId,
AfterQuery, AfterQuery,
#{poolname := PoolName} = State) -> #{poolname := PoolName} = State) ->
Request = {Action, Collection, Selector, Docs}, Request = {Action, Collection, Selector, Docs},
?SLOG(debug, #{msg => "mongodb connector received request", ?TRACE("QUERY", "mongodb_connector_received",
request => Request, connector => InstId, #{request => Request, connector => InstId, state => State}),
state => State}),
case ecpool:pick_and_do(PoolName, case ecpool:pick_and_do(PoolName,
{?MODULE, mongo_query, [Action, Collection, Selector, Docs]}, {?MODULE, mongo_query, [Action, Collection, Selector, Docs]},
no_handover) of no_handover) of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "mongodb connector do query failed", ?SLOG(error, #{msg => "mongodb_connector_do_query_failed",
request => Request, reason => Reason, request => Request, reason => Reason,
connector => InstId}), connector => InstId}),
emqx_resource:query_failed(AfterQuery), emqx_resource:query_failed(AfterQuery),
@ -178,18 +179,22 @@ health_check(PoolName) ->
%% =================================================================== %% ===================================================================
%% TODO: log reasons
check_worker_health(Worker) -> check_worker_health(Worker) ->
case ecpool_worker:client(Worker) of case ecpool_worker:client(Worker) of
{ok, Conn} -> {ok, Conn} ->
%% we don't care if this returns something or not, we just to test the connection %% we don't care if this returns something or not, we just to test the connection
try mongo_api:find_one(Conn, <<"foo">>, #{}, #{}) of try mongo_api:find_one(Conn, <<"foo">>, #{}, #{}) of
{error, _} -> false; {error, _Reason} ->
false;
_ -> _ ->
true true
catch catch
_Class:_Error -> false _ : _ ->
false
end; end;
_ -> false _ ->
false
end. end.
connect(Opts) -> connect(Opts) ->

View File

@ -29,7 +29,7 @@
, bridges/0 , bridges/0
]). ]).
-export([on_message_received/2]). -export([on_message_received/3]).
%% callbacks of behaviour emqx_resource %% callbacks of behaviour emqx_resource
-export([ on_start/2 -export([ on_start/2
@ -68,10 +68,6 @@ fields("put") ->
fields("post") -> fields("post") ->
[ {type, mk(mqtt, #{desc => "The Connector Type"})} [ {type, mk(mqtt, #{desc => "The Connector Type"})}
, {name, mk(binary(),
#{ desc => "The Connector Name"
, example => <<"my_mqtt_connector">>
})}
] ++ fields("put"). ] ++ fields("put").
%% =================================================================== %% ===================================================================
@ -105,26 +101,29 @@ drop_bridge(Name) ->
case supervisor:terminate_child(?MODULE, Name) of case supervisor:terminate_child(?MODULE, Name) of
ok -> ok ->
supervisor:delete_child(?MODULE, Name); supervisor:delete_child(?MODULE, Name);
{error, not_found} ->
ok;
{error, Error} -> {error, Error} ->
{error, Error} {error, Error}
end. end.
%% =================================================================== %% ===================================================================
%% When use this bridge as a data source, ?MODULE:on_message_received/2 will be called %% When use this bridge as a data source, ?MODULE:on_message_received will be called
%% if the bridge received msgs from the remote broker. %% if the bridge received msgs from the remote broker.
on_message_received(Msg, HookPoint) -> on_message_received(Msg, HookPoint, InstId) ->
_ = emqx_resource:query(InstId, {message_received, Msg}),
emqx:run_hook(HookPoint, [Msg]). emqx:run_hook(HookPoint, [Msg]).
%% =================================================================== %% ===================================================================
on_start(InstId, Conf) -> on_start(InstId, Conf) ->
InstanceId = binary_to_atom(InstId, utf8), InstanceId = binary_to_atom(InstId, utf8),
?SLOG(info, #{msg => "starting mqtt connector", ?SLOG(info, #{msg => "starting_mqtt_connector",
connector => InstanceId, config => Conf}), connector => InstanceId, config => Conf}),
BasicConf = basic_config(Conf), BasicConf = basic_config(Conf),
BridgeConf = BasicConf#{ BridgeConf = BasicConf#{
name => InstanceId, name => InstanceId,
clientid => clientid(maps:get(clientid, Conf, InstId)), clientid => clientid(InstId),
subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined)), subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined), InstId),
forwards => make_forward_confs(maps:get(egress, Conf, undefined)) forwards => make_forward_confs(maps:get(egress, Conf, undefined))
}, },
case ?MODULE:create_bridge(BridgeConf) of case ?MODULE:create_bridge(BridgeConf) of
@ -139,19 +138,21 @@ on_start(InstId, Conf) ->
end. end.
on_stop(_InstId, #{name := InstanceId}) -> on_stop(_InstId, #{name := InstanceId}) ->
?SLOG(info, #{msg => "stopping mqtt connector", ?SLOG(info, #{msg => "stopping_mqtt_connector",
connector => InstanceId}), connector => InstanceId}),
case ?MODULE:drop_bridge(InstanceId) of case ?MODULE:drop_bridge(InstanceId) of
ok -> ok; ok -> ok;
{error, not_found} -> ok; {error, not_found} -> ok;
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "stop mqtt connector", ?SLOG(error, #{msg => "stop_mqtt_connector",
connector => InstanceId, reason => Reason}) connector => InstanceId, reason => Reason})
end. end.
on_query(_InstId, {message_received, _Msg}, AfterQuery, _State) ->
emqx_resource:query_success(AfterQuery);
on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) -> on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) ->
?SLOG(debug, #{msg => "send msg to remote node", message => Msg, ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
connector => InstanceId}),
emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg), emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
emqx_resource:query_success(AfterQuery). emqx_resource:query_success(AfterQuery).
@ -167,15 +168,15 @@ ensure_mqtt_worker_started(InstanceId) ->
{error, Reason} -> {error, Reason} {error, Reason} -> {error, Reason}
end. end.
make_sub_confs(EmptyMap) when map_size(EmptyMap) == 0 -> make_sub_confs(EmptyMap, _) when map_size(EmptyMap) == 0 ->
undefined; undefined;
make_sub_confs(undefined) -> make_sub_confs(undefined, _) ->
undefined; undefined;
make_sub_confs(SubRemoteConf) -> make_sub_confs(SubRemoteConf, InstId) ->
case maps:take(hookpoint, SubRemoteConf) of case maps:take(hookpoint, SubRemoteConf) of
error -> SubRemoteConf; error -> SubRemoteConf;
{HookPoint, SubConf} -> {HookPoint, SubConf} ->
MFA = {?MODULE, on_message_received, [HookPoint]}, MFA = {?MODULE, on_message_received, [HookPoint, InstId]},
SubConf#{on_message_received => MFA} SubConf#{on_message_received => MFA}
end. end.
@ -208,7 +209,7 @@ basic_config(#{
username => User, username => User,
password => Password, password => Password,
clean_start => CleanStart, clean_start => CleanStart,
keepalive => KeepAlive, keepalive => ms_to_s(KeepAlive),
retry_interval => RetryIntv, retry_interval => RetryIntv,
max_inflight => MaxInflight, max_inflight => MaxInflight,
ssl => EnableSsl, ssl => EnableSsl,
@ -216,5 +217,8 @@ basic_config(#{
if_record_metrics => true if_record_metrics => true
}. }.
ms_to_s(Ms) ->
erlang:ceil(Ms / 1000).
clientid(Id) -> clientid(Id) ->
iolist_to_binary([Id, ":", atom_to_list(node())]). iolist_to_binary([Id, ":", atom_to_list(node())]).

View File

@ -56,7 +56,7 @@ on_start(InstId, #{server := {Host, Port},
auto_reconnect := AutoReconn, auto_reconnect := AutoReconn,
pool_size := PoolSize, pool_size := PoolSize,
ssl := SSL } = Config) -> ssl := SSL } = Config) ->
?SLOG(info, #{msg => "starting mysql connector", ?SLOG(info, #{msg => "starting_mysql_connector",
connector => InstId, config => Config}), connector => InstId, config => Config}),
SslOpts = case maps:get(enable, SSL) of SslOpts = case maps:get(enable, SSL) of
true -> true ->
@ -75,7 +75,7 @@ on_start(InstId, #{server := {Host, Port},
{ok, #{poolname => PoolName}}. {ok, #{poolname => PoolName}}.
on_stop(InstId, #{poolname := PoolName}) -> on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping mysql connector", ?SLOG(info, #{msg => "stopping_mysql_connector",
connector => InstId}), connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName). emqx_plugin_libs_pool:stop_pool(PoolName).
@ -84,14 +84,13 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) ->
on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State); on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State);
on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) -> on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) ->
?SLOG(debug, #{msg => "mysql connector received sql query", ?TRACE("QUERY", "mysql_connector_received", #{connector => InstId, sql => SQL, state => State}),
connector => InstId, sql => SQL, state => State}),
case Result = ecpool:pick_and_do( case Result = ecpool:pick_and_do(
PoolName, PoolName,
{mysql, query, [SQL, Params, Timeout]}, {mysql, query, [SQL, Params, Timeout]},
no_handover) of no_handover) of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "mysql connector do sql query failed", ?SLOG(error, #{msg => "mysql_connector_do_sql_query_failed",
connector => InstId, sql => SQL, reason => Reason}), connector => InstId, sql => SQL, reason => Reason}),
emqx_resource:query_failed(AfterQuery); emqx_resource:query_failed(AfterQuery);
_ -> _ ->

View File

@ -32,7 +32,9 @@
-export([connect/1]). -export([connect/1]).
-export([query/3]). -export([ query/3
, prepared_query/4
]).
-export([do_health_check/1]). -export([do_health_check/1]).
@ -56,7 +58,7 @@ on_start(InstId, #{server := {Host, Port},
auto_reconnect := AutoReconn, auto_reconnect := AutoReconn,
pool_size := PoolSize, pool_size := PoolSize,
ssl := SSL } = Config) -> ssl := SSL } = Config) ->
?SLOG(info, #{msg => "starting postgresql connector", ?SLOG(info, #{msg => "starting_postgresql_connector",
connector => InstId, config => Config}), connector => InstId, config => Config}),
SslOpts = case maps:get(enable, SSL) of SslOpts = case maps:get(enable, SSL) of
true -> true ->
@ -82,15 +84,19 @@ on_stop(InstId, #{poolname := PoolName}) ->
connector => InstId}), connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName). emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) -> on_query(InstId, QueryParams, AfterQuery, #{poolname := PoolName} = State) ->
on_query(InstId, {sql, SQL, []}, AfterQuery, State); {Command, Args} = case QueryParams of
on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> {query, SQL} -> {query, [SQL, []]};
?SLOG(debug, #{msg => "postgresql connector received sql query", {query, SQL, Params} -> {query, [SQL, Params]};
connector => InstId, sql => SQL, state => State}), {prepared_query, Name, SQL} -> {prepared_query, [Name, SQL, []]};
case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of {prepared_query, Name, SQL, Params} -> {prepared_query, [Name, SQL, Params]}
end,
?TRACE("QUERY", "postgresql_connector_received",
#{connector => InstId, command => Command, args => Args, state => State}),
case Result = ecpool:pick_and_do(PoolName, {?MODULE, Command, Args}, no_handover) of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{ ?SLOG(error, #{
msg => "postgresql connector do sql query failed", msg => "postgresql_connector_do_sql_query_failed",
connector => InstId, sql => SQL, reason => Reason}), connector => InstId, sql => SQL, reason => Reason}),
emqx_resource:query_failed(AfterQuery); emqx_resource:query_failed(AfterQuery);
_ -> _ ->
@ -117,6 +123,9 @@ connect(Opts) ->
query(Conn, SQL, Params) -> query(Conn, SQL, Params) ->
epgsql:equery(Conn, SQL, Params). epgsql:equery(Conn, SQL, Params).
prepared_query(Conn, Name, SQL, Params) ->
epgsql:prepared_query2(Conn, Name, SQL, Params).
conn_opts(Opts) -> conn_opts(Opts) ->
conn_opts(Opts, []). conn_opts(Opts, []).
conn_opts([], Acc) -> conn_opts([], Acc) ->

View File

@ -20,12 +20,19 @@
-include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl").
-type server() :: tuple(). -type server() :: tuple().
%% {"127.0.0.1", 7000}
%% For eredis:start_link/1~7
-reflect_type([server/0]). -reflect_type([server/0]).
-typerefl_from_string({server/0, ?MODULE, to_server}). -typerefl_from_string({server/0, ?MODULE, to_server}).
-export([to_server/1]). -type servers() :: list().
%% [{"127.0.0.1", 7000}, {"127.0.0.2", 7000}]
%% For eredis_cluster
-reflect_type([servers/0]).
-typerefl_from_string({servers/0, ?MODULE, to_servers}).
-export([ to_server/1
, to_servers/1]).
-export([roots/0, fields/1]). -export([roots/0, fields/1]).
@ -63,14 +70,14 @@ fields(single) ->
redis_fields() ++ redis_fields() ++
emqx_connector_schema_lib:ssl_fields(); emqx_connector_schema_lib:ssl_fields();
fields(cluster) -> fields(cluster) ->
[ {servers, #{type => hoconsc:array(server())}} [ {servers, #{type => servers()}}
, {redis_type, #{type => hoconsc:enum([cluster]), , {redis_type, #{type => hoconsc:enum([cluster]),
default => cluster}} default => cluster}}
] ++ ] ++
redis_fields() ++ redis_fields() ++
emqx_connector_schema_lib:ssl_fields(); emqx_connector_schema_lib:ssl_fields();
fields(sentinel) -> fields(sentinel) ->
[ {servers, #{type => hoconsc:array(server())}} [ {servers, #{type => servers()}}
, {redis_type, #{type => hoconsc:enum([sentinel]), , {redis_type, #{type => hoconsc:enum([sentinel]),
default => sentinel}} default => sentinel}}
, {sentinel, #{type => string()}} , {sentinel, #{type => string()}}
@ -87,7 +94,7 @@ on_start(InstId, #{redis_type := Type,
pool_size := PoolSize, pool_size := PoolSize,
auto_reconnect := AutoReconn, auto_reconnect := AutoReconn,
ssl := SSL } = Config) -> ssl := SSL } = Config) ->
?SLOG(info, #{msg => "starting redis connector", ?SLOG(info, #{msg => "starting_redis_connector",
connector => InstId, config => Config}), connector => InstId, config => Config}),
Servers = case Type of Servers = case Type of
single -> [{servers, [maps:get(server, Config)]}]; single -> [{servers, [maps:get(server, Config)]}];
@ -120,20 +127,20 @@ on_start(InstId, #{redis_type := Type,
{ok, #{poolname => PoolName, type => Type}}. {ok, #{poolname => PoolName, type => Type}}.
on_stop(InstId, #{poolname := PoolName}) -> on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping redis connector", ?SLOG(info, #{msg => "stopping_redis_connector",
connector => InstId}), connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName). emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) -> on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) ->
?SLOG(debug, #{msg => "redis connector received cmd query", ?TRACE("QUERY", "redis_connector_received",
connector => InstId, sql => Command, state => State}), #{connector => InstId, sql => Command, state => State}),
Result = case Type of Result = case Type of
cluster -> eredis_cluster:q(PoolName, Command); cluster -> eredis_cluster:q(PoolName, Command);
_ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover) _ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover)
end, end,
case Result of case Result of
{error, Reason} -> {error, Reason} ->
?SLOG(error, #{msg => "redis connector do cmd query failed", ?SLOG(error, #{msg => "redis_connector_do_cmd_query_failed",
connector => InstId, sql => Command, reason => Reason}), connector => InstId, sql => Command, reason => Reason}),
emqx_resource:query_failed(AfterCommand); emqx_resource:query_failed(AfterCommand);
_ -> _ ->
@ -181,7 +188,23 @@ redis_fields() ->
]. ].
to_server(Server) -> to_server(Server) ->
case string:tokens(Server, ":") of try {ok, parse_server(Server)}
[Host, Port] -> {ok, {Host, list_to_integer(Port)}}; catch
_ -> {error, Server} throw : Error ->
Error
end.
to_servers(Servers) ->
try {ok, lists:map(fun parse_server/1, string:tokens(Servers, ", "))}
catch
throw : _Reason ->
{error, Servers}
end.
parse_server(Server) ->
case string:tokens(Server, ": ") of
[Host, Port] ->
{Host, list_to_integer(Port)};
_ ->
throw({error, Server})
end. end.

View File

@ -158,27 +158,23 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent)
RC =:= ?RC_NO_MATCHING_SUBSCRIBERS -> RC =:= ?RC_NO_MATCHING_SUBSCRIBERS ->
Parent ! {batch_ack, PktId}, ok; Parent ! {batch_ack, PktId}, ok;
handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) -> handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) ->
?SLOG(warning, #{msg => "publish to remote node falied", ?SLOG(warning, #{msg => "publish_to_remote_node_falied",
packet_id => PktId, reason_code => RC}). packet_id => PktId, reason_code => RC}).
handle_publish(Msg, undefined) -> handle_publish(Msg, undefined) ->
?SLOG(error, #{msg => "cannot publish to local broker as" ?SLOG(error, #{msg => "cannot_publish_to_local_broker_as"
" 'ingress' is not configured", "_'ingress'_is_not_configured",
message => Msg}); message => Msg});
handle_publish(Msg, Vars) -> handle_publish(Msg0, Vars) ->
?SLOG(debug, #{msg => "publish to local broker", Msg = format_msg_received(Msg0),
?SLOG(debug, #{msg => "publish_to_local_broker",
message => Msg, vars => Vars}), message => Msg, vars => Vars}),
emqx_metrics:inc('bridge.mqtt.message_received_from_remote', 1),
case Vars of case Vars of
#{on_message_received := {Mod, Func, Args}} -> #{on_message_received := {Mod, Func, Args}} ->
_ = erlang:apply(Mod, Func, [Msg | Args]); _ = erlang:apply(Mod, Func, [Msg | Args]);
_ -> ok _ -> ok
end, end,
case maps:get(local_topic, Vars, undefined) of maybe_publish_to_local_broker(Msg0, Vars).
undefined -> ok;
_Topic ->
emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars))
end.
handle_disconnected(Reason, Parent) -> handle_disconnected(Reason, Parent) ->
Parent ! {disconnected, self(), Reason}. Parent ! {disconnected, self(), Reason}.
@ -198,3 +194,45 @@ sub_remote_topics(ClientPid, #{remote_topic := FromTopic, remote_qos := QoS}) ->
process_config(Config) -> process_config(Config) ->
maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config). maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config).
maybe_publish_to_local_broker(#{topic := Topic} = Msg, #{remote_topic := SubTopic} = Vars) ->
case maps:get(local_topic, Vars, undefined) of
undefined ->
ok; %% local topic is not set, discard it
_ ->
case emqx_topic:match(Topic, SubTopic) of
true ->
_ = emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)),
ok;
false ->
?SLOG(warning, #{msg => "discard_message_as_topic_not_matched",
message => Msg, subscribed => SubTopic, got_topic => Topic})
end
end.
format_msg_received(#{dup := Dup, payload := Payload, properties := Props,
qos := QoS, retain := Retain, topic := Topic}) ->
#{event => '$bridges/mqtt',
id => emqx_guid:to_hexstr(emqx_guid:gen()),
payload => Payload,
topic => Topic,
qos => QoS,
dup => Dup,
retain => Retain,
pub_props => printable_maps(Props),
timestamp => erlang:system_time(millisecond)
}.
printable_maps(undefined) -> #{};
printable_maps(Headers) ->
maps:fold(
fun ('User-Property', V0, AccIn) when is_list(V0) ->
AccIn#{
'User-Property' => maps:from_list(V0),
'User-Property-Pairs' => [#{
key => Key,
value => Value
} || {Key, Value} <- V0]
};
(K, V0, AccIn) -> AccIn#{K => V0}
end, #{}, Headers).

View File

@ -61,12 +61,12 @@ make_pub_vars(Mountpoint, Conf) when is_map(Conf) ->
-> exp_msg(). -> exp_msg().
to_remote_msg(#message{flags = Flags0} = Msg, Vars) -> to_remote_msg(#message{flags = Flags0} = Msg, Vars) ->
Retain0 = maps:get(retain, Flags0, false), Retain0 = maps:get(retain, Flags0, false),
MapMsg = maps:put(retain, Retain0, emqx_message:to_map(Msg)), MapMsg = maps:put(retain, Retain0, emqx_rule_events:eventmsg_publish(Msg)),
to_remote_msg(MapMsg, Vars); to_remote_msg(MapMsg, Vars);
to_remote_msg(MapMsg, #{remote_topic := TopicToken, payload := PayloadToken, to_remote_msg(MapMsg, #{remote_topic := TopicToken, payload := PayloadToken,
remote_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) -> remote_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) ->
Topic = replace_vars_in_str(TopicToken, MapMsg), Topic = replace_vars_in_str(TopicToken, MapMsg),
Payload = replace_vars_in_str(PayloadToken, MapMsg), Payload = process_payload(PayloadToken, MapMsg),
QoS = replace_simple_var(QoSToken, MapMsg), QoS = replace_simple_var(QoSToken, MapMsg),
Retain = replace_simple_var(RetainToken, MapMsg), Retain = replace_simple_var(RetainToken, MapMsg),
#mqtt_msg{qos = QoS, #mqtt_msg{qos = QoS,
@ -82,13 +82,18 @@ to_broker_msg(#{dup := Dup, properties := Props} = MapMsg,
#{local_topic := TopicToken, payload := PayloadToken, #{local_topic := TopicToken, payload := PayloadToken,
local_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) -> local_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) ->
Topic = replace_vars_in_str(TopicToken, MapMsg), Topic = replace_vars_in_str(TopicToken, MapMsg),
Payload = replace_vars_in_str(PayloadToken, MapMsg), Payload = process_payload(PayloadToken, MapMsg),
QoS = replace_simple_var(QoSToken, MapMsg), QoS = replace_simple_var(QoSToken, MapMsg),
Retain = replace_simple_var(RetainToken, MapMsg), Retain = replace_simple_var(RetainToken, MapMsg),
set_headers(Props, set_headers(Props,
emqx_message:set_flags(#{dup => Dup, retain => Retain}, emqx_message:set_flags(#{dup => Dup, retain => Retain},
emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))). emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))).
process_payload([], Msg) ->
emqx_json:encode(Msg);
process_payload(Tks, Msg) ->
replace_vars_in_str(Tks, Msg).
%% Replace a string contains vars to another string in which the placeholders are replace by the %% Replace a string contains vars to another string in which the placeholders are replace by the
%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be: %% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be:
%% "a: 1". %% "a: 1".

View File

@ -39,7 +39,7 @@ fields("config") ->
fields("connector") -> fields("connector") ->
[ {mode, [ {mode,
sc(hoconsc:enum([cluster_singleton, cluster_shareload]), sc(hoconsc:enum([cluster_shareload]),
#{ default => cluster_shareload #{ default => cluster_shareload
, desc => """ , desc => """
The mode of the MQTT Bridge. Can be one of 'cluster_singleton' or 'cluster_shareload'<br> The mode of the MQTT Bridge. Can be one of 'cluster_singleton' or 'cluster_shareload'<br>
@ -55,12 +55,17 @@ clientid conflicts between different nodes. And we can only use shared subscript
topic filters for 'remote_topic' of ingress connections. topic filters for 'remote_topic' of ingress connections.
""" """
})} })}
, {name,
sc(binary(),
#{ nullable => true
, desc => "Connector name, used as a human-readable description of the connector."
})}
, {server, , {server,
sc(emqx_schema:ip_port(), sc(emqx_schema:ip_port(),
#{ default => "127.0.0.1:1883" #{ default => "127.0.0.1:1883"
, desc => "The host and port of the remote MQTT broker" , desc => "The host and port of the remote MQTT broker"
})} })}
, {reconnect_interval, mk_duration("reconnect interval", #{default => "30s"})} , {reconnect_interval, mk_duration("reconnect interval", #{default => "15s"})}
, {proto_ver, , {proto_ver,
sc(hoconsc:enum([v3, v4, v5]), sc(hoconsc:enum([v3, v4, v5]),
#{ default => v4 #{ default => v4
@ -76,17 +81,13 @@ topic filters for 'remote_topic' of ingress connections.
#{ default => "emqx" #{ default => "emqx"
, desc => "The password of the MQTT protocol" , desc => "The password of the MQTT protocol"
})} })}
, {clientid,
sc(binary(),
#{ desc => "The clientid of the MQTT protocol"
})}
, {clean_start, , {clean_start,
sc(boolean(), sc(boolean(),
#{ default => true #{ default => true
, desc => "The clean-start or the clean-session of the MQTT protocol" , desc => "The clean-start or the clean-session of the MQTT protocol"
})} })}
, {keepalive, mk_duration("keepalive", #{default => "300s"})} , {keepalive, mk_duration("keepalive", #{default => "300s"})}
, {retry_interval, mk_duration("retry interval", #{default => "30s"})} , {retry_interval, mk_duration("retry interval", #{default => "15s"})}
, {max_inflight, , {max_inflight,
sc(integer(), sc(integer(),
#{ default => 32 #{ default => 32

View File

@ -188,7 +188,7 @@ callback_mode() -> [state_functions].
%% @doc Config should be a map(). %% @doc Config should be a map().
init(#{name := Name} = ConnectOpts) -> init(#{name := Name} = ConnectOpts) ->
?SLOG(debug, #{msg => "starting bridge worker", ?SLOG(debug, #{msg => "starting_bridge_worker",
name => Name}), name => Name}),
erlang:process_flag(trap_exit, true), erlang:process_flag(trap_exit, true),
Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})), Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})),
@ -335,7 +335,7 @@ common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) ->
NewQ = replayq:append(Q, [Msg]), NewQ = replayq:append(Q, [Msg]),
{keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}}; {keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}};
common(StateName, Type, Content, #{name := Name} = State) -> common(StateName, Type, Content, #{name := Name} = State) ->
?SLOG(notice, #{msg => "Bridge discarded event", ?SLOG(notice, #{msg => "bridge_discarded_event",
name => Name, type => Type, state_name => StateName, name => Name, type => Type, state_name => StateName,
content => Content}), content => Content}),
{keep_state, State}. {keep_state, State}.
@ -349,7 +349,7 @@ do_connect(#{connect_opts := ConnectOpts,
{ok, State#{connection => Conn}}; {ok, State#{connection => Conn}};
{error, Reason} -> {error, Reason} ->
ConnectOpts1 = obfuscate(ConnectOpts), ConnectOpts1 = obfuscate(ConnectOpts),
?SLOG(error, #{msg => "Failed to connect", ?SLOG(error, #{msg => "failed_to_connect",
config => ConnectOpts1, reason => Reason}), config => ConnectOpts1, reason => Reason}),
{error, Reason, State} {error, Reason, State}
end. end.
@ -386,8 +386,8 @@ pop_and_send_loop(#{replayq := Q} = State, N) ->
end. end.
do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) -> do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) ->
?SLOG(error, #{msg => "cannot forward messages to remote broker" ?SLOG(error, #{msg => "cannot_forward_messages_to_remote_broker"
" as 'egress' is not configured", "_as_'egress'_is_not_configured",
messages => Msg}); messages => Msg});
do_send(#{inflight := Inflight, do_send(#{inflight := Inflight,
connection := Connection, connection := Connection,
@ -398,7 +398,7 @@ do_send(#{inflight := Inflight,
emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'), emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'),
emqx_connector_mqtt_msg:to_remote_msg(Message, Vars) emqx_connector_mqtt_msg:to_remote_msg(Message, Vars)
end, end,
?SLOG(debug, #{msg => "publish to remote broker", ?SLOG(debug, #{msg => "publish_to_remote_broker",
message => Msg, vars => Vars}), message => Msg, vars => Vars}),
case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(Msg)]) of case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(Msg)]) of
{ok, Refs} -> {ok, Refs} ->

View File

@ -22,15 +22,15 @@
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct.hrl").
-define(CONF_DEFAULT, <<"connectors: {}">>). %% output functions
-export([ inspect/3
]).
-define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>). -define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
-define(CONNECTR_TYPE, <<"mqtt">>). -define(CONNECTR_TYPE, <<"mqtt">>).
-define(CONNECTR_NAME, <<"test_connector">>). -define(CONNECTR_NAME, <<"test_connector">>).
-define(CONNECTR_ID, <<"mqtt:test_connector">>).
-define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>). -define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>).
-define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>). -define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>).
-define(BRIDGE_ID_INGRESS, <<"mqtt:ingress_test_bridge">>).
-define(BRIDGE_ID_EGRESS, <<"mqtt:egress_test_bridge">>).
-define(MQTT_CONNECOTR(Username), -define(MQTT_CONNECOTR(Username),
#{ #{
<<"server">> => <<"127.0.0.1:1883">>, <<"server">> => <<"127.0.0.1:1883">>,
@ -70,6 +70,9 @@
<<"failed">> := FAILED, <<"rate">> := SPEED, <<"failed">> := FAILED, <<"rate">> := SPEED,
<<"rate_last5m">> := SPEED5M, <<"rate_max">> := SPEEDMAX}). <<"rate_last5m">> := SPEED5M, <<"rate_max">> := SPEEDMAX}).
inspect(Selected, _Envs, _Args) ->
persistent_term:put(?MODULE, #{inspect => Selected}).
all() -> all() ->
emqx_common_test_helpers:all(?MODULE). emqx_common_test_helpers:all(?MODULE).
@ -92,21 +95,38 @@ init_per_suite(Config) ->
%% some testcases (may from other app) already get emqx_connector started %% some testcases (may from other app) already get emqx_connector started
_ = application:stop(emqx_resource), _ = application:stop(emqx_resource),
_ = application:stop(emqx_connector), _ = application:stop(emqx_connector),
ok = emqx_common_test_helpers:start_apps([emqx_connector, emqx_bridge, emqx_dashboard]), ok = emqx_common_test_helpers:start_apps([emqx_rule_engine, emqx_connector,
ok = emqx_config:init_load(emqx_connector_schema, ?CONF_DEFAULT), emqx_bridge, emqx_dashboard]),
ok = emqx_config:init_load(emqx_connector_schema, <<"connectors: {}">>),
ok = emqx_config:init_load(emqx_rule_engine_schema, <<"rule_engine {rules {}}">>),
ok = emqx_config:init_load(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT), ok = emqx_config:init_load(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
Config. Config.
end_per_suite(_Config) -> end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx_connector, emqx_bridge, emqx_dashboard]), emqx_common_test_helpers:stop_apps([emqx_rule_engine, emqx_connector, emqx_bridge, emqx_dashboard]),
ok. ok.
init_per_testcase(_, Config) -> init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
Config. Config.
end_per_testcase(_, _Config) -> end_per_testcase(_, _Config) ->
clear_resources(),
ok. ok.
clear_resources() ->
lists:foreach(fun(#{id := Id}) ->
ok = emqx_rule_engine:delete_rule(Id)
end, emqx_rule_engine:get_rules()),
lists:foreach(fun(#{id := Id}) ->
ok = emqx_bridge:remove(Id)
end, emqx_bridge:list()),
lists:foreach(fun(#{<<"id">> := Id}) ->
ok = emqx_connector:delete(Id)
end, emqx_connector:list()).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Testcases %% Testcases
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
@ -123,8 +143,7 @@ t_mqtt_crud_apis(_) ->
, <<"name">> => ?CONNECTR_NAME , <<"name">> => ?CONNECTR_NAME
}), }),
%ct:pal("---connector: ~p", [Connector]), #{ <<"id">> := ConnctorID
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
, <<"type">> := ?CONNECTR_TYPE , <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?CONNECTR_NAME , <<"name">> := ?CONNECTR_NAME
, <<"server">> := <<"127.0.0.1:1883">> , <<"server">> := <<"127.0.0.1:1883">>
@ -132,23 +151,13 @@ t_mqtt_crud_apis(_) ->
, <<"password">> := <<"">> , <<"password">> := <<"">>
, <<"proto_ver">> := <<"v4">> , <<"proto_ver">> := <<"v4">>
, <<"ssl">> := #{<<"enable">> := false} , <<"ssl">> := #{<<"enable">> := false}
}, jsx:decode(Connector)), } = jsx:decode(Connector),
%% create a again returns an error
{ok, 400, RetMsg} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
?assertMatch(
#{ <<"code">> := _
, <<"message">> := <<"connector already exists">>
}, jsx:decode(RetMsg)),
%% update the request-path of the connector %% update the request-path of the connector
User2 = <<"user2">>, User2 = <<"user2">>,
{ok, 200, Connector2} = request(put, uri(["connectors", ?CONNECTR_ID]), {ok, 200, Connector2} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR(User2)), ?MQTT_CONNECOTR(User2)),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID ?assertMatch(#{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:1883">> , <<"server">> := <<"127.0.0.1:1883">>
, <<"username">> := User2 , <<"username">> := User2
, <<"password">> := <<"">> , <<"password">> := <<"">>
@ -158,7 +167,7 @@ t_mqtt_crud_apis(_) ->
%% list all connectors again, assert Connector2 is in it %% list all connectors again, assert Connector2 is in it
{ok, 200, Connector2Str} = request(get, uri(["connectors"]), []), {ok, 200, Connector2Str} = request(get, uri(["connectors"]), []),
?assertMatch([#{ <<"id">> := ?CONNECTR_ID ?assertMatch([#{ <<"id">> := ConnctorID
, <<"type">> := ?CONNECTR_TYPE , <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?CONNECTR_NAME , <<"name">> := ?CONNECTR_NAME
, <<"server">> := <<"127.0.0.1:1883">> , <<"server">> := <<"127.0.0.1:1883">>
@ -169,8 +178,8 @@ t_mqtt_crud_apis(_) ->
}], jsx:decode(Connector2Str)), }], jsx:decode(Connector2Str)),
%% get the connector by id %% get the connector by id
{ok, 200, Connector3Str} = request(get, uri(["connectors", ?CONNECTR_ID]), []), {ok, 200, Connector3Str} = request(get, uri(["connectors", ConnctorID]), []),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID ?assertMatch(#{ <<"id">> := ConnctorID
, <<"type">> := ?CONNECTR_TYPE , <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?CONNECTR_NAME , <<"name">> := ?CONNECTR_NAME
, <<"server">> := <<"127.0.0.1:1883">> , <<"server">> := <<"127.0.0.1:1883">>
@ -181,11 +190,11 @@ t_mqtt_crud_apis(_) ->
}, jsx:decode(Connector3Str)), }, jsx:decode(Connector3Str)),
%% delete the connector %% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
%% update a deleted connector returns an error %% update a deleted connector returns an error
{ok, 404, ErrMsg2} = request(put, uri(["connectors", ?CONNECTR_ID]), {ok, 404, ErrMsg2} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR(User2)), ?MQTT_CONNECOTR(User2)),
?assertMatch( ?assertMatch(
#{ <<"code">> := _ #{ <<"code">> := _
@ -194,10 +203,6 @@ t_mqtt_crud_apis(_) ->
ok. ok.
t_mqtt_conn_bridge_ingress(_) -> t_mqtt_conn_bridge_ingress(_) ->
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
User1 = <<"user1">>, User1 = <<"user1">>,
{ok, 201, Connector} = request(post, uri(["connectors"]), {ok, 201, Connector} = request(post, uri(["connectors"]),
@ -205,28 +210,28 @@ t_mqtt_conn_bridge_ingress(_) ->
, <<"name">> => ?CONNECTR_NAME , <<"name">> => ?CONNECTR_NAME
}), }),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID #{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:1883">> , <<"server">> := <<"127.0.0.1:1883">>
, <<"num_of_bridges">> := 0 , <<"num_of_bridges">> := 0
, <<"username">> := User1 , <<"username">> := User1
, <<"password">> := <<"">> , <<"password">> := <<"">>
, <<"proto_ver">> := <<"v4">> , <<"proto_ver">> := <<"v4">>
, <<"ssl">> := #{<<"enable">> := false} , <<"ssl">> := #{<<"enable">> := false}
}, jsx:decode(Connector)), } = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST %% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now %% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_INGRESS(?CONNECTR_ID)#{ ?MQTT_BRIDGE_INGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_INGRESS <<"name">> => ?BRIDGE_NAME_INGRESS
}), }),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_INGRESS #{ <<"id">> := BridgeIDIngress
, <<"type">> := <<"mqtt">> , <<"type">> := <<"mqtt">>
, <<"status">> := <<"connected">> , <<"status">> := <<"connected">>
, <<"connector">> := ?CONNECTR_ID , <<"connector">> := ConnctorID
}, jsx:decode(Bridge)), } = jsx:decode(Bridge),
%% we now test if the bridge works as expected %% we now test if the bridge works as expected
@ -236,8 +241,8 @@ t_mqtt_conn_bridge_ingress(_) ->
emqx:subscribe(LocalTopic), emqx:subscribe(LocalTopic),
%% PUBLISH a message to the 'remote' broker, as we have only one broker, %% PUBLISH a message to the 'remote' broker, as we have only one broker,
%% the remote broker is also the local one. %% the remote broker is also the local one.
wait_for_resource_ready(BridgeIDIngress, 5),
emqx:publish(emqx_message:make(RemoteTopic, Payload)), emqx:publish(emqx_message:make(RemoteTopic, Payload)),
%% we should receive a message on the local broker, with specified topic %% we should receive a message on the local broker, with specified topic
?assert( ?assert(
receive receive
@ -252,25 +257,21 @@ t_mqtt_conn_bridge_ingress(_) ->
end), end),
%% get the connector by id, verify the num_of_bridges now is 1 %% get the connector by id, verify the num_of_bridges now is 1
{ok, 200, Connector1Str} = request(get, uri(["connectors", ?CONNECTR_ID]), []), {ok, 200, Connector1Str} = request(get, uri(["connectors", ConnctorID]), []),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID ?assertMatch(#{ <<"id">> := ConnctorID
, <<"num_of_bridges">> := 1 , <<"num_of_bridges">> := 1
}, jsx:decode(Connector1Str)), }, jsx:decode(Connector1Str)),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_INGRESS]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% delete the connector %% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
ok. ok.
t_mqtt_conn_bridge_egress(_) -> t_mqtt_conn_bridge_egress(_) ->
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
User1 = <<"user1">>, User1 = <<"user1">>,
{ok, 201, Connector} = request(post, uri(["connectors"]), {ok, 201, Connector} = request(post, uri(["connectors"]),
@ -279,29 +280,28 @@ t_mqtt_conn_bridge_egress(_) ->
}), }),
%ct:pal("---connector: ~p", [Connector]), %ct:pal("---connector: ~p", [Connector]),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID #{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:1883">> , <<"server">> := <<"127.0.0.1:1883">>
, <<"username">> := User1 , <<"username">> := User1
, <<"password">> := <<"">> , <<"password">> := <<"">>
, <<"proto_ver">> := <<"v4">> , <<"proto_ver">> := <<"v4">>
, <<"ssl">> := #{<<"enable">> := false} , <<"ssl">> := #{<<"enable">> := false}
}, jsx:decode(Connector)), } = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST %% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now %% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{ ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}), }),
%ct:pal("---bridge: ~p", [Bridge]), #{ <<"id">> := BridgeIDEgress
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
, <<"type">> := ?CONNECTR_TYPE , <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?BRIDGE_NAME_EGRESS , <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"connected">> , <<"status">> := <<"connected">>
, <<"connector">> := ?CONNECTR_ID , <<"connector">> := ConnctorID
}, jsx:decode(Bridge)), } = jsx:decode(Bridge),
%% we now test if the bridge works as expected %% we now test if the bridge works as expected
LocalTopic = <<"local_topic/1">>, LocalTopic = <<"local_topic/1">>,
@ -310,6 +310,7 @@ t_mqtt_conn_bridge_egress(_) ->
emqx:subscribe(RemoteTopic), emqx:subscribe(RemoteTopic),
%% PUBLISH a message to the 'local' broker, as we have only one broker, %% PUBLISH a message to the 'local' broker, as we have only one broker,
%% the remote broker is also the local one. %% the remote broker is also the local one.
wait_for_resource_ready(BridgeIDEgress, 5),
emqx:publish(emqx_message:make(LocalTopic, Payload)), emqx:publish(emqx_message:make(LocalTopic, Payload)),
%% we should receive a message on the "remote" broker, with specified topic %% we should receive a message on the "remote" broker, with specified topic
@ -326,19 +327,19 @@ t_mqtt_conn_bridge_egress(_) ->
end), end),
%% verify the metrics of the bridge %% verify the metrics of the bridge
{ok, 200, BridgeStr} = request(get, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS ?assertMatch(#{ <<"id">> := BridgeIDEgress
, <<"metrics">> := ?metrics(1, 1, 0, _, _, _) , <<"metrics">> := ?metrics(1, 1, 0, _, _, _)
, <<"node_metrics">> := , <<"node_metrics">> :=
[#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}] [#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}]
}, jsx:decode(BridgeStr)), }, jsx:decode(BridgeStr)),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% delete the connector %% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []), {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
ok. ok.
@ -346,10 +347,6 @@ t_mqtt_conn_bridge_egress(_) ->
%% - update a connector should also update all of the the bridges %% - update a connector should also update all of the the bridges
%% - cannot delete a connector that is used by at least one bridge %% - cannot delete a connector that is used by at least one bridge
t_mqtt_conn_update(_) -> t_mqtt_conn_update(_) ->
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
{ok, 201, Connector} = request(post, uri(["connectors"]), {ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>) ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)
@ -358,44 +355,41 @@ t_mqtt_conn_update(_) ->
}), }),
%ct:pal("---connector: ~p", [Connector]), %ct:pal("---connector: ~p", [Connector]),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID #{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:1883">> , <<"server">> := <<"127.0.0.1:1883">>
}, jsx:decode(Connector)), } = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST %% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now %% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{ ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}), }),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS #{ <<"id">> := BridgeIDEgress
, <<"type">> := <<"mqtt">> , <<"type">> := <<"mqtt">>
, <<"name">> := ?BRIDGE_NAME_EGRESS , <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"connected">> , <<"status">> := <<"connected">>
, <<"connector">> := ?CONNECTR_ID , <<"connector">> := ConnctorID
}, jsx:decode(Bridge)), } = jsx:decode(Bridge),
wait_for_resource_ready(BridgeIDEgress, 2),
%% then we try to update 'server' of the connector, to an unavailable IP address %% then we try to update 'server' of the connector, to an unavailable IP address
%% the update should fail because of 'unreachable' or 'connrefused' %% the update should fail because of 'unreachable' or 'connrefused'
{ok, 400, _ErrorMsg} = request(put, uri(["connectors", ?CONNECTR_ID]), {ok, 400, _ErrorMsg} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR2(<<"127.0.0.1:2603">>)), ?MQTT_CONNECOTR2(<<"127.0.0.1:2603">>)),
%% we fix the 'server' parameter to a normal one, it should work %% we fix the 'server' parameter to a normal one, it should work
{ok, 200, _} = request(put, uri(["connectors", ?CONNECTR_ID]), {ok, 200, _} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR2(<<"127.0.0.1 : 1883">>)), ?MQTT_CONNECOTR2(<<"127.0.0.1 : 1883">>)),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% delete the connector %% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []). {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []).
t_mqtt_conn_update2(_) -> t_mqtt_conn_update2(_) ->
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
%% but this connector is point to a unreachable server "2603" %% but this connector is point to a unreachable server "2603"
{ok, 201, Connector} = request(post, uri(["connectors"]), {ok, 201, Connector} = request(post, uri(["connectors"]),
@ -404,38 +398,71 @@ t_mqtt_conn_update2(_) ->
, <<"name">> => ?CONNECTR_NAME , <<"name">> => ?CONNECTR_NAME
}), }),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID #{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:2603">> , <<"server">> := <<"127.0.0.1:2603">>
}, jsx:decode(Connector)), } = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST %% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now %% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]), {ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{ ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE, <<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}), }),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS #{ <<"id">> := BridgeIDEgress
, <<"type">> := <<"mqtt">> , <<"type">> := <<"mqtt">>
, <<"name">> := ?BRIDGE_NAME_EGRESS , <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"disconnected">> , <<"status">> := <<"disconnected">>
, <<"connector">> := ?CONNECTR_ID , <<"connector">> := ConnctorID
}, jsx:decode(Bridge)), } = jsx:decode(Bridge),
%% We try to fix the 'server' parameter, to another unavailable server..
%% The update should success: we don't check the connectivity of the new config
%% if the resource is now disconnected.
{ok, 200, _} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR2(<<"127.0.0.1:2604">>)),
%% we fix the 'server' parameter to a normal one, it should work %% we fix the 'server' parameter to a normal one, it should work
{ok, 200, _} = request(put, uri(["connectors", ?CONNECTR_ID]), {ok, 200, _} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)), ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)),
{ok, 200, BridgeStr} = request(get, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS ?assertMatch(#{ <<"id">> := BridgeIDEgress
, <<"status">> := <<"connected">> , <<"status">> := <<"connected">>
}, jsx:decode(BridgeStr)), }, jsx:decode(BridgeStr)),
%% delete the bridge %% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% delete the connector %% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []), {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []). {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []).
t_mqtt_conn_update3(_) ->
%% we add a mqtt connector, using POST
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)
#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
#{ <<"id">> := ConnctorID } = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
#{ <<"id">> := BridgeIDEgress
, <<"connector">> := ConnctorID
} = jsx:decode(Bridge),
wait_for_resource_ready(BridgeIDEgress, 2),
%% delete the connector should fail because it is in use by a bridge
{ok, 403, _} = request(delete, uri(["connectors", ConnctorID]), []),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
%% the connector now can be deleted without problems
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
t_mqtt_conn_testing(_) -> t_mqtt_conn_testing(_) ->
%% APIs for testing the connectivity %% APIs for testing the connectivity
%% then we add a mqtt connector, using POST %% then we add a mqtt connector, using POST
@ -450,6 +477,153 @@ t_mqtt_conn_testing(_) ->
<<"name">> => ?BRIDGE_NAME_EGRESS <<"name">> => ?BRIDGE_NAME_EGRESS
}). }).
t_ingress_mqtt_bridge_with_rules(_) ->
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
#{ <<"id">> := ConnctorID } = jsx:decode(Connector),
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_INGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_INGRESS
}),
#{ <<"id">> := BridgeIDIngress } = jsx:decode(Bridge),
{ok, 201, Rule} = request(post, uri(["rules"]),
#{<<"name">> => <<"A rule get messages from a source mqtt bridge">>,
<<"enable">> => true,
<<"outputs">> => [#{<<"function">> => "emqx_connector_api_SUITE:inspect"}],
<<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">>
}),
#{<<"id">> := RuleId} = jsx:decode(Rule),
%% we now test if the bridge works as expected
RemoteTopic = <<"remote_topic/1">>,
LocalTopic = <<"local_topic/", RemoteTopic/binary>>,
Payload = <<"hello">>,
emqx:subscribe(LocalTopic),
%% PUBLISH a message to the 'remote' broker, as we have only one broker,
%% the remote broker is also the local one.
wait_for_resource_ready(BridgeIDIngress, 5),
emqx:publish(emqx_message:make(RemoteTopic, Payload)),
%% we should receive a message on the local broker, with specified topic
?assert(
receive
{deliver, LocalTopic, #message{payload = Payload}} ->
ct:pal("local broker got message: ~p on topic ~p", [Payload, LocalTopic]),
true;
Msg ->
ct:pal("Msg: ~p", [Msg]),
false
after 100 ->
false
end),
%% and also the rule should be matched, with matched + 1:
{ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
#{ <<"id">> := RuleId
, <<"metrics">> := #{<<"matched">> := 1}
} = jsx:decode(Rule1),
%% we also check if the outputs of the rule is triggered
?assertMatch(#{inspect := #{
event := '$bridges/mqtt',
id := MsgId,
payload := Payload,
topic := RemoteTopic,
qos := 0,
dup := false,
retain := false,
pub_props := #{},
timestamp := _
}} when is_binary(MsgId), persistent_term:get(?MODULE)),
{ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
t_egress_mqtt_bridge_with_rules(_) ->
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
#{ <<"id">> := ConnctorID } = jsx:decode(Connector),
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
#{ <<"id">> := BridgeIDEgress } = jsx:decode(Bridge),
{ok, 201, Rule} = request(post, uri(["rules"]),
#{<<"name">> => <<"A rule send messages to a sink mqtt bridge">>,
<<"enable">> => true,
<<"outputs">> => [BridgeIDEgress],
<<"sql">> => <<"SELECT * from \"t/1\"">>
}),
#{<<"id">> := RuleId} = jsx:decode(Rule),
%% we now test if the bridge works as expected
LocalTopic = <<"local_topic/1">>,
RemoteTopic = <<"remote_topic/", LocalTopic/binary>>,
Payload = <<"hello">>,
emqx:subscribe(RemoteTopic),
%% PUBLISH a message to the 'local' broker, as we have only one broker,
%% the remote broker is also the local one.
wait_for_resource_ready(BridgeIDEgress, 5),
emqx:publish(emqx_message:make(LocalTopic, Payload)),
%% we should receive a message on the "remote" broker, with specified topic
?assert(
receive
{deliver, RemoteTopic, #message{payload = Payload}} ->
ct:pal("local broker got message: ~p on topic ~p", [Payload, RemoteTopic]),
true;
Msg ->
ct:pal("Msg: ~p", [Msg]),
false
after 100 ->
false
end),
emqx:unsubscribe(RemoteTopic),
%% PUBLISH a message to the rule.
Payload2 = <<"hi">>,
RuleTopic = <<"t/1">>,
RemoteTopic2 = <<"remote_topic/", RuleTopic/binary>>,
emqx:subscribe(RemoteTopic2),
wait_for_resource_ready(BridgeIDEgress, 5),
emqx:publish(emqx_message:make(RuleTopic, Payload2)),
{ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
#{ <<"id">> := RuleId
, <<"metrics">> := #{<<"matched">> := 1}
} = jsx:decode(Rule1),
%% we should receive a message on the "remote" broker, with specified topic
?assert(
receive
{deliver, RemoteTopic2, #message{payload = Payload2}} ->
ct:pal("local broker got message: ~p on topic ~p", [Payload2, RemoteTopic2]),
true;
Msg ->
ct:pal("Msg: ~p", [Msg]),
false
after 100 ->
false
end),
%% verify the metrics of the bridge
{ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
?assertMatch(#{ <<"id">> := BridgeIDEgress
, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)
, <<"node_metrics">> :=
[#{<<"node">> := _, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)}]
}, jsx:decode(BridgeStr)),
{ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% HTTP Request %% HTTP Request
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -483,3 +657,13 @@ auth_header_() ->
{ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
{"Authorization", "Bearer " ++ binary_to_list(Token)}. {"Authorization", "Bearer " ++ binary_to_list(Token)}.
wait_for_resource_ready(InstId, 0) ->
ct:pal("--- bridge ~p: ~p", [InstId, emqx_bridge:lookup(InstId)]),
ct:fail(wait_resource_timeout);
wait_for_resource_ready(InstId, Retry) ->
case emqx_bridge:lookup(InstId) of
{ok, #{resource_data := #{status := started}}} -> ok;
_ ->
timer:sleep(100),
wait_for_resource_ready(InstId, Retry-1)
end.

View File

@ -1,4 +1,4 @@
{deps, [ {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} {deps, [ {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.6"}}}
, {emqx, {path, "../emqx"}} , {emqx, {path, "../emqx"}}
]}. ]}.

View File

@ -151,9 +151,9 @@ authorize(Req) ->
ok -> ok ->
ok; ok;
{error, token_timeout} -> {error, token_timeout} ->
return_unauthorized(<<"TOKEN_TIME_OUT">>, <<"POST '/login', get new token">>); {401, 'TOKEN_TIME_OUT', <<"Token expired, get new token by POST /login">>};
{error, not_found} -> {error, not_found} ->
return_unauthorized(<<"BAD_TOKEN">>, <<"POST '/login'">>) {401, 'BAD_TOKEN', <<"Get a token by POST /login">>}
end; end;
_ -> _ ->
return_unauthorized(<<"AUTHORIZATION_HEADER_ERROR">>, return_unauthorized(<<"AUTHORIZATION_HEADER_ERROR">>,

View File

@ -123,7 +123,7 @@ schema("/users/:username") ->
#{in => path, example => <<"admin">>})}], #{in => path, example => <<"admin">>})}],
'requestBody' => [ 'requestBody' => [
{ description { description
, mk(emqx_schema:unicode_binary(), , mk(binary(),
#{desc => <<"User description">>, example => <<"administrator">>})} #{desc => <<"User description">>, example => <<"administrator">>})}
], ],
responses => #{ responses => #{
@ -176,7 +176,7 @@ schema("/users/:username/change_pwd") ->
fields(user) -> fields(user) ->
[ [
{description, {description,
mk(emqx_schema:unicode_binary(), mk(binary(),
#{desc => <<"User description">>, example => "administrator"})}, #{desc => <<"User description">>, example => "administrator"})},
{username, {username,
mk(binary(), mk(binary(),

View File

@ -312,6 +312,9 @@ responses(Responses, Module) ->
response(Status, Bin, {Acc, RefsAcc, Module}) when is_binary(Bin) -> response(Status, Bin, {Acc, RefsAcc, Module}) when is_binary(Bin) ->
{Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module}; {Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module};
%% Support swagger raw object(file download).
response(Status, #{content := _} = Content, {Acc, RefsAcc, Module}) ->
{Acc#{integer_to_binary(Status) => Content}, RefsAcc, Module};
response(Status, ?REF(StructName), {Acc, RefsAcc, Module}) -> response(Status, ?REF(StructName), {Acc, RefsAcc, Module}) ->
response(Status, ?R_REF(Module, StructName), {Acc, RefsAcc, Module}); response(Status, ?R_REF(Module, StructName), {Acc, RefsAcc, Module});
response(Status, ?R_REF(_Mod, _Name) = RRef, {Acc, RefsAcc, Module}) -> response(Status, ?R_REF(_Mod, _Name) = RRef, {Acc, RefsAcc, Module}) ->
@ -423,8 +426,10 @@ typename_to_spec("duration_ms()", _Mod) -> #{type => string, example => <<"32s">
typename_to_spec("percent()", _Mod) -> #{type => number, example => <<"12%">>}; typename_to_spec("percent()", _Mod) -> #{type => number, example => <<"12%">>};
typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>}; typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>};
typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>}; typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>};
typename_to_spec("ip_ports()", _Mod) -> #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>}; typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>};
typename_to_spec("server()", Mod) -> typename_to_spec("ip_port()", Mod); typename_to_spec("server()", Mod) -> typename_to_spec("ip_port()", Mod);
typename_to_spec("servers()", Mod) -> typename_to_spec("ip_ports()", Mod);
typename_to_spec("connect_timeout()", Mod) -> typename_to_spec("timeout()", Mod); typename_to_spec("connect_timeout()", Mod) -> typename_to_spec("timeout()", Mod);
typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, example => infinity}, typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, example => infinity},
#{type => integer, example => 100}], example => infinity}; #{type => integer, example => 100}], example => infinity};

View File

@ -16,9 +16,16 @@
-module(emqx_coap_impl). -module(emqx_coap_impl).
-behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/include/emqx_gateway.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl").
-behaviour(emqx_gateway_impl). -import(emqx_gateway_utils,
[ normalize_config/1
, start_listeners/4
, stop_listeners/2
]).
%% APIs %% APIs
-export([ reg/0 -export([ reg/0
@ -30,8 +37,6 @@
, on_gateway_unload/2 , on_gateway_unload/2
]). ]).
-include_lib("emqx/include/logger.hrl").
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% APIs %% APIs
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -51,12 +56,20 @@ unreg() ->
on_gateway_load(_Gateway = #{name := GwName, on_gateway_load(_Gateway = #{name := GwName,
config := Config config := Config
}, Ctx) -> }, Ctx) ->
Listeners = emqx_gateway_utils:normalize_config(Config), Listeners = normalize_config(Config),
ListenerPids = lists:map(fun(Lis) -> ModCfg = #{frame_mod => emqx_coap_frame,
start_listener(GwName, Ctx, Lis) chann_mod => emqx_coap_channel
end, Listeners), },
case start_listeners(
{ok, ListenerPids, #{ctx => Ctx}}. Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
{ok, ListenerPids, #{ctx => Ctx}};
{error, {Reason, Listener}} ->
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway), GwName = maps:get(name, Gateway),
@ -76,63 +89,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
on_gateway_unload(_Gateway = #{ name := GwName, on_gateway_unload(_Gateway = #{ name := GwName,
config := Config config := Config
}, _GwState) -> }, _GwState) ->
Listeners = emqx_gateway_utils:normalize_config(Config), Listeners = normalize_config(Config),
lists:foreach(fun(Lis) -> stop_listeners(GwName, Listeners).
stop_listener(GwName, Lis)
end, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{ctx => Ctx,
listener => {GwName, Type, LisName},
frame_mod => emqx_coap_frame,
chann_mod => emqx_coap_channel
},
MFA = {emqx_gateway_conn, start_link, [NCfg]},
do_start_listener(Type, Name, ListenOn, SocketOpts, MFA).
do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_udp(Name, ListenOn, SocketOpts, MFA);
do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_dtls(Name, ListenOn, SocketOpts, MFA).
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.

View File

@ -533,6 +533,20 @@ params_client_searching_in_qs() ->
mk(binary(), mk(binary(),
M#{desc => <<"Match the client socket connected datatime less than " M#{desc => <<"Match the client socket connected datatime less than "
"a certain value">>})} "a certain value">>})}
, {endpoint_name,
mk(binary(),
M#{desc => <<"Match the lwm2m client's endpoint name">>})}
, {like_endpoint_name,
mk(binary(),
M#{desc => <<"Use sub-string to match lwm2m client's endpoint name">>})}
, {gte_lifetime,
mk(binary(),
M#{desc => <<"Match the lwm2m client registered lifetime greater "
"than a certain value">>})}
, {lte_lifetime,
mk(binary(),
M#{desc => <<"Match the lwm2m client registered lifetime less than "
"a certain value">>})}
]. ].
params_paging() -> params_paging() ->

View File

@ -580,7 +580,7 @@ common_listener_opts() ->
#{ nullable => {true, recursively} #{ nullable => {true, recursively}
, desc => <<"The authenticatior for this listener">> , desc => <<"The authenticatior for this listener">>
})} })}
]. ] ++ emqx_gateway_schema:proxy_protocol_opts().
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% examples %% examples

View File

@ -28,6 +28,8 @@
%, 'gateway-banned'/1 %, 'gateway-banned'/1
]). ]).
-elvis([{elvis_style, function_naming_convention, disable}]).
-spec load() -> ok. -spec load() -> ok.
load() -> load() ->
Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)],
@ -50,18 +52,24 @@ is_cmd(Fun) ->
%% Cmds %% Cmds
gateway(["list"]) -> gateway(["list"]) ->
lists:foreach(fun(#{name := Name} = Gateway) -> lists:foreach(
%% TODO: More infos: listeners?, connected? fun (#{name := Name, status := unloaded}) ->
Status = maps:get(status, Gateway, stopped), print("Gateway(name=~ts, status=unloaded)\n", [Name]);
print("Gateway(name=~ts, status=~ts)~n", [Name, Status]) (#{name := Name, status := stopped, stopped_at := StoppedAt}) ->
end, emqx_gateway:list()); print("Gateway(name=~ts, status=stopped, stopped_at=~ts)\n",
[Name, StoppedAt]);
(#{name := Name, status := running, current_connections := ConnCnt,
started_at := StartedAt}) ->
print("Gateway(name=~ts, status=running, clients=~w, started_at=~ts)\n",
[Name, ConnCnt, StartedAt])
end, emqx_gateway_http:gateways(all));
gateway(["lookup", Name]) -> gateway(["lookup", Name]) ->
case emqx_gateway:lookup(atom(Name)) of case emqx_gateway:lookup(atom(Name)) of
undefined -> undefined ->
print("undefined~n"); print("undefined\n");
Info -> Info ->
print("~p~n", [Info]) print("~p\n", [Info])
end; end;
gateway(["load", Name, Conf]) -> gateway(["load", Name, Conf]) ->
@ -70,17 +78,17 @@ gateway(["load", Name, Conf]) ->
emqx_json:decode(Conf, [return_maps]) emqx_json:decode(Conf, [return_maps])
) of ) of
{ok, _} -> {ok, _} ->
print("ok~n"); print("ok\n");
{error, Reason} -> {error, Reason} ->
print("Error: ~p~n", [Reason]) print("Error: ~p\n", [Reason])
end; end;
gateway(["unload", Name]) -> gateway(["unload", Name]) ->
case emqx_gateway_conf:unload_gateway(bin(Name)) of case emqx_gateway_conf:unload_gateway(bin(Name)) of
ok -> ok ->
print("ok~n"); print("ok\n");
{error, Reason} -> {error, Reason} ->
print("Error: ~p~n", [Reason]) print("Error: ~p\n", [Reason])
end; end;
gateway(["stop", Name]) -> gateway(["stop", Name]) ->
@ -89,9 +97,9 @@ gateway(["stop", Name]) ->
#{<<"enable">> => <<"false">>} #{<<"enable">> => <<"false">>}
) of ) of
{ok, _} -> {ok, _} ->
print("ok~n"); print("ok\n");
{error, Reason} -> {error, Reason} ->
print("Error: ~p~n", [Reason]) print("Error: ~p\n", [Reason])
end; end;
gateway(["start", Name]) -> gateway(["start", Name]) ->
@ -100,9 +108,9 @@ gateway(["start", Name]) ->
#{<<"enable">> => <<"true">>} #{<<"enable">> => <<"true">>}
) of ) of
{ok, _} -> {ok, _} ->
print("ok~n"); print("ok\n");
{error, Reason} -> {error, Reason} ->
print("Error: ~p~n", [Reason]) print("Error: ~p\n", [Reason])
end; end;
gateway(_) -> gateway(_) ->
@ -123,7 +131,7 @@ gateway(_) ->
'gateway-registry'(["list"]) -> 'gateway-registry'(["list"]) ->
lists:foreach( lists:foreach(
fun({Name, #{cbkmod := CbMod}}) -> fun({Name, #{cbkmod := CbMod}}) ->
print("Registered Name: ~ts, Callback Module: ~ts~n", [Name, CbMod]) print("Registered Name: ~ts, Callback Module: ~ts\n", [Name, CbMod])
end, end,
emqx_gateway_registry:list()); emqx_gateway_registry:list());
@ -137,7 +145,7 @@ gateway(_) ->
InfoTab = emqx_gateway_cm:tabname(info, Name), InfoTab = emqx_gateway_cm:tabname(info, Name),
case ets:info(InfoTab) of case ets:info(InfoTab) of
undefined -> undefined ->
print("Bad Gateway Name.~n"); print("Bad Gateway Name.\n");
_ -> _ ->
dump(InfoTab, client) dump(InfoTab, client)
end; end;
@ -145,7 +153,7 @@ gateway(_) ->
'gateway-clients'(["lookup", Name, ClientId]) -> 'gateway-clients'(["lookup", Name, ClientId]) ->
ChanTab = emqx_gateway_cm:tabname(chan, Name), ChanTab = emqx_gateway_cm:tabname(chan, Name),
case ets:lookup(ChanTab, bin(ClientId)) of case ets:lookup(ChanTab, bin(ClientId)) of
[] -> print("Not Found.~n"); [] -> print("Not Found.\n");
[Chann] -> [Chann] ->
InfoTab = emqx_gateway_cm:tabname(info, Name), InfoTab = emqx_gateway_cm:tabname(info, Name),
[ChannInfo] = ets:lookup(InfoTab, Chann), [ChannInfo] = ets:lookup(InfoTab, Chann),
@ -154,8 +162,8 @@ gateway(_) ->
'gateway-clients'(["kick", Name, ClientId]) -> 'gateway-clients'(["kick", Name, ClientId]) ->
case emqx_gateway_cm:kick_session(Name, bin(ClientId)) of case emqx_gateway_cm:kick_session(Name, bin(ClientId)) of
ok -> print("ok~n"); ok -> print("ok\n");
_ -> print("Not Found.~n") _ -> print("Not Found.\n")
end; end;
'gateway-clients'(_) -> 'gateway-clients'(_) ->
@ -171,11 +179,11 @@ gateway(_) ->
Tab = emqx_gateway_metrics:tabname(Name), Tab = emqx_gateway_metrics:tabname(Name),
case ets:info(Tab) of case ets:info(Tab) of
undefined -> undefined ->
print("Bad Gateway Name.~n"); print("Bad Gateway Name.\n");
_ -> _ ->
lists:foreach( lists:foreach(
fun({K, V}) -> fun({K, V}) ->
print("~-30s: ~w~n", [K, V]) print("~-30s: ~w\n", [K, V])
end, lists:sort(ets:tab2list(Tab))) end, lists:sort(ets:tab2list(Tab)))
end; end;
@ -232,7 +240,7 @@ print_record({client, {_, Infos, Stats}}) ->
print("Client(~ts, username=~ts, peername=~ts, " print("Client(~ts, username=~ts, peername=~ts, "
"clean_start=~ts, keepalive=~w, " "clean_start=~ts, keepalive=~w, "
"subscriptions=~w, delivered_msgs=~w, " "subscriptions=~w, delivered_msgs=~w, "
"connected=~ts, created_at=~w, connected_at=~w)~n", "connected=~ts, created_at=~w, connected_at=~w)\n",
[format(K, maps:get(K, Info)) || K <- InfoKeys]). [format(K, maps:get(K, Info)) || K <- InfoKeys]).
print(S) -> emqx_ctl:print(S). print(S) -> emqx_ctl:print(S).

View File

@ -50,6 +50,8 @@
-export([namespace/0, roots/0 , fields/1]). -export([namespace/0, roots/0 , fields/1]).
-export([proxy_protocol_opts/0]).
namespace() -> gateway. namespace() -> gateway.
roots() -> [gateway]. roots() -> [gateway].

View File

@ -18,6 +18,7 @@
-module(emqx_gateway_utils). -module(emqx_gateway_utils).
-include("emqx_gateway.hrl"). -include("emqx_gateway.hrl").
-include_lib("emqx/include/logger.hrl").
-export([ childspec/2 -export([ childspec/2
, childspec/3 , childspec/3
@ -26,6 +27,12 @@
, find_sup_child/2 , find_sup_child/2
]). ]).
-export([ start_listeners/4
, start_listener/4
, stop_listeners/2
, stop_listener/2
]).
-export([ apply/2 -export([ apply/2
, format_listenon/1 , format_listenon/1
, parse_listenon/1 , parse_listenon/1
@ -89,9 +96,15 @@ childspec(Id, Type, Mod, Args) ->
-spec supervisor_ret(supervisor:startchild_ret()) -spec supervisor_ret(supervisor:startchild_ret())
-> {ok, pid()} -> {ok, pid()}
| {error, supervisor:startchild_err()}. | {error, supervisor:startchild_err()}.
supervisor_ret({ok, Pid, _Info}) -> {ok, Pid}; supervisor_ret({ok, Pid, _Info}) ->
supervisor_ret({error, {Reason, _Child}}) -> {error, Reason}; {ok, Pid};
supervisor_ret(Ret) -> Ret. supervisor_ret({error, {Reason, Child}}) ->
case element(1, Child) == child of
true -> {error, Reason};
_ -> {error, {Reason, Child}}
end;
supervisor_ret(Ret) ->
Ret.
-spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id()) -spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id())
-> false -> false
@ -102,6 +115,120 @@ find_sup_child(Sup, ChildId) ->
{_Id, Pid, _Type, _Mods} -> {ok, Pid} {_Id, Pid, _Type, _Mods} -> {ok, Pid}
end. end.
%% @doc start listeners. close all listeners if someone failed
-spec start_listeners(Listeners :: list(),
GwName :: atom(),
Ctx :: map(),
ModCfg)
-> {ok, [pid()]}
| {error, term()}
when ModCfg :: #{frame_mod := atom(), chann_mod := atom()}.
start_listeners(Listeners, GwName, Ctx, ModCfg) ->
start_listeners(Listeners, GwName, Ctx, ModCfg, []).
start_listeners([], _, _, _, Acc) ->
{ok, lists:map(fun({listener, {_, Pid}}) -> Pid end, Acc)};
start_listeners([L | Ls], GwName, Ctx, ModCfg, Acc) ->
case start_listener(GwName, Ctx, L, ModCfg) of
{ok, {ListenerId, ListenOn, Pid}} ->
NAcc = Acc ++ [{listener, {{ListenerId, ListenOn}, Pid}}],
start_listeners(Ls, GwName, Ctx, ModCfg, NAcc);
{error, Reason} ->
lists:foreach(fun({listener, {{ListenerId, ListenOn}, _}}) ->
esockd:close({ListenerId, ListenOn})
end, Acc),
{error, {Reason, L}}
end.
-spec start_listener(GwName :: atom(),
Ctx :: emqx_gateway_ctx:context(),
Listener :: tuple(),
ModCfg :: map())
-> {ok, {ListenerId :: atom(), esockd:listen_on(), pid()}}
| {error, term()}.
start_listener(GwName, Ctx,
{Type, LisName, ListenOn, SocketOpts, Cfg}, ModCfg) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = maps:merge(Cfg, ModCfg),
case start_listener(GwName, Ctx, Type,
LisName, ListenOn, SocketOpts, NCfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
{ok, {ListenerId, ListenOn, Pid}};
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
emqx_gateway_utils:supervisor_ret({error, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{ ctx => Ctx
, listener => {GwName, Type, LisName}
},
NSocketOpts = merge_default(Type, SocketOpts),
MFA = {emqx_gateway_conn, start_link, [NCfg]},
do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA).
merge_default(Udp, Options) ->
{Key, Default} = case Udp of
udp ->
{udp_options, default_udp_options()};
dtls ->
{udp_options, default_udp_options()};
tcp ->
{tcp_options, default_tcp_options()};
ssl ->
{tcp_options, default_tcp_options()}
end,
case lists:keytake(Key, 1, Options) of
{value, {Key, TcpOpts}, Options1} ->
[{Key, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{Key, Default} | Options]
end.
do_start_listener(Type, Name, ListenOn, SocketOpts, MFA)
when Type == tcp;
Type == ssl ->
esockd:open(Name, ListenOn, SocketOpts, MFA);
do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_udp(Name, ListenOn, SocketOpts, MFA);
do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_dtls(Name, ListenOn, SocketOpts, MFA).
-spec stop_listeners(GwName :: atom(), Listeners :: list()) -> ok.
stop_listeners(GwName, Listeners) ->
lists:foreach(fun(L) -> stop_listener(GwName, L) end, Listeners).
-spec stop_listener(GwName :: atom(), Listener :: tuple()) -> ok.
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.
apply({M, F, A}, A2) when is_atom(M), apply({M, F, A}, A2) when is_atom(M),
is_atom(M), is_atom(M),
is_list(A), is_list(A),

View File

@ -19,6 +19,14 @@
-behaviour(emqx_gateway_impl). -behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
-import(emqx_gateway_utils,
[ normalize_config/1
, start_listeners/4
, stop_listeners/2
]).
%% APIs %% APIs
-export([ reg/0 -export([ reg/0
, unreg/0 , unreg/0
@ -29,8 +37,6 @@
, on_gateway_unload/2 , on_gateway_unload/2
]). ]).
-include_lib("emqx/include/logger.hrl").
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% APIs %% APIs
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -47,6 +53,73 @@ unreg() ->
%% emqx_gateway_registry callbacks %% emqx_gateway_registry callbacks
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
on_gateway_load(_Gateway = #{ name := GwName,
config := Config
}, Ctx) ->
%% XXX: How to monitor it ?
%% Start grpc client pool & client channel
PoolName = pool_name(GwName),
PoolSize = emqx_vm:schedulers() * 2,
{ok, PoolSup} = emqx_pool_sup:start_link(
PoolName, hash, PoolSize,
{emqx_exproto_gcli, start_link, []}),
_ = start_grpc_client_channel(GwName,
maps:get(handler, Config, undefined)
),
%% XXX: How to monitor it ?
_ = start_grpc_server(GwName, maps:get(server, Config, undefined)),
NConfig = maps:without(
[server, handler],
Config#{pool_name => PoolName}
),
Listeners = emqx_gateway_utils:normalize_config(
NConfig#{handler => GwName}
),
ModCfg = #{frame_mod => emqx_exproto_frame,
chann_mod => emqx_exproto_channel
},
case start_listeners(
Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
{ok, ListenerPids, _GwState = #{ctx => Ctx, pool => PoolSup}};
{error, {Reason, Listener}} ->
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway),
try
%% XXX: 1. How hot-upgrade the changes ???
%% XXX: 2. Check the New confs first before destroy old instance ???
on_gateway_unload(Gateway, GwState),
on_gateway_load(Gateway#{config => Config}, Ctx)
catch
Class : Reason : Stk ->
logger:error("Failed to update ~ts; "
"reason: {~0p, ~0p} stacktrace: ~0p",
[GwName, Class, Reason, Stk]),
{error, {Class, Reason}}
end.
on_gateway_unload(_Gateway = #{ name := GwName,
config := Config
}, _GwState = #{pool := PoolSup}) ->
Listeners = emqx_gateway_utils:normalize_config(Config),
%% Stop funcs???
exit(PoolSup, kill),
stop_grpc_server(GwName),
stop_grpc_client_channel(GwName),
stop_listeners(GwName, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_grpc_server(_GwName, undefined) -> start_grpc_server(_GwName, undefined) ->
undefined; undefined;
start_grpc_server(GwName, Options = #{bind := ListenOn}) -> start_grpc_server(GwName, Options = #{bind := ListenOn}) ->
@ -103,140 +176,9 @@ stop_grpc_client_channel(GwName) ->
_ = grpc_client_sup:stop_channel_pool(GwName), _ = grpc_client_sup:stop_channel_pool(GwName),
ok. ok.
on_gateway_load(_Gateway = #{ name := GwName,
config := Config
}, Ctx) ->
%% XXX: How to monitor it ?
%% Start grpc client pool & client channel
PoolName = pool_name(GwName),
PoolSize = emqx_vm:schedulers() * 2,
{ok, PoolSup} = emqx_pool_sup:start_link(
PoolName, hash, PoolSize,
{emqx_exproto_gcli, start_link, []}),
_ = start_grpc_client_channel(GwName,
maps:get(handler, Config, undefined)
),
%% XXX: How to monitor it ?
_ = start_grpc_server(GwName, maps:get(server, Config, undefined)),
NConfig = maps:without(
[server, handler],
Config#{pool_name => PoolName}
),
Listeners = emqx_gateway_utils:normalize_config(
NConfig#{handler => GwName}
),
ListenerPids = lists:map(fun(Lis) ->
start_listener(GwName, Ctx, Lis)
end, Listeners),
{ok, ListenerPids, _GwState = #{ctx => Ctx, pool => PoolSup}}.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway),
try
%% XXX: 1. How hot-upgrade the changes ???
%% XXX: 2. Check the New confs first before destroy old instance ???
on_gateway_unload(Gateway, GwState),
on_gateway_load(Gateway#{config => Config}, Ctx)
catch
Class : Reason : Stk ->
logger:error("Failed to update ~ts; "
"reason: {~0p, ~0p} stacktrace: ~0p",
[GwName, Class, Reason, Stk]),
{error, {Class, Reason}}
end.
on_gateway_unload(_Gateway = #{ name := GwName,
config := Config
}, _GwState = #{pool := PoolSup}) ->
Listeners = emqx_gateway_utils:normalize_config(Config),
%% Stop funcs???
exit(PoolSup, kill),
stop_grpc_server(GwName),
stop_grpc_client_channel(GwName),
lists:foreach(fun(Lis) ->
stop_listener(GwName, Lis)
end, Listeners).
pool_name(GwName) -> pool_name(GwName) ->
list_to_atom(lists:concat([GwName, "_gcli_pool"])). list_to_atom(lists:concat([GwName, "_gcli_pool"])).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{
ctx => Ctx,
listener => {GwName, Type, LisName},
frame_mod => emqx_exproto_frame,
chann_mod => emqx_exproto_channel
},
MFA = {emqx_gateway_conn, start_link, [NCfg]},
NSockOpts = merge_default_by_type(Type, SocketOpts),
do_start_listener(Type, Name, ListenOn, NSockOpts, MFA).
do_start_listener(Type, Name, ListenOn, Opts, MFA)
when Type == tcp;
Type == ssl ->
esockd:open(Name, ListenOn, Opts, MFA);
do_start_listener(udp, Name, ListenOn, Opts, MFA) ->
esockd:open_udp(Name, ListenOn, Opts, MFA);
do_start_listener(dtls, Name, ListenOn, Opts, MFA) ->
esockd:open_dtls(Name, ListenOn, Opts, MFA).
merge_default_by_type(Type, Options) when Type =:= tcp;
Type =:= ssl ->
Default = emqx_gateway_utils:default_tcp_options(),
case lists:keytake(tcp_options, 1, Options) of
{value, {tcp_options, TcpOpts}, Options1} ->
[{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{tcp_options, Default} | Options]
end;
merge_default_by_type(Type, Options) when Type =:= udp;
Type =:= dtls ->
Default = emqx_gateway_utils:default_udp_options(),
case lists:keytake(udp_options, 1, Options) of
{value, {udp_options, TcpOpts}, Options1} ->
[{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{udp_options, Default} | Options]
end.
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST). -ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args). console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else. -else.

View File

@ -19,6 +19,8 @@
-behaviour(emqx_gateway_impl). -behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
%% APIs %% APIs
-export([ reg/0 -export([ reg/0
, unreg/0 , unreg/0
@ -29,8 +31,6 @@
, on_gateway_unload/2 , on_gateway_unload/2
]). ]).
-include_lib("emqx/include/logger.hrl").
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% APIs %% APIs
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -54,10 +54,20 @@ on_gateway_load(_Gateway = #{ name := GwName,
case emqx_lwm2m_xml_object_db:start_link(XmlDir) of case emqx_lwm2m_xml_object_db:start_link(XmlDir) of
{ok, RegPid} -> {ok, RegPid} ->
Listeners = emqx_gateway_utils:normalize_config(Config), Listeners = emqx_gateway_utils:normalize_config(Config),
ListenerPids = lists:map(fun(Lis) -> ModCfg = #{frame_mod => emqx_coap_frame,
start_listener(GwName, Ctx, Lis) chann_mod => emqx_lwm2m_channel
end, Listeners), },
{ok, ListenerPids, _GwState = #{ctx => Ctx, registry => RegPid}}; case emqx_gateway_utils:start_listeners(
Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
{ok, ListenerPids, #{ctx => Ctx, registry => RegPid}};
{error, {Reason, Listener}} ->
_ = emqx_lwm2m_xml_object_db:stop(),
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end;
{error, Reason} -> {error, Reason} ->
throw({badconf, #{ key => xml_dir throw({badconf, #{ key => xml_dir
, value => XmlDir , value => XmlDir
@ -85,73 +95,4 @@ on_gateway_unload(_Gateway = #{ name := GwName,
}, _GwState = #{registry := RegPid}) -> }, _GwState = #{registry := RegPid}) ->
exit(RegPid, kill), exit(RegPid, kill),
Listeners = emqx_gateway_utils:normalize_config(Config), Listeners = emqx_gateway_utils:normalize_config(Config),
lists:foreach(fun(Lis) -> emqx_gateway_utils:stop_listeners(GwName, Listeners).
stop_listener(GwName, Lis)
end, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{ ctx => Ctx
, listener => {GwName, Type, LisName}
, frame_mod => emqx_coap_frame
, chann_mod => emqx_lwm2m_channel
},
NSocketOpts = merge_default(SocketOpts),
MFA = {emqx_gateway_conn, start_link, [NCfg]},
do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA).
merge_default(Options) ->
Default = emqx_gateway_utils:default_udp_options(),
case lists:keytake(udp_options, 1, Options) of
{value, {udp_options, TcpOpts}, Options1} ->
[{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{udp_options, Default} | Options]
end.
do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_udp(Name, ListenOn, SocketOpts, MFA);
do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_dtls(Name, ListenOn, SocketOpts, MFA).
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.

View File

@ -19,6 +19,14 @@
-behaviour(emqx_gateway_impl). -behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
-import(emqx_gateway_utils,
[ normalize_config/1
, start_listeners/4
, stop_listeners/2
]).
%% APIs %% APIs
-export([ reg/0 -export([ reg/0
, unreg/0 , unreg/0
@ -29,8 +37,6 @@
, on_gateway_unload/2 , on_gateway_unload/2
]). ]).
-include_lib("emqx/include/logger.hrl").
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% APIs %% APIs
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -70,12 +76,23 @@ on_gateway_load(_Gateway = #{ name := GwName,
[broadcast, predefined], [broadcast, predefined],
Config#{registry => emqx_sn_registry:lookup_name(RegistrySvr)} Config#{registry => emqx_sn_registry:lookup_name(RegistrySvr)}
), ),
Listeners = emqx_gateway_utils:normalize_config(NConfig), Listeners = emqx_gateway_utils:normalize_config(NConfig),
ListenerPids = lists:map(fun(Lis) -> ModCfg = #{frame_mod => emqx_sn_frame,
start_listener(GwName, Ctx, Lis) chann_mod => emqx_sn_channel
end, Listeners), },
{ok, ListenerPids, _InstaState = #{ctx => Ctx}}.
case start_listeners(
Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
{ok, ListenerPids, _GwState = #{ctx => Ctx}};
{error, {Reason, Listener}} ->
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway), GwName = maps:get(name, Gateway),
@ -95,68 +112,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
on_gateway_unload(_Gateway = #{ name := GwName, on_gateway_unload(_Gateway = #{ name := GwName,
config := Config config := Config
}, _GwState) -> }, _GwState) ->
Listeners = emqx_gateway_utils:normalize_config(Config), Listeners = normalize_config(Config),
lists:foreach(fun(Lis) -> stop_listeners(GwName, Listeners).
stop_listener(GwName, Lis)
end, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{
ctx => Ctx,
listene => {GwName, Type, LisName},
frame_mod => emqx_sn_frame,
chann_mod => emqx_sn_channel
},
esockd:open_udp(Name, ListenOn, merge_default(SocketOpts),
{emqx_gateway_conn, start_link, [NCfg]}).
merge_default(Options) ->
Default = emqx_gateway_utils:default_udp_options(),
case lists:keytake(udp_options, 1, Options) of
{value, {udp_options, TcpOpts}, Options1} ->
[{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{udp_options, Default} | Options]
end.
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.

View File

@ -18,6 +18,15 @@
-behaviour(emqx_gateway_impl). -behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/include/emqx_gateway.hrl").
-import(emqx_gateway_utils,
[ normalize_config/1
, start_listeners/4
, stop_listeners/2
]).
%% APIs %% APIs
-export([ reg/0 -export([ reg/0
, unreg/0 , unreg/0
@ -28,9 +37,6 @@
, on_gateway_unload/2 , on_gateway_unload/2
]). ]).
-include_lib("emqx_gateway/include/emqx_gateway.hrl").
-include_lib("emqx/include/logger.hrl").
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% APIs %% APIs
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -52,15 +58,22 @@ unreg() ->
on_gateway_load(_Gateway = #{ name := GwName, on_gateway_load(_Gateway = #{ name := GwName,
config := Config config := Config
}, Ctx) -> }, Ctx) ->
%% Step1. Fold the config to listeners Listeners = normalize_config(Config),
Listeners = emqx_gateway_utils:normalize_config(Config), ModCfg = #{frame_mod => emqx_stomp_frame,
%% Step2. Start listeners or escokd:specs chann_mod => emqx_stomp_channel
ListenerPids = lists:map(fun(Lis) -> },
start_listener(GwName, Ctx, Lis) case start_listeners(
end, Listeners), Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
%% FIXME: How to throw an exception to interrupt the restart logic ? %% FIXME: How to throw an exception to interrupt the restart logic ?
%% FIXME: Assign ctx to GwState %% FIXME: Assign ctx to GwState
{ok, ListenerPids, _GwState = #{ctx => Ctx}}. {ok, ListenerPids, _GwState = #{ctx => Ctx}};
{error, {Reason, Listener}} ->
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway), GwName = maps:get(name, Gateway),
@ -80,68 +93,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
on_gateway_unload(_Gateway = #{ name := GwName, on_gateway_unload(_Gateway = #{ name := GwName,
config := Config config := Config
}, _GwState) -> }, _GwState) ->
Listeners = emqx_gateway_utils:normalize_config(Config), Listeners = normalize_config(Config),
lists:foreach(fun(Lis) -> stop_listeners(GwName, Listeners).
stop_listener(GwName, Lis)
end, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{
ctx => Ctx,
listener => {GwName, Type, LisName}, %% Used for authn
frame_mod => emqx_stomp_frame,
chann_mod => emqx_stomp_channel
},
esockd:open(Name, ListenOn, merge_default(SocketOpts),
{emqx_gateway_conn, start_link, [NCfg]}).
merge_default(Options) ->
Default = emqx_gateway_utils:default_tcp_options(),
case lists:keytake(tcp_options, 1, Options) of
{value, {tcp_options, TcpOpts}, Options1} ->
[{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{tcp_options, Default} | Options]
end.
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.

View File

@ -0,0 +1,150 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_gateway_cli_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-define(GP(S), begin S, receive {fmt, P} -> P; O -> O end end).
%% this parses to #{}, will not cause config cleanup
%% so we will need call emqx_config:erase
-define(CONF_DEFAULT, <<"
gateway {}
">>).
%%--------------------------------------------------------------------
%% Setup
%%--------------------------------------------------------------------
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:erase(gateway),
emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]),
Conf.
end_per_suite(Conf) ->
emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]),
Conf.
init_per_testcase(_, Conf) ->
Self = self(),
ok = meck:new(emqx_ctl, [passthrough, no_history, no_link]),
ok = meck:expect(emqx_ctl, usage,
fun(L) -> emqx_ctl:format_usage(L) end),
ok = meck:expect(emqx_ctl, print,
fun(Fmt) ->
Self ! {fmt, emqx_ctl:format(Fmt)}
end),
ok = meck:expect(emqx_ctl, print,
fun(Fmt, Args) ->
Self ! {fmt, emqx_ctl:format(Fmt, Args)}
end),
Conf.
end_per_testcase(_, _) ->
meck:unload([emqx_ctl]),
ok.
%%--------------------------------------------------------------------
%% Cases
%%--------------------------------------------------------------------
%% TODO:
t_load_unload(_) ->
ok.
t_gateway_registry_usage(_) ->
?assertEqual(
["gateway-registry list # List all registered gateways\n"],
emqx_gateway_cli:'gateway-registry'(usage)).
t_gateway_registry_list(_) ->
emqx_gateway_cli:'gateway-registry'(["list"]),
?assertEqual(
"Registered Name: coap, Callback Module: emqx_coap_impl\n"
"Registered Name: exproto, Callback Module: emqx_exproto_impl\n"
"Registered Name: lwm2m, Callback Module: emqx_lwm2m_impl\n"
"Registered Name: mqttsn, Callback Module: emqx_sn_impl\n"
"Registered Name: stomp, Callback Module: emqx_stomp_impl\n"
, acc_print()).
t_gateway_usage(_) ->
?assertEqual(
["gateway list # List all gateway\n",
"gateway lookup <Name> # Lookup a gateway detailed informations\n",
"gateway load <Name> <JsonConf> # Load a gateway with config\n",
"gateway unload <Name> # Unload the gateway\n",
"gateway stop <Name> # Stop the gateway\n",
"gateway start <Name> # Start the gateway\n"],
emqx_gateway_cli:gateway(usage)
).
t_gateway_list(_) ->
emqx_gateway_cli:gateway(["list"]),
?assertEqual(
"Gateway(name=coap, status=unloaded)\n"
"Gateway(name=exproto, status=unloaded)\n"
"Gateway(name=lwm2m, status=unloaded)\n"
"Gateway(name=mqttsn, status=unloaded)\n"
"Gateway(name=stomp, status=unloaded)\n"
, acc_print()).
t_gateway_load(_) ->
ok.
t_gateway_unload(_) ->
ok.
t_gateway_start(_) ->
ok.
t_gateway_stop(_) ->
ok.
t_gateway_clients_usage(_) ->
ok.
t_gateway_clients_list(_) ->
ok.
t_gateway_clients_lookup(_) ->
ok.
t_gateway_clients_kick(_) ->
ok.
t_gateway_metrcis_usage(_) ->
ok.
t_gateway_metrcis(_) ->
ok.
acc_print() ->
lists:concat(lists:reverse(acc_print([]))).
acc_print(Acc) ->
receive
{fmt, S} -> acc_print([S|Acc])
after 200 ->
Acc
end.

View File

@ -85,7 +85,6 @@ reboot_apps() ->
, esockd , esockd
, ranch , ranch
, cowboy , cowboy
, emqx_conf
, emqx , emqx
, emqx_prometheus , emqx_prometheus
, emqx_modules , emqx_modules
@ -96,7 +95,6 @@ reboot_apps() ->
, emqx_resource , emqx_resource
, emqx_rule_engine , emqx_rule_engine
, emqx_bridge , emqx_bridge
, emqx_bridge_mqtt
, emqx_plugin_libs , emqx_plugin_libs
, emqx_management , emqx_management
, emqx_retainer , emqx_retainer
@ -112,17 +110,18 @@ sorted_reboot_apps() ->
app_deps(App) -> app_deps(App) ->
case application:get_key(App, applications) of case application:get_key(App, applications) of
undefined -> []; undefined -> undefined;
{ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List) {ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List)
end. end.
sorted_reboot_apps(Apps) -> sorted_reboot_apps(Apps) ->
G = digraph:new(), G = digraph:new(),
try try
lists:foreach(fun({App, Deps}) -> add_app(G, App, Deps) end, Apps), NoDepApps = add_apps_to_digraph(G, Apps),
case digraph_utils:topsort(G) of case digraph_utils:topsort(G) of
Sorted when is_list(Sorted) -> Sorted when is_list(Sorted) ->
Sorted; %% ensure emqx_conf boot up first
[emqx_conf | Sorted ++ (NoDepApps -- Sorted)];
false -> false ->
Loops = find_loops(G), Loops = find_loops(G),
error({circular_application_dependency, Loops}) error({circular_application_dependency, Loops})
@ -131,23 +130,33 @@ sorted_reboot_apps(Apps) ->
digraph:delete(G) digraph:delete(G)
end. end.
add_app(G, App, undefined) -> %% Build a dependency graph from the provided application list.
%% Return top-sort result of the apps.
%% Isolated apps without which are not dependency of any other apps are
%% put to the end of the list in the original order.
add_apps_to_digraph(G, Apps) ->
lists:foldl(fun
({App, undefined}, Acc) ->
?SLOG(debug, #{msg => "app_is_not_loaded", app => App}),
Acc;
({App, []}, Acc) ->
Acc ++ [App]; %% use '++' to keep the original order
({App, Deps}, Acc) ->
add_app_deps_to_digraph(G, App, Deps),
Acc
end, [], Apps).
add_app_deps_to_digraph(G, App, undefined) ->
?SLOG(debug, #{msg => "app_is_not_loaded", app => App}), ?SLOG(debug, #{msg => "app_is_not_loaded", app => App}),
%% not loaded %% not loaded
add_app(G, App, []); add_app_deps_to_digraph(G, App, []);
% We ALWAYS want to add `emqx_conf', even if no other app declare a add_app_deps_to_digraph(_G, _App, []) ->
% dependency on it. Otherwise, emqx may fail to load the config
% schemas, especially in the test profile.
add_app(G, App = emqx_conf, []) ->
digraph:add_vertex(G, App),
ok; ok;
add_app(_G, _App, []) -> add_app_deps_to_digraph(G, App, [Dep | Deps]) ->
ok;
add_app(G, App, [Dep | Deps]) ->
digraph:add_vertex(G, App), digraph:add_vertex(G, App),
digraph:add_vertex(G, Dep), digraph:add_vertex(G, Dep),
digraph:add_edge(G, Dep, App), %% dep -> app as dependency digraph:add_edge(G, Dep, App), %% dep -> app as dependency
add_app(G, App, Deps). add_app_deps_to_digraph(G, App, Deps).
find_loops(G) -> find_loops(G) ->
lists:filtermap( lists:filtermap(

View File

@ -38,7 +38,7 @@ sorted_reboot_apps_cycle_test() ->
check_order(Apps) -> check_order(Apps) ->
AllApps = lists:usort(lists:append([[A | Deps] || {A, Deps} <- Apps])), AllApps = lists:usort(lists:append([[A | Deps] || {A, Deps} <- Apps])),
Sorted = emqx_machine_boot:sorted_reboot_apps(Apps), [emqx_conf | Sorted] = emqx_machine_boot:sorted_reboot_apps(Apps),
case length(AllApps) =:= length(Sorted) of case length(AllApps) =:= length(Sorted) of
true -> ok; true -> ok;
false -> error({AllApps, Sorted}) false -> error({AllApps, Sorted})

View File

@ -30,6 +30,7 @@
-export([ node_query/5 -export([ node_query/5
, cluster_query/4 , cluster_query/4
, select_table_with_count/5 , select_table_with_count/5
, b2i/1
]). ]).
-export([do_query/6]). -export([do_query/6]).

View File

@ -91,16 +91,17 @@ fields(app) ->
"""They are useful for accessing public data anonymously,""" """They are useful for accessing public data anonymously,"""
"""and are used to associate API requests.""", """and are used to associate API requests.""",
example => <<"MzAyMjk3ODMwMDk0NjIzOTUxNjcwNzQ0NzQ3MTE2NDYyMDI">>})}, example => <<"MzAyMjk3ODMwMDk0NjIzOTUxNjcwNzQ0NzQ3MTE2NDYyMDI">>})},
{expired_at, hoconsc:mk(emqx_schema:rfc3339_system_time(), {expired_at, hoconsc:mk(hoconsc:union([undefined, emqx_schema:rfc3339_system_time()]),
#{desc => "No longer valid datetime", #{desc => "No longer valid datetime",
example => <<"2021-12-05T02:01:34.186Z">>, example => <<"2021-12-05T02:01:34.186Z">>,
nullable => true nullable => true,
default => undefined
})}, })},
{created_at, hoconsc:mk(emqx_schema:rfc3339_system_time(), {created_at, hoconsc:mk(emqx_schema:rfc3339_system_time(),
#{desc => "ApiKey create datetime", #{desc => "ApiKey create datetime",
example => <<"2021-12-01T00:00:00.000Z">> example => <<"2021-12-01T00:00:00.000Z">>
})}, })},
{desc, hoconsc:mk(emqx_schema:unicode_binary(), {desc, hoconsc:mk(binary(),
#{example => <<"Note">>, nullable => true})}, #{example => <<"Note">>, nullable => true})},
{enable, hoconsc:mk(boolean(), #{desc => "Enable/Disable", nullable => true})} {enable, hoconsc:mk(boolean(), #{desc => "Enable/Disable", nullable => true})}
]; ];
@ -136,13 +137,19 @@ api_key(post, #{body := App}) ->
#{ #{
<<"name">> := Name, <<"name">> := Name,
<<"desc">> := Desc0, <<"desc">> := Desc0,
<<"expired_at">> := ExpiredAt,
<<"enable">> := Enable <<"enable">> := Enable
} = App, } = App,
%% undefined is never expired
ExpiredAt0 = maps:get(<<"expired_at">>, App, <<"undefined">>),
ExpiredAt =
case ExpiredAt0 of
<<"undefined">> -> undefined;
_ -> ExpiredAt0
end,
Desc = unicode:characters_to_binary(Desc0, unicode), Desc = unicode:characters_to_binary(Desc0, unicode),
case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of
{ok, NewApp} -> {200, format(NewApp)}; {ok, NewApp} -> {200, format(NewApp)};
{error, Reason} -> {400, Reason} {error, Reason} -> {400, io_lib:format("~p", [Reason])}
end. end.
api_key_by_name(get, #{bindings := #{name := Name}}) -> api_key_by_name(get, #{bindings := #{name := Name}}) ->
@ -164,8 +171,13 @@ api_key_by_name(put, #{bindings := #{name := Name}, body := Body}) ->
{error, not_found} -> {404, <<"NOT_FOUND">>} {error, not_found} -> {404, <<"NOT_FOUND">>}
end. end.
format(App = #{expired_at := ExpiredAt, created_at := CreateAt}) -> format(App = #{expired_at := ExpiredAt0, created_at := CreateAt}) ->
ExpiredAt =
case ExpiredAt0 of
undefined -> <<"undefined">>;
_ -> list_to_binary(calendar:system_time_to_rfc3339(ExpiredAt0))
end,
App#{ App#{
expired_at => list_to_binary(calendar:system_time_to_rfc3339(ExpiredAt)), expired_at => ExpiredAt,
created_at => list_to_binary(calendar:system_time_to_rfc3339(CreateAt)) created_at => list_to_binary(calendar:system_time_to_rfc3339(CreateAt))
}. }.

View File

@ -101,7 +101,7 @@ fields(ban) ->
desc => <<"Banned type clientid, username, peerhost">>, desc => <<"Banned type clientid, username, peerhost">>,
nullable => false, nullable => false,
example => username})}, example => username})},
{who, hoconsc:mk(emqx_schema:unicode_binary(), #{ {who, hoconsc:mk(binary(), #{
desc => <<"Client info as banned type">>, desc => <<"Client info as banned type">>,
nullable => false, nullable => false,
example => <<"Badass坏"/utf8>>})}, example => <<"Badass坏"/utf8>>})},
@ -109,19 +109,17 @@ fields(ban) ->
desc => <<"Commander">>, desc => <<"Commander">>,
nullable => true, nullable => true,
example => <<"mgmt_api">>})}, example => <<"mgmt_api">>})},
{reason, hoconsc:mk(emqx_schema:unicode_binary(), #{ {reason, hoconsc:mk(binary(), #{
desc => <<"Banned reason">>, desc => <<"Banned reason">>,
nullable => true, nullable => true,
example => <<"Too many requests">>})}, example => <<"Too many requests">>})},
{at, hoconsc:mk(binary(), #{ {at, hoconsc:mk(emqx_schema:rfc3339_system_time(), #{
desc => <<"Create banned time, rfc3339, now if not specified">>, desc => <<"Create banned time, rfc3339, now if not specified">>,
nullable => true, nullable => true,
validator => fun is_rfc3339/1,
example => <<"2021-10-25T21:48:47+08:00">>})}, example => <<"2021-10-25T21:48:47+08:00">>})},
{until, hoconsc:mk(binary(), #{ {until, hoconsc:mk(emqx_schema:rfc3339_system_time(), #{
desc => <<"Cancel banned time, rfc3339, now + 5 minute if not specified">>, desc => <<"Cancel banned time, rfc3339, now + 5 minute if not specified">>,
nullable => true, nullable => true,
validator => fun is_rfc3339/1,
example => <<"2021-10-25T21:53:47+08:00">>}) example => <<"2021-10-25T21:53:47+08:00">>})
} }
]; ];
@ -130,22 +128,19 @@ fields(meta) ->
emqx_dashboard_swagger:fields(limit) ++ emqx_dashboard_swagger:fields(limit) ++
[{count, hoconsc:mk(integer(), #{example => 1})}]. [{count, hoconsc:mk(integer(), #{example => 1})}].
is_rfc3339(Time) ->
try
emqx_banned:to_timestamp(Time),
ok
catch _:_ -> {error, Time}
end.
banned(get, #{query_string := Params}) -> banned(get, #{query_string := Params}) ->
Response = emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN), Response = emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN),
{200, Response}; {200, Response};
banned(post, #{body := Body}) -> banned(post, #{body := Body}) ->
case emqx_banned:create(emqx_banned:parse(Body)) of case emqx_banned:parse(Body) of
{ok, Banned} -> {error, Reason} ->
{200, format(Banned)}; {400, #{code => 'PARAMS_ERROR', message => list_to_binary(Reason)}};
Ban ->
case emqx_banned:create(Ban) of
{ok, Banned} -> {200, format(Banned)};
{error, {already_exist, Old}} -> {error, {already_exist, Old}} ->
{400, #{code => 'ALREADY_EXISTED', message => format(Old)}} {400, #{code => 'ALREADY_EXISTED', message => format(Old)}}
end
end. end.
delete_banned(delete, #{bindings := Params}) -> delete_banned(delete, #{bindings := Params}) ->

View File

@ -107,9 +107,14 @@ schema("/trace/:name/download") ->
get => #{ get => #{
description => "Download trace log by name", description => "Download trace log by name",
parameters => [hoconsc:ref(name)], parameters => [hoconsc:ref(name)],
%% todo zip file octet-stream
responses => #{ responses => #{
200 => <<"TODO octet-stream">> 200 =>
#{description => "A trace zip file",
content => #{
'application/octet-stream' =>
#{schema => #{type => "string", format => "binary"}}
}
}
} }
} }
}; };
@ -124,9 +129,12 @@ schema("/trace/:name/log") ->
hoconsc:ref(position), hoconsc:ref(position),
hoconsc:ref(node) hoconsc:ref(node)
], ],
%% todo response data
responses => #{ responses => #{
200 => <<"TODO">> 200 =>
[
{items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}
| fields(bytes) ++ fields(position)
]
} }
} }
}. }.
@ -209,6 +217,7 @@ fields(position) ->
default => 0 default => 0
})}]. })}].
-define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$"). -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$").
validate_name(Name) -> validate_name(Name) ->
@ -296,7 +305,12 @@ download_trace_log(get, #{bindings := #{name := Name}}) ->
ZipFileName = ZipDir ++ binary_to_list(Name) ++ ".zip", ZipFileName = ZipDir ++ binary_to_list(Name) ++ ".zip",
{ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]), {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]),
emqx_trace:delete_files_after_send(ZipFileName, Zips), emqx_trace:delete_files_after_send(ZipFileName, Zips),
{200, ZipFile}; Headers = #{
<<"content-type">> => <<"application/x-zip">>,
<<"content-disposition">> =>
iolist_to_binary("attachment; filename=" ++ filename:basename(ZipFile))
},
{200, Headers, {file, ZipFile}};
{error, not_found} -> ?NOT_FOUND(Name) {error, not_found} -> ?NOT_FOUND(Name)
end. end.
@ -324,11 +338,10 @@ cluster_call(Mod, Fun, Args, Timeout) ->
BadNodes =/= [] andalso ?LOG(error, "rpc call failed on ~p ~p", [BadNodes, {Mod, Fun, Args}]), BadNodes =/= [] andalso ?LOG(error, "rpc call failed on ~p ~p", [BadNodes, {Mod, Fun, Args}]),
GoodRes. GoodRes.
stream_log_file(get, #{bindings := #{name := Name}, query_string := Query} = T) -> stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) ->
Node0 = maps:get(<<"node">>, Query, atom_to_binary(node())), Node0 = maps:get(<<"node">>, Query, atom_to_binary(node())),
Position = maps:get(<<"position">>, Query, 0), Position = maps:get(<<"position">>, Query, 0),
Bytes = maps:get(<<"bytes">>, Query, 1000), Bytes = maps:get(<<"bytes">>, Query, 1000),
logger:error("~p", [T]),
case to_node(Node0) of case to_node(Node0) of
{ok, Node} -> {ok, Node} ->
case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of

View File

@ -37,7 +37,7 @@
api_secret_hash = <<>> :: binary() | '_', api_secret_hash = <<>> :: binary() | '_',
enable = true :: boolean() | '_', enable = true :: boolean() | '_',
desc = <<>> :: binary() | '_', desc = <<>> :: binary() | '_',
expired_at = 0 :: integer() | '_', expired_at = 0 :: integer() | undefined | '_',
created_at = 0 :: integer() | '_' created_at = 0 :: integer() | '_'
}). }).

View File

@ -18,6 +18,7 @@
-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/logger.hrl").
-include("emqx_mgmt.hrl"). -include("emqx_mgmt.hrl").
@ -386,18 +387,20 @@ trace(["list"]) ->
emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst]) emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst])
end, emqx_trace_handler:running()); end, emqx_trace_handler:running());
trace(["stop", Operation, ClientId]) -> trace(["stop", Operation, Filter0]) ->
case trace_type(Operation) of case trace_type(Operation, Filter0) of
{ok, Type} -> trace_off(Type, ClientId); {ok, Type, Filter} -> trace_off(Type, Filter);
error -> trace([]) error -> trace([])
end; end;
trace(["start", Operation, ClientId, LogFile]) -> trace(["start", Operation, ClientId, LogFile]) ->
trace(["start", Operation, ClientId, LogFile, "all"]); trace(["start", Operation, ClientId, LogFile, "all"]);
trace(["start", Operation, ClientId, LogFile, Level]) -> trace(["start", Operation, Filter0, LogFile, Level]) ->
case trace_type(Operation) of case trace_type(Operation, Filter0) of
{ok, Type} -> trace_on(Type, ClientId, list_to_existing_atom(Level), LogFile); {ok, Type, Filter} ->
trace_on(name(Filter0), Type, Filter,
list_to_existing_atom(Level), LogFile);
error -> trace([]) error -> trace([])
end; end;
@ -417,20 +420,23 @@ trace(_) ->
"Stop tracing for a client ip on local node"} "Stop tracing for a client ip on local node"}
]). ]).
trace_on(Who, Name, Level, LogFile) -> trace_on(Name, Type, Filter, Level, LogFile) ->
case emqx_trace_handler:install(Who, Name, Level, LogFile) of case emqx_trace_handler:install(Name, Type, Filter, Level, LogFile) of
ok -> ok ->
emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]); emqx_trace:check(),
emqx_ctl:print("trace ~s ~s successfully~n", [Filter, Name]);
{error, Error} -> {error, Error} ->
emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Who, Name, Error]) emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Filter, Name, Error])
end. end.
trace_off(Who, Name) -> trace_off(Type, Filter) ->
case emqx_trace_handler:uninstall(Who, Name) of ?TRACE("CLI", "trace_stopping", #{Type => Filter}),
case emqx_trace_handler:uninstall(Type, name(Filter)) of
ok -> ok ->
emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]); emqx_trace:check(),
emqx_ctl:print("stop tracing ~s ~s successfully~n", [Type, Filter]);
{error, Error} -> {error, Error} ->
emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Name, Error]) emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Type, Filter, Error])
end. end.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
@ -459,9 +465,9 @@ traces(["delete", Name]) ->
traces(["start", Name, Operation, Filter]) -> traces(["start", Name, Operation, Filter]) ->
traces(["start", Name, Operation, Filter, "900"]); traces(["start", Name, Operation, Filter, "900"]);
traces(["start", Name, Operation, Filter, DurationS]) -> traces(["start", Name, Operation, Filter0, DurationS]) ->
case trace_type(Operation) of case trace_type(Operation, Filter0) of
{ok, Type} -> trace_cluster_on(Name, Type, Filter, DurationS); {ok, Type, Filter} -> trace_cluster_on(Name, Type, Filter, DurationS);
error -> traces([]) error -> traces([])
end; end;
@ -503,10 +509,10 @@ trace_cluster_off(Name) ->
{error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error]) {error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error])
end. end.
trace_type("client") -> {ok, clientid}; trace_type("client", ClientId) -> {ok, clientid, list_to_binary(ClientId)};
trace_type("topic") -> {ok, topic}; trace_type("topic", Topic) -> {ok, topic, list_to_binary(Topic)};
trace_type("ip_address") -> {ok, ip_address}; trace_type("ip_address", IP) -> {ok, ip_address, IP};
trace_type(_) -> error. trace_type(_, _) -> error.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% @doc Listeners Command %% @doc Listeners Command
@ -716,3 +722,6 @@ format_listen_on({Addr, Port}) when is_list(Addr) ->
io_lib:format("~ts:~w", [Addr, Port]); io_lib:format("~ts:~w", [Addr, Port]);
format_listen_on({Addr, Port}) when is_tuple(Addr) -> format_listen_on({Addr, Port}) when is_tuple(Addr) ->
io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]). io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]).
name(Filter) ->
iolist_to_binary(["CLI-", Filter]).

View File

@ -23,7 +23,7 @@
all() -> [{group, parallel}, {group, sequence}]. all() -> [{group, parallel}, {group, sequence}].
suite() -> [{timetrap, {minutes, 1}}]. suite() -> [{timetrap, {minutes, 1}}].
groups() -> [ groups() -> [
{parallel, [parallel], [t_create, t_update, t_delete, t_authorize]}, {parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]},
{sequence, [], [t_create_failed]} {sequence, [], [t_create_failed]}
]. ].
@ -137,7 +137,15 @@ t_authorize(_Config) ->
}, },
?assertMatch({ok, #{<<"api_key">> := _, <<"enable">> := true}}, update_app(Name, Expired)), ?assertMatch({ok, #{<<"api_key">> := _, <<"enable">> := true}}, update_app(Name, Expired)),
?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)), ?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)),
ok.
t_create_unexpired_app(_Config) ->
Name1 = <<"EMQX-UNEXPIRED-API-KEY-1">>,
Name2 = <<"EMQX-UNEXPIRED-API-KEY-2">>,
{ok, Create1} = create_unexpired_app(Name1, #{}),
?assertMatch(#{<<"expired_at">> := <<"undefined">>}, Create1),
{ok, Create2} = create_unexpired_app(Name2, #{expired_at => <<"undefined">>}),
?assertMatch(#{<<"expired_at">> := <<"undefined">>}, Create2),
ok. ok.
@ -170,6 +178,15 @@ create_app(Name) ->
Error -> Error Error -> Error
end. end.
create_unexpired_app(Name, Params) ->
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params),
case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of
{ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
Error -> Error
end.
delete_app(Name) -> delete_app(Name) ->
DeletePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]), DeletePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]),
emqx_mgmt_api_test_util:request_api(delete, DeletePath). emqx_mgmt_api_test_util:request_api(delete, DeletePath).

View File

@ -0,0 +1,144 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_mgmt_banned_api_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_mgmt_api_test_util:init_suite(),
Config.
end_per_suite(_) ->
emqx_mgmt_api_test_util:end_suite().
t_create(_Config) ->
Now = erlang:system_time(second),
At = emqx_banned:to_rfc3339(Now),
Until = emqx_banned:to_rfc3339(Now + 3),
ClientId = <<"TestClient测试"/utf8>>,
By = <<"banned suite测试组"/utf8>>,
Reason = <<"test测试"/utf8>>,
As = <<"clientid">>,
ClientIdBanned = #{
as => As,
who => ClientId,
by => By,
reason => Reason,
at => At,
until => Until
},
{ok, ClientIdBannedRes} = create_banned(ClientIdBanned),
?assertEqual(#{<<"as">> => As,
<<"at">> => At,
<<"by">> => By,
<<"reason">> => Reason,
<<"until">> => Until,
<<"who">> => ClientId
}, ClientIdBannedRes),
PeerHost = <<"192.168.2.13">>,
PeerHostBanned = #{
as => <<"peerhost">>,
who => PeerHost,
by => By,
reason => Reason,
at => At,
until => Until
},
{ok, PeerHostBannedRes} = create_banned(PeerHostBanned),
?assertEqual(#{<<"as">> => <<"peerhost">>,
<<"at">> => At,
<<"by">> => By,
<<"reason">> => Reason,
<<"until">> => Until,
<<"who">> => PeerHost
}, PeerHostBannedRes),
{ok, #{<<"data">> := List}} = list_banned(),
Bans = lists:sort(lists:map(fun(#{<<"who">> := W, <<"as">> := A}) -> {A, W} end, List)),
?assertEqual([{<<"clientid">>, ClientId}, {<<"peerhost">>, PeerHost}], Bans),
ok.
t_create_failed(_Config) ->
Now = erlang:system_time(second),
At = emqx_banned:to_rfc3339(Now),
Until = emqx_banned:to_rfc3339(Now + 10),
Who = <<"BadHost"/utf8>>,
By = <<"banned suite测试组"/utf8>>,
Reason = <<"test测试"/utf8>>,
As = <<"peerhost">>,
BadPeerHost = #{
as => As,
who => Who,
by => By,
reason => Reason,
at => At,
until => Until
},
BadRequest = {error, {"HTTP/1.1", 400, "Bad Request"}},
?assertEqual(BadRequest, create_banned(BadPeerHost)),
Expired = BadPeerHost#{until => emqx_banned:to_rfc3339(Now - 1),
who => <<"127.0.0.1">>},
?assertEqual(BadRequest, create_banned(Expired)),
ok.
t_delete(_Config) ->
Now = erlang:system_time(second),
At = emqx_banned:to_rfc3339(Now),
Until = emqx_banned:to_rfc3339(Now + 3),
Who = <<"TestClient-"/utf8>>,
By = <<"banned suite 中"/utf8>>,
Reason = <<"test测试"/utf8>>,
As = <<"clientid">>,
Banned = #{
as => clientid,
who => Who,
by => By,
reason => Reason,
at => At,
until => Until
},
{ok, _} = create_banned(Banned),
?assertMatch({ok, _}, delete_banned(binary_to_list(As), binary_to_list(Who))),
?assertMatch({error,{"HTTP/1.1",404,"Not Found"}},
delete_banned(binary_to_list(As), binary_to_list(Who))),
ok.
list_banned() ->
Path = emqx_mgmt_api_test_util:api_path(["banned"]),
case emqx_mgmt_api_test_util:request_api(get, Path) of
{ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])};
Error -> Error
end.
create_banned(Banned) ->
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["banned"]),
case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Banned) of
{ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
Error -> Error
end.
delete_banned(As, Who) ->
DeletePath = emqx_mgmt_api_test_util:api_path(["banned", As, Who]),
emqx_mgmt_api_test_util:request_api(delete, DeletePath).
to_rfc3339(Sec) ->
list_to_binary(calendar:system_time_to_rfc3339(Sec)).

Some files were not shown because too many files have changed in this diff Show More