yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/release-5.0-beta.3' into merge-5.0-beta.3-to-master

This commit is contained in:
Zaiming (Stone) Shi 2022-01-03 11:39:06 +01:00
parent 852033a955 2259f3ba64
commit 2898fa76e1
131 changed files with 3034 additions and 1682 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/run_api_tests.yaml vendored
View File

@ -61,7 +61,7 @@ jobs:
- uses: actions/checkout@v2
with:
repository: emqx/emqx-fvt
ref: 1.0.2-dev1
ref: 1.0.3-dev1
path: .
- uses: actions/setup-java@v1
with:
@ -93,7 +93,7 @@ jobs:
run: |
/opt/jmeter/bin/jmeter.sh \
-Jjmeter.save.saveservice.output_format=xml -n \
-t .ci/api-test-suite/${{ matrix.script_name }}.jmx \
-t api-test-suite/${{ matrix.script_name }}.jmx \
-Demqx_ip="127.0.0.1" \
-l jmeter_logs/${{ matrix.script_name }}.jtl \
-j jmeter_logs/logs/${{ matrix.script_name }}.log

2
Makefile
View File

@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3
export EMQX_DEFAULT_RUNNER = alpine:3.14
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v0.10.0
export EMQX_DASHBOARD_VERSION ?= v0.14.0
export DOCKERFILE := deploy/docker/Dockerfile
export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
ifeq ($(OS),Windows_NT)

35
apps/emqx/include/logger.hrl
View File

@ -59,15 +59,32 @@
%% structured logging
-define(SLOG(Level, Data),
%% check 'allow' here, only evaluate Data when necessary
case logger:allow(Level, ?MODULE) of
true ->
logger:log(Level, (Data), #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
, line => ?LINE
});
false ->
ok
end).
?SLOG(Level, Data, #{})).
%% structured logging, meta is for handler's filter.
-define(SLOG(Level, Data, Meta),
%% check 'allow' here, only evaluate Data and Meta when necessary
case logger:allow(Level, ?MODULE) of
true ->
logger:log(Level, (Data), (Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
, line => ?LINE
}));
false ->
ok
end).
-define(TRACE_FILTER, emqx_trace_filter).
%% Only evaluate when necessary
-define(TRACE(Event, Msg, Meta),
begin
case persistent_term:get(?TRACE_FILTER, undefined) of
undefined -> ok;
[] -> ok;
List ->
emqx_trace:log(List, Event, Msg, Meta)
end
end).
%% print to 'user' group leader
-define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).

2
apps/emqx/rebar.config
View File

@ -11,7 +11,7 @@
{deps,
[ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.2"}}}
, {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
, {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}}
, {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.6"}}}
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}

4
apps/emqx/src/emqx_authentication_config.erl
View File

@ -187,7 +187,7 @@ convert_certs(CertsDir, Config) ->
{ok, SSL} ->
new_ssl_config(Config, SSL);
{error, Reason} ->
?SLOG(error, Reason#{msg => bad_ssl_config}),
?SLOG(error, Reason#{msg => "bad_ssl_config"}),
throw({bad_ssl_config, Reason})
end.
@ -199,7 +199,7 @@ convert_certs(CertsDir, NewConfig, OldConfig) ->
ok = emqx_tls_lib:delete_ssl_files(CertsDir, NewSSL1, OldSSL),
new_ssl_config(NewConfig, NewSSL1);
{error, Reason} ->
?SLOG(error, Reason#{msg => bad_ssl_config}),
?SLOG(error, Reason#{msg => "bad_ssl_config"}),
throw({bad_ssl_config, Reason})
end.

64
apps/emqx/src/emqx_banned.erl
View File

@ -37,7 +37,6 @@
, info/1
, format/1
, parse/1
, to_timestamp/1
]).
%% gen_server callbacks
@ -53,6 +52,11 @@
-define(BANNED_TAB, ?MODULE).
-ifdef(TEST).
-compile(export_all).
-compile(nowarn_export_all).
-endif.
%%--------------------------------------------------------------------
%% Mnesia bootstrap
%%--------------------------------------------------------------------
@ -106,32 +110,36 @@ format(#banned{who = Who0,
}.
parse(Params) ->
Who = pares_who(Params),
By = maps:get(<<"by">>, Params, <<"mgmt_api">>),
Reason = maps:get(<<"reason">>, Params, <<"">>),
At = parse_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)),
Until = parse_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60),
#banned{
who = Who,
by = By,
reason = Reason,
at = At,
until = Until
}.
case pares_who(Params) of
{error, Reason} -> {error, Reason};
Who ->
By = maps:get(<<"by">>, Params, <<"mgmt_api">>),
Reason = maps:get(<<"reason">>, Params, <<"">>),
At = maps:get(<<"at">>, Params, erlang:system_time(second)),
Until = maps:get(<<"until">>, Params, At + 5 * 60),
case Until > erlang:system_time(second) of
true ->
#banned{
who = Who,
by = By,
reason = Reason,
at = At,
until = Until
};
false ->
{error, "already_expired"}
end
end.
pares_who(#{as := As, who := Who}) ->
pares_who(#{<<"as">> => As, <<"who">> => Who});
pares_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) ->
{ok, Peerhost} = inet:parse_address(binary_to_list(Peerhost0)),
{peerhost, Peerhost};
case inet:parse_address(binary_to_list(Peerhost0)) of
{ok, Peerhost} -> {peerhost, Peerhost};
{error, einval} -> {error, "bad peerhost"}
end;
pares_who(#{<<"as">> := As, <<"who">> := Who}) ->
{As, Who}.
parse_time(undefined, Default) ->
Default;
parse_time(Rfc3339, _Default) ->
to_timestamp(Rfc3339).
maybe_format_host({peerhost, Host}) ->
AddrBinary = list_to_binary(inet:ntoa(Host)),
{peerhost, AddrBinary};
@ -141,11 +149,6 @@ maybe_format_host({As, Who}) ->
to_rfc3339(Timestamp) ->
list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])).
to_timestamp(Rfc3339) when is_binary(Rfc3339) ->
to_timestamp(binary_to_list(Rfc3339));
to_timestamp(Rfc3339) ->
calendar:rfc3339_to_system_time(Rfc3339, [{unit, second}]).
-spec(create(emqx_types:banned() | map()) ->
{ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}).
create(#{who := Who,
@ -168,10 +171,11 @@ create(Banned = #banned{who = Who}) ->
mria:dirty_write(?BANNED_TAB, Banned),
{ok, Banned};
[OldBanned = #banned{until = Until}] ->
case Until < erlang:system_time(second) of
true ->
{error, {already_exist, OldBanned}};
false ->
%% Don't support shorten or extend the until time by overwrite.
%% We don't support update api yet, user must delete then create new one.
case Until > erlang:system_time(second) of
true -> {error, {already_exist, OldBanned}};
false -> %% overwrite expired one is ok.
mria:dirty_write(?BANNED_TAB, Banned),
{ok, Banned}
end

14
apps/emqx/src/emqx_broker.erl
View File

@ -204,9 +204,9 @@ publish(Msg) when is_record(Msg, message) ->
_ = emqx_trace:publish(Msg),
emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'),
case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of
#message{headers = #{allow_publish := false}} ->
?SLOG(debug, #{msg => "message_not_published",
payload => emqx_message:to_log_map(Msg)}),
#message{headers = #{allow_publish := false}, topic = Topic} ->
?TRACE("MQTT", "msg_publish_not_allowed", #{message => emqx_message:to_log_map(Msg),
topic => Topic}),
[];
Msg1 = #message{topic = Topic} ->
emqx_persistent_session:persist_message(Msg1),
@ -226,7 +226,9 @@ safe_publish(Msg) when is_record(Msg, message) ->
reason => Reason,
payload => emqx_message:to_log_map(Msg),
stacktrace => Stk
}),
},
#{topic => Msg#message.topic}
),
[]
end.
@ -280,7 +282,7 @@ forward(Node, To, Delivery, async) ->
msg => "async_forward_msg_to_node_failed",
node => Node,
reason => Reason
}),
}, #{topic => To}),
{error, badrpc}
end;
@ -291,7 +293,7 @@ forward(Node, To, Delivery, sync) ->
msg => "sync_forward_msg_to_node_failed",
node => Node,
reason => Reason
}),
}, #{topic => To}),
{error, badrpc};
Result ->
emqx_metrics:inc('messages.forward'), Result

16
apps/emqx/src/emqx_channel.erl
View File

@ -292,7 +292,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
fun check_banned/2
], ConnPkt, Channel#channel{conn_state = connecting}) of
{ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
?SLOG(debug, #{msg => "recv_packet", packet => emqx_packet:format(Packet)}),
?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
NChannel1 = NChannel#channel{
will_msg = emqx_packet:will_msg(NConnPkt),
alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
@ -550,9 +550,8 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
{error, Rc = ?RC_NOT_AUTHORIZED, NChannel} ->
?SLOG(warning, #{
msg => "cannot_publish_to_topic",
topic => Topic,
reason => emqx_reason_codes:name(Rc)
}),
}, #{topic => Topic}),
case emqx:get_config([authorization, deny_action], ignore) of
ignore ->
case QoS of
@ -568,9 +567,8 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
{error, Rc = ?RC_QUOTA_EXCEEDED, NChannel} ->
?SLOG(warning, #{
msg => "cannot_publish_to_topic",
topic => Topic,
reason => emqx_reason_codes:name(Rc)
}),
}, #{topic => Topic}),
case QoS of
?QOS_0 ->
ok = emqx_metrics:inc('packets.publish.dropped'),
@ -585,7 +583,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
msg => "cannot_publish_to_topic",
topic => Topic,
reason => emqx_reason_codes:name(Rc)
}),
}, #{topic => Topic}),
handle_out(disconnect, Rc, NChannel)
end.
@ -635,7 +633,7 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2},
msg => "dropped_qos2_packet",
reason => emqx_reason_codes:name(RC),
packet_id => PacketId
}),
}, #{topic => Msg#message.topic}),
ok = emqx_metrics:inc('packets.publish.dropped'),
handle_out(disconnect, RC, Channel)
end.
@ -687,7 +685,7 @@ process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Ac
?SLOG(warning, #{
msg => "cannot_subscribe_topic_filter",
reason => emqx_reason_codes:name(ReasonCode)
}),
}, #{topic => TopicFilter}),
process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc])
end.
@ -703,7 +701,7 @@ do_subscribe(TopicFilter, SubOpts = #{qos := QoS}, Channel =
?SLOG(warning, #{
msg => "cannot_subscribe_topic_filter",
reason => emqx_reason_codes:text(RC)
}),
}, #{topic => NTopicFilter}),
{RC, Channel}
end.

31
apps/emqx/src/emqx_cm.erl
View File

@ -375,7 +375,7 @@ discard_session(ClientId) when is_binary(ClientId) ->
-spec kick_or_kill(kick | discard, module(), pid()) -> ok.
kick_or_kill(Action, ConnMod, Pid) ->
try
%% this is essentailly a gen_server:call implemented in emqx_connection
%% this is essentially a gen_server:call implemented in emqx_connection
%% and emqx_ws_connection.
%% the handle_call is implemented in emqx_channel
ok = apply(ConnMod, call, [Pid, Action, ?T_KICK])
@ -390,19 +390,12 @@ kick_or_kill(Action, ConnMod, Pid) ->
ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action});
_ : {timeout, {gen_server, call, _}} ->
?tp(warning, "session_kick_timeout",
#{pid => Pid,
action => Action,
stale_channel => stale_channel_info(Pid)
}),
#{pid => Pid, action => Action, stale_channel => stale_channel_info(Pid)}),
ok = force_kill(Pid);
_ : Error : St ->
?tp(error, "session_kick_exception",
#{pid => Pid,
action => Action,
reason => Error,
stacktrace => St,
stale_channel => stale_channel_info(Pid)
}),
#{pid => Pid, action => Action, reason => Error, stacktrace => St,
stale_channel => stale_channel_info(Pid)}),
ok = force_kill(Pid)
end.
@ -448,20 +441,22 @@ kick_session(Action, ClientId, ChanPid) ->
, action => Action
, error => Error
, reason => Reason
})
},
#{clientid => ClientId})
end.
kick_session(ClientId) ->
case lookup_channels(ClientId) of
[] ->
?SLOG(warning, #{msg => "kicked_an_unknown_session",
clientid => ClientId}),
?SLOG(warning, #{msg => "kicked_an_unknown_session"},
#{clientid => ClientId}),
ok;
ChanPids ->
case length(ChanPids) > 1 of
true ->
?SLOG(warning, #{msg => "more_than_one_channel_found",
chan_pids => ChanPids});
chan_pids => ChanPids},
#{clientid => ClientId});
false -> ok
end,
lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids)
@ -478,12 +473,12 @@ with_channel(ClientId, Fun) ->
Pids -> Fun(lists:last(Pids))
end.
%% @doc Get all registed channel pids. Debugg/test interface
%% @doc Get all registered channel pids. Debug/test interface
all_channels() ->
Pat = [{{'_', '$1'}, [], ['$1']}],
ets:select(?CHAN_TAB, Pat).
%% @doc Get all registed clientIDs. Debugg/test interface
%% @doc Get all registered clientIDs. Debug/test interface
all_client_ids() ->
Pat = [{{'$1', '_'}, [], ['$1']}],
ets:select(?CHAN_TAB, Pat).
@ -511,7 +506,7 @@ lookup_channels(local, ClientId) ->
rpc_call(Node, Fun, Args, Timeout) ->
case rpc:call(Node, ?MODULE, Fun, Args, 2 * Timeout) of
{badrpc, Reason} ->
%% since eqmx app 4.3.10, the 'kick' and 'discard' calls hanndler
%% since emqx app 4.3.10, the 'kick' and 'discard' calls handler
%% should catch all exceptions and always return 'ok'.
%% This leaves 'badrpc' only possible when there is problem
%% calling the remote node.

5
apps/emqx/src/emqx_config.erl
View File

@ -262,8 +262,9 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) ->
{ok, RawRichConf} ->
init_load(SchemaMod, RawRichConf);
{error, Reason} ->
?SLOG(error, #{msg => failed_to_load_hocon_conf,
?SLOG(error, #{msg => "failed_to_load_hocon_conf",
reason => Reason,
pwd => file:get_cwd(),
include_dirs => IncDir
}),
error(failed_to_load_hocon_conf)
@ -396,7 +397,7 @@ save_to_override_conf(RawConf, Opts) ->
case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
ok -> ok;
{error, Reason} ->
?SLOG(error, #{msg => failed_to_write_override_file,
?SLOG(error, #{msg => "failed_to_write_override_file",
filename => FileName,
reason => Reason}),
{error, Reason}

21
apps/emqx/src/emqx_connection.erl
View File

@ -449,14 +449,12 @@ handle_msg({'$gen_cast', Req}, State) ->
{ok, NewState};
handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl ->
?SLOG(debug, #{msg => "RECV_data", data => Data, transport => Inet}),
Oct = iolist_size(Data),
inc_counter(incoming_bytes, Oct),
ok = emqx_metrics:inc('bytes.received', Oct),
when_bytes_in(Oct, Data, State);
handle_msg({quic, Data, _Sock, _, _, _}, State) ->
?SLOG(debug, #{msg => "RECV_data", data => Data, transport => quic}),
Oct = iolist_size(Data),
inc_counter(incoming_bytes, Oct),
ok = emqx_metrics:inc('bytes.received', Oct),
@ -528,7 +526,7 @@ handle_msg({connack, ConnAck}, State) ->
handle_outgoing(ConnAck, State);
handle_msg({close, Reason}, State) ->
?SLOG(debug, #{msg => "force_socket_close", reason => Reason}),
?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}),
handle_info({sock_closed, Reason}, close_socket(State));
handle_msg({event, connected}, State = #state{channel = Channel}) ->
@ -566,7 +564,8 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport,
Channel1 = emqx_channel:set_conn_state(disconnected, Channel),
emqx_congestion:cancel_alarms(Socket, Transport, Channel1),
emqx_channel:terminate(Reason, Channel1),
close_socket_ok(State)
close_socket_ok(State),
?TRACE("SOCKET", "tcp_socket_terminated", #{reason => Reason})
catch
E : C : S ->
?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S})
@ -716,7 +715,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) ->
ok = inc_incoming_stats(Packet),
?SLOG(debug, #{msg => "RECV_packet", packet => emqx_packet:format(Packet)}),
?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
with_channel(handle_in, [Packet], State);
handle_incoming(FrameError, State) ->
@ -755,15 +754,13 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
<<>> -> ?SLOG(warning, #{
msg => "packet_is_discarded",
reason => "frame_is_too_large",
packet => emqx_packet:format(Packet)
packet => emqx_packet:format(Packet, hidden)
}),
ok = emqx_metrics:inc('delivery.dropped.too_large'),
ok = emqx_metrics:inc('delivery.dropped'),
<<>>;
Data -> ?SLOG(debug, #{
msg => "SEND_packet",
packet => emqx_packet:format(Packet)
}),
Data ->
?TRACE("MQTT", "mqtt_packet_sent", #{packet => Packet}),
ok = inc_outgoing_stats(Packet),
Data
catch
@ -875,7 +872,7 @@ check_limiter(Needs,
{ok, Limiter2} ->
WhenOk(Data, Msgs, State#state{limiter = Limiter2});
{pause, Time, Limiter2} ->
?SLOG(warning, #{msg => "pause time dueto rate limit",
?SLOG(warning, #{msg => "pause_time_dueto_rate_limit",
needs => Needs,
time_in_ms => Time}),
@ -915,7 +912,7 @@ retry_limiter(#state{limiter = Limiter} = State) ->
, limiter_timer = undefined
});
{pause, Time, Limiter2} ->
?SLOG(warning, #{msg => "pause time dueto rate limit",
?SLOG(warning, #{msg => "pause_time_dueto_rate_limit",
types => Types,
time_in_ms => Time}),

6
apps/emqx/src/emqx_flapping.erl
View File

@ -118,11 +118,10 @@ handle_cast({detected, #flapping{clientid = ClientId,
true -> %% Flapping happened:(
?SLOG(warning, #{
msg => "flapping_detected",
client_id => ClientId,
peer_host => fmt_host(PeerHost),
detect_cnt => DetectCnt,
wind_time_in_ms => WindTime
}),
}, #{clientid => ClientId}),
Now = erlang:system_time(second),
Banned = #banned{who = {clientid, ClientId},
by = <<"flapping detector">>,
@ -134,11 +133,10 @@ handle_cast({detected, #flapping{clientid = ClientId,
false ->
?SLOG(warning, #{
msg => "client_disconnected",
client_id => ClientId,
peer_host => fmt_host(PeerHost),
detect_cnt => DetectCnt,
interval => Interval
})
}, #{clientid => ClientId})
end,
{noreply, State};

10
apps/emqx/src/emqx_logger.erl
View File

@ -197,15 +197,7 @@ critical(Metadata, Format, Args) when is_map(Metadata) ->
set_metadata_clientid(<<>>) ->
ok;
set_metadata_clientid(ClientId) ->
try
%% try put string format client-id metadata so
%% so the log is not like <<"...">>
Id = unicode:characters_to_list(ClientId, utf8),
set_proc_metadata(#{clientid => Id})
catch
_: _->
ok
end.
set_proc_metadata(#{clientid => ClientId}).
-spec(set_metadata_peername(peername_str()) -> ok).
set_metadata_peername(Peername) ->

73
apps/emqx/src/emqx_logger_textfmt.erl
View File

@ -18,22 +18,77 @@
-export([format/2]).
-export([check_config/1]).
-export([try_format_unicode/1]).
check_config(X) -> logger_formatter:check_config(X).
format(#{msg := {report, Report}, meta := Meta} = Event, Config) when is_map(Report) ->
logger_formatter:format(Event#{msg := {report, enrich(Report, Meta)}}, Config);
format(#{msg := Msg, meta := Meta} = Event, Config) ->
NewMsg = enrich_fmt(Msg, Meta),
logger_formatter:format(Event#{msg := NewMsg}, Config).
format(#{msg := {report, Report0}, meta := Meta} = Event, Config) when is_map(Report0) ->
Report1 = enrich_report_mfa(Report0, Meta),
Report2 = enrich_report_clientid(Report1, Meta),
Report3 = enrich_report_peername(Report2, Meta),
Report4 = enrich_report_topic(Report3, Meta),
logger_formatter:format(Event#{msg := {report, Report4}}, Config);
format(#{msg := {string, String}} = Event, Config) ->
format(Event#{msg => {"~ts ", String}}, Config);
format(#{msg := Msg0, meta := Meta} = Event, Config) ->
Msg1 = enrich_client_info(Msg0, Meta),
Msg2 = enrich_mfa(Msg1, Meta),
Msg3 = enrich_topic(Msg2, Meta),
logger_formatter:format(Event#{msg := Msg3}, Config).
enrich(Report, #{mfa := Mfa, line := Line}) ->
try_format_unicode(Char) ->
List =
try
case unicode:characters_to_list(Char) of
{error, _, _} -> error;
{incomplete, _, _} -> error;
Binary -> Binary
end
catch _:_ ->
error
end,
case List of
error -> io_lib:format("~0p", [Char]);
_ -> List
end.
enrich_report_mfa(Report, #{mfa := Mfa, line := Line}) ->
Report#{mfa => mfa(Mfa), line => Line};
enrich(Report, _) -> Report.
enrich_report_mfa(Report, _) -> Report.
enrich_fmt({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) ->
enrich_report_clientid(Report, #{clientid := ClientId}) ->
Report#{clientid => try_format_unicode(ClientId)};
enrich_report_clientid(Report, _) -> Report.
enrich_report_peername(Report, #{peername := Peername}) ->
Report#{peername => Peername};
enrich_report_peername(Report, _) -> Report.
%% clientid and peername always in emqx_conn's process metadata.
%% topic can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2
enrich_report_topic(Report, #{topic := Topic}) ->
Report#{topic => try_format_unicode(Topic)};
enrich_report_topic(Report = #{topic := Topic}, _) ->
Report#{topic => try_format_unicode(Topic)};
enrich_report_topic(Report, _) -> Report.
enrich_mfa({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) ->
{Fmt ++ " mfa: ~ts line: ~w", Args ++ [mfa(Mfa), Line]};
enrich_fmt(Msg, _) ->
enrich_mfa(Msg, _) ->
Msg.
enrich_client_info({Fmt, Args}, #{clientid := ClientId, peername := Peer}) when is_list(Fmt) ->
{" ~ts@~ts " ++ Fmt, [ClientId, Peer | Args] };
enrich_client_info({Fmt, Args}, #{clientid := ClientId}) when is_list(Fmt) ->
{" ~ts " ++ Fmt, [ClientId | Args]};
enrich_client_info({Fmt, Args}, #{peername := Peer}) when is_list(Fmt) ->
{" ~ts " ++ Fmt, [Peer | Args]};
enrich_client_info(Msg, _) ->
Msg.
enrich_topic({Fmt, Args}, #{topic := Topic}) when is_list(Fmt) ->
{" topic: ~ts" ++ Fmt, [Topic | Args]};
enrich_topic(Msg, _) ->
Msg.
mfa({M, F, A}) -> atom_to_list(M) ++ ":" ++ atom_to_list(F) ++ "/" ++ integer_to_list(A).

166
apps/emqx/src/emqx_packet.erl
View File

@ -44,7 +44,11 @@
, will_msg/1
]).
-export([format/1]).
-export([ format/1
, format/2
]).
-export([encode_hex/1]).
-define(TYPE_NAMES,
{ 'CONNECT'
@ -435,25 +439,28 @@ will_msg(#mqtt_packet_connect{clientid = ClientId,
%% @doc Format packet
-spec(format(emqx_types:packet()) -> iolist()).
format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}) ->
format_header(Header, format_variable(Variable, Payload)).
format(Packet) -> format(Packet, emqx_trace_handler:payload_encode()).
%% @doc Format packet
-spec(format(emqx_types:packet(), hex | text | hidden) -> iolist()).
format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, PayloadEncode) ->
HeaderIO = format_header(Header),
case format_variable(Variable, Payload, PayloadEncode) of
"" -> HeaderIO;
VarIO -> [HeaderIO,",", VarIO]
end.
format_header(#mqtt_packet_header{type = Type,
dup = Dup,
qos = QoS,
retain = Retain}, S) ->
S1 = case S == undefined of
true -> <<>>;
false -> [", ", S]
end,
io_lib:format("~ts(Q~p, R~p, D~p~ts)", [type_name(Type), QoS, i(Retain), i(Dup), S1]).
retain = Retain}) ->
io_lib:format("~ts(Q~p, R~p, D~p)", [type_name(Type), QoS, i(Retain), i(Dup)]).
format_variable(undefined, _) ->
undefined;
format_variable(Variable, undefined) ->
format_variable(Variable);
format_variable(Variable, Payload) ->
io_lib:format("~ts, Payload=~0p", [format_variable(Variable), Payload]).
format_variable(undefined, _, _) -> "";
format_variable(Variable, undefined, PayloadEncode) ->
format_variable(Variable, PayloadEncode);
format_variable(Variable, Payload, PayloadEncode) ->
[format_variable(Variable, PayloadEncode), format_payload(Payload, PayloadEncode)].
format_variable(#mqtt_packet_connect{
proto_ver = ProtoVer,
@ -467,57 +474,140 @@ format_variable(#mqtt_packet_connect{
will_topic = WillTopic,
will_payload = WillPayload,
username = Username,
password = Password}) ->
Format = "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts",
Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)],
{Format1, Args1} = if
WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~ts, Payload=~0p)",
Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]};
true -> {Format, Args}
end,
io_lib:format(Format1, Args1);
password = Password},
PayloadEncode) ->
Base = io_lib:format(
"ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts",
[ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)]),
case WillFlag of
true ->
[Base, io_lib:format(", Will(Q~p, R~p, Topic=~ts ",
[WillQoS, i(WillRetain), WillTopic]),
format_payload(WillPayload, PayloadEncode), ")"];
false ->
Base
end;
format_variable(#mqtt_packet_disconnect
{reason_code = ReasonCode}) ->
{reason_code = ReasonCode}, _) ->
io_lib:format("ReasonCode=~p", [ReasonCode]);
format_variable(#mqtt_packet_connack{ack_flags = AckFlags,
reason_code = ReasonCode}) ->
reason_code = ReasonCode}, _) ->
io_lib:format("AckFlags=~p, ReasonCode=~p", [AckFlags, ReasonCode]);
format_variable(#mqtt_packet_publish{topic_name = TopicName,
packet_id = PacketId}) ->
packet_id = PacketId}, _) ->
io_lib:format("Topic=~ts, PacketId=~p", [TopicName, PacketId]);
format_variable(#mqtt_packet_puback{packet_id = PacketId,
reason_code = ReasonCode}) ->
reason_code = ReasonCode}, _) ->
io_lib:format("PacketId=~p, ReasonCode=~p", [PacketId, ReasonCode]);
format_variable(#mqtt_packet_subscribe{packet_id = PacketId,
topic_filters = TopicFilters}) ->
io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, TopicFilters]);
topic_filters = TopicFilters}, _) ->
[io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=",
format_topic_filters(TopicFilters)];
format_variable(#mqtt_packet_unsubscribe{packet_id = PacketId,
topic_filters = Topics}) ->
io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, Topics]);
topic_filters = Topics}, _) ->
[io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=",
format_topic_filters(Topics)];
format_variable(#mqtt_packet_suback{packet_id = PacketId,
reason_codes = ReasonCodes}) ->
reason_codes = ReasonCodes}, _) ->
io_lib:format("PacketId=~p, ReasonCodes=~p", [PacketId, ReasonCodes]);
format_variable(#mqtt_packet_unsuback{packet_id = PacketId}) ->
format_variable(#mqtt_packet_unsuback{packet_id = PacketId}, _) ->
io_lib:format("PacketId=~p", [PacketId]);
format_variable(#mqtt_packet_auth{reason_code = ReasonCode}) ->
format_variable(#mqtt_packet_auth{reason_code = ReasonCode}, _) ->
io_lib:format("ReasonCode=~p", [ReasonCode]);
format_variable(PacketId) when is_integer(PacketId) ->
format_variable(PacketId, _) when is_integer(PacketId) ->
io_lib:format("PacketId=~p", [PacketId]).
format_password(undefined) -> undefined;
format_password(_Password) -> '******'.
format_password(undefined) -> "undefined";
format_password(_Password) -> "******".
format_payload(Payload, text) -> ["Payload=", io_lib:format("~ts", [Payload])];
format_payload(Payload, hex) -> ["Payload(hex)=", encode_hex(Payload)];
format_payload(_, hidden) -> "Payload=******".
i(true) -> 1;
i(false) -> 0;
i(I) when is_integer(I) -> I.
format_topic_filters(Filters) ->
["[",
lists:join(",",
lists:map(
fun({TopicFilter, SubOpts}) ->
io_lib:format("~ts(~p)", [TopicFilter, SubOpts]);
(TopicFilter) ->
io_lib:format("~ts", [TopicFilter])
end, Filters)),
"]"].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Hex encoding functions
%% Copy from binary:encode_hex/1 (was only introduced in OTP24).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-define(HEX(X), (hex(X)):16).
-compile({inline,[hex/1]}).
-spec encode_hex(Bin) -> Bin2 when
Bin :: binary(),
Bin2 :: <<_:_*16>>.
encode_hex(Data) when byte_size(Data) rem 8 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G),?HEX(H)>> || <<A,B,C,D,E,F,G,H>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 7 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G)>> || <<A,B,C,D,E,F,G>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 6 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F)>> || <<A,B,C,D,E,F>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 5 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E)>> || <<A,B,C,D,E>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 4 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C),?HEX(D)>> || <<A,B,C,D>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 3 =:= 0 ->
<< <<?HEX(A),?HEX(B),?HEX(C)>> || <<A,B,C>> <= Data >>;
encode_hex(Data) when byte_size(Data) rem 2 =:= 0 ->
<< <<?HEX(A),?HEX(B)>> || <<A,B>> <= Data >>;
encode_hex(Data) when is_binary(Data) ->
<< <<?HEX(N)>> || <<N>> <= Data >>;
encode_hex(Bin) ->
erlang:error(badarg, [Bin]).
hex(X) ->
element(
X+1, {16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, 16#3041,
16#3042, 16#3043, 16#3044, 16#3045, 16#3046,
16#3130, 16#3131, 16#3132, 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3141,
16#3142, 16#3143, 16#3144, 16#3145, 16#3146,
16#3230, 16#3231, 16#3232, 16#3233, 16#3234, 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3241,
16#3242, 16#3243, 16#3244, 16#3245, 16#3246,
16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, 16#3337, 16#3338, 16#3339, 16#3341,
16#3342, 16#3343, 16#3344, 16#3345, 16#3346,
16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, 16#3439, 16#3441,
16#3442, 16#3443, 16#3444, 16#3445, 16#3446,
16#3530, 16#3531, 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3541,
16#3542, 16#3543, 16#3544, 16#3545, 16#3546,
16#3630, 16#3631, 16#3632, 16#3633, 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3641,
16#3642, 16#3643, 16#3644, 16#3645, 16#3646,
16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, 16#3736, 16#3737, 16#3738, 16#3739, 16#3741,
16#3742, 16#3743, 16#3744, 16#3745, 16#3746,
16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, 16#3838, 16#3839, 16#3841,
16#3842, 16#3843, 16#3844, 16#3845, 16#3846,
16#3930, 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, 16#3941,
16#3942, 16#3943, 16#3944, 16#3945, 16#3946,
16#4130, 16#4131, 16#4132, 16#4133, 16#4134, 16#4135, 16#4136, 16#4137, 16#4138, 16#4139, 16#4141,
16#4142, 16#4143, 16#4144, 16#4145, 16#4146,
16#4230, 16#4231, 16#4232, 16#4233, 16#4234, 16#4235, 16#4236, 16#4237, 16#4238, 16#4239, 16#4241,
16#4242, 16#4243, 16#4244, 16#4245, 16#4246,
16#4330, 16#4331, 16#4332, 16#4333, 16#4334, 16#4335, 16#4336, 16#4337, 16#4338, 16#4339, 16#4341,
16#4342, 16#4343, 16#4344, 16#4345, 16#4346,
16#4430, 16#4431, 16#4432, 16#4433, 16#4434, 16#4435, 16#4436, 16#4437, 16#4438, 16#4439, 16#4441,
16#4442, 16#4443, 16#4444, 16#4445, 16#4446,
16#4530, 16#4531, 16#4532, 16#4533, 16#4534, 16#4535, 16#4536, 16#4537, 16#4538, 16#4539, 16#4541,
16#4542, 16#4543, 16#4544, 16#4545, 16#4546,
16#4630, 16#4631, 16#4632, 16#4633, 16#4634, 16#4635, 16#4636, 16#4637, 16#4638, 16#4639, 16#4641,
16#4642, 16#4643, 16#4644, 16#4645, 16#4646}).

28
apps/emqx/src/emqx_schema.erl
View File

@ -38,7 +38,6 @@
-type ip_port() :: tuple().
-type cipher() :: map().
-type rfc3339_system_time() :: integer().
-type unicode_binary() :: binary().
-typerefl_from_string({duration/0, emqx_schema, to_duration}).
-typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}).
@ -52,7 +51,6 @@
-typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}).
-typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}).
-typerefl_from_string({rfc3339_system_time/0, emqx_schema, rfc3339_to_system_time}).
-typerefl_from_string({unicode_binary/0, emqx_schema, to_unicode_binary}).
-export([ validate_heap_size/1
, parse_user_lookup_fun/1
@ -66,8 +64,7 @@
to_bar_separated_list/1, to_ip_port/1,
to_erl_cipher_suite/1,
to_comma_separated_atoms/1,
rfc3339_to_system_time/1,
to_unicode_binary/1]).
rfc3339_to_system_time/1]).
-behaviour(hocon_schema).
@ -76,8 +73,7 @@
comma_separated_list/0, bar_separated_list/0, ip_port/0,
cipher/0,
comma_separated_atoms/0,
rfc3339_system_time/0,
unicode_binary/0]).
rfc3339_system_time/0]).
-export([namespace/0, roots/0, roots/1, fields/1]).
-export([conf_get/2, conf_get/3, keys/2, filter/1]).
@ -184,6 +180,12 @@ roots(low) ->
, {"latency_stats",
sc(ref("latency_stats"),
#{})}
, {"trace",
sc(ref("trace"),
#{desc => """
Real-time filtering logs for the ClientID or Topic or IP for debugging.
"""
})}
].
fields("persistent_session_store") ->
@ -981,6 +983,17 @@ when deactivated, but after the retention time.
fields("latency_stats") ->
[ {"samples", sc(integer(), #{default => 10,
desc => "the number of smaples for calculate the average latency of delivery"})}
];
fields("trace") ->
[ {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{
default => text,
desc => """
Determine the format of the payload format in the trace file.<br>
`text`: Text-based protocol or plain text protocol. It is recommended when payload is json encode.<br>
`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.<br>
`hidden`: payload is obfuscated as `******`
"""
})}
].
mqtt_listener() ->
@ -1390,9 +1403,6 @@ rfc3339_to_system_time(DateTime) ->
{error, bad_rfc3339_timestamp}
end.
to_unicode_binary(Str) ->
{ok, unicode:characters_to_binary(Str)}.
to_bar_separated_list(Str) ->
{ok, string:tokens(Str, "| ")}.

12
apps/emqx/src/emqx_session.erl
View File

@ -535,16 +535,20 @@ enqueue(Msg, Session = #session{mqueue = Q}) when is_record(Msg, message) ->
(Dropped =/= undefined) andalso log_dropped(Dropped, Session),
Session#session{mqueue = NewQ}.
log_dropped(Msg = #message{qos = QoS}, #session{mqueue = Q}) ->
case (QoS == ?QOS_0) andalso (not emqx_mqueue:info(store_qos0, Q)) of
log_dropped(Msg = #message{qos = QoS, topic = Topic}, #session{mqueue = Q}) ->
Payload = emqx_message:to_log_map(Msg),
#{store_qos0 := StoreQos0} = QueueInfo = emqx_mqueue:info(Q),
case (QoS == ?QOS_0) andalso (not StoreQos0) of
true ->
ok = emqx_metrics:inc('delivery.dropped.qos0_msg'),
?SLOG(warning, #{msg => "dropped_qos0_msg",
payload => emqx_message:to_log_map(Msg)});
queue => QueueInfo,
payload => Payload}, #{topic => Topic});
false ->
ok = emqx_metrics:inc('delivery.dropped.queue_full'),
?SLOG(warning, #{msg => "dropped_msg_due_to_mqueue_is_full",
payload => emqx_message:to_log_map(Msg)})
queue => QueueInfo,
payload => Payload}, #{topic => Topic})
end.
enrich_fun(Session = #session{subscriptions = Subs}) ->

2
apps/emqx/src/emqx_session_router.erl
View File

@ -260,7 +260,7 @@ code_change(_OldVsn, State, _Extra) ->
init_resume_worker(RemotePid, SessionID, #{ pmon := Pmon } = State) ->
case emqx_session_router_worker_sup:start_worker(SessionID, RemotePid) of
{error, What} ->
?SLOG(error, #{msg => "Could not start resume worker", reason => What}),
?SLOG(error, #{msg => "failed_to_start_resume_worker", reason => What}),
error;
{ok, Pid} ->
Pmon1 = emqx_pmon:monitor(Pid, Pmon),

118
apps/emqx/src/emqx_trace/emqx_trace.erl
View File

@ -26,6 +26,7 @@
-export([ publish/1
, subscribe/3
, unsubscribe/2
, log/4
]).
-export([ start_link/0
@ -36,6 +37,7 @@
, delete/1
, clear/0
, update/2
, check/0
]).
-export([ format/1
@ -50,6 +52,7 @@
-define(TRACE, ?MODULE).
-define(MAX_SIZE, 30).
-define(OWN_KEYS, [level, filters, filter_default, handlers]).
-ifdef(TEST).
-export([ log_file/2
@ -80,27 +83,53 @@ mnesia(boot) ->
publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore;
publish(#message{from = From, topic = Topic, payload = Payload}) when
is_binary(From); is_atom(From) ->
emqx_logger:info(
#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
"PUBLISH to ~s: ~0p",
[Topic, Payload]
).
?TRACE("PUBLISH", "publish_to", #{topic => Topic, payload => Payload}).
subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore;
subscribe(Topic, SubId, SubOpts) ->
emqx_logger:info(
#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
"~ts SUBSCRIBE ~ts: Options: ~0p",
[SubId, Topic, SubOpts]
).
?TRACE("SUBSCRIBE", "subscribe", #{topic => Topic, sub_opts => SubOpts, sub_id => SubId}).
unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore;
unsubscribe(Topic, SubOpts) ->
emqx_logger:info(
#{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
"~ts UNSUBSCRIBE ~ts: Options: ~0p",
[maps:get(subid, SubOpts, ""), Topic, SubOpts]
).
?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}).
log(List, Event, Msg, Meta0) ->
Meta =
case logger:get_process_metadata() of
undefined -> Meta0;
ProcMeta -> maps:merge(ProcMeta, Meta0)
end,
Log = #{level => trace, event => Event, meta => Meta, msg => Msg},
log_filter(List, Log).
log_filter([], _Log) -> ok;
log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
case FilterFun(Log0, {Filter, Name}) of
stop -> stop;
ignore -> ignore;
Log ->
case logger_config:get(ets:whereis(logger), Id) of
{ok, #{module := Module} = HandlerConfig0} ->
HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
try Module:log(Log, HandlerConfig)
catch C:R:S ->
case logger:remove_handler(Id) of
ok ->
logger:internal_log(error, {removed_failing_handler, Id, C, R, S});
{error,{not_found,_}} ->
%% Probably already removed by other client
%% Don't report again
ok;
{error,Reason} ->
logger:internal_log(error,
{removed_handler_failed, Id, Reason, C, R, S})
end
end;
{error, {not_found, Id}} -> ok;
{error, Reason} -> logger:internal_log(error, {find_handle_id_failed, Id, Reason})
end
end,
log_filter(Rest, Log0).
-spec(start_link() -> emqx_types:startlink_ret()).
start_link() ->
@ -161,6 +190,9 @@ update(Name, Enable) ->
end,
transaction(Tran).
check() ->
gen_server:call(?MODULE, check).
-spec get_trace_filename(Name :: binary()) ->
{ok, FileName :: string()} | {error, not_found}.
get_trace_filename(Name) ->
@ -196,15 +228,17 @@ format(Traces) ->
init([]) ->
ok = mria:wait_for_tables([?TRACE]),
erlang:process_flag(trap_exit, true),
OriginLogLevel = emqx_logger:get_primary_log_level(),
ok = filelib:ensure_dir(trace_dir()),
ok = filelib:ensure_dir(zip_dir()),
{ok, _} = mnesia:subscribe({table, ?TRACE, simple}),
Traces = get_enable_trace(),
ok = update_log_primary_level(Traces, OriginLogLevel),
TRef = update_trace(Traces),
{ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}.
update_trace_handler(),
{ok, #{timer => TRef, monitors => #{}}}.
handle_call(check, _From, State) ->
{_, NewState} = handle_info({mnesia_table_event, check}, State),
{reply, ok, NewState};
handle_call(Req, _From, State) ->
?SLOG(error, #{unexpected_call => Req}),
{reply, ok, State}.
@ -223,11 +257,10 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor
lists:foreach(fun file:delete/1, Files),
{noreply, State#{monitors => NewMonitors}}
end;
handle_info({timeout, TRef, update_trace},
#{timer := TRef, primary_log_level := OriginLogLevel} = State) ->
handle_info({timeout, TRef, update_trace}, #{timer := TRef} = State) ->
Traces = get_enable_trace(),
ok = update_log_primary_level(Traces, OriginLogLevel),
NextTRef = update_trace(Traces),
update_trace_handler(),
{noreply, State#{timer => NextTRef}};
handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) ->
@ -238,11 +271,11 @@ handle_info(Info, State) ->
?SLOG(error, #{unexpected_info => Info}),
{noreply, State}.
terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) ->
ok = set_log_primary_level(OriginLogLevel),
terminate(_Reason, #{timer := TRef}) ->
_ = mnesia:unsubscribe({table, ?TRACE, simple}),
emqx_misc:cancel_timer(TRef),
stop_all_trace_handler(),
update_trace_handler(),
_ = file:del_dir_r(zip_dir()),
ok.
@ -270,7 +303,7 @@ update_trace(Traces) ->
disable_finished(Finished),
Started = emqx_trace_handler:running(),
{NeedRunning, AllStarted} = start_trace(Running, Started),
NeedStop = AllStarted -- NeedRunning,
NeedStop = filter_cli_handler(AllStarted) -- NeedRunning,
ok = stop_trace(NeedStop, Started),
clean_stale_trace_files(),
NextTime = find_closest_time(Traces, Now),
@ -308,10 +341,10 @@ disable_finished(Traces) ->
start_trace(Traces, Started0) ->
Started = lists:map(fun(#{name := Name}) -> Name end, Started0),
lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) ->
lists:foldl(fun(#?TRACE{name = Name} = Trace,
{Running, StartedAcc}) ->
case lists:member(Name, StartedAcc) of
true ->
{[Name | Running], StartedAcc};
true -> {[Name | Running], StartedAcc};
false ->
case start_trace(Trace) of
ok -> {[Name | Running], [Name | StartedAcc]};
@ -330,9 +363,11 @@ start_trace(Trace) ->
emqx_trace_handler:install(Who, debug, log_file(Name, Start)).
stop_trace(Finished, Started) ->
lists:foreach(fun(#{name := Name, type := Type}) ->
lists:foreach(fun(#{name := Name, type := Type, filter := Filter}) ->
case lists:member(Name, Finished) of
true -> emqx_trace_handler:uninstall(Type, Name);
true ->
?TRACE("API", "trace_stopping", #{Type => Filter}),
emqx_trace_handler:uninstall(Type, Name);
false -> ok
end
end, Started).
@ -419,7 +454,7 @@ to_trace(#{type := ip_address, ip_address := Filter} = Trace, Rec) ->
case validate_ip_address(Filter) of
ok ->
Trace0 = maps:without([type, ip_address], Trace),
to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = Filter});
to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = binary_to_list(Filter)});
Error -> Error
end;
to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])};
@ -481,11 +516,20 @@ transaction(Tran) ->
{aborted, Reason} -> {error, Reason}
end.
update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel);
update_log_primary_level(_, _) -> set_log_primary_level(debug).
set_log_primary_level(NewLevel) ->
case NewLevel =/= emqx_logger:get_primary_log_level() of
true -> emqx_logger:set_primary_log_level(NewLevel);
false -> ok
update_trace_handler() ->
case emqx_trace_handler:running() of
[] -> persistent_term:erase(?TRACE_FILTER);
Running ->
List = lists:map(fun(#{id := Id, filter_fun := FilterFun,
filter := Filter, name := Name}) ->
{Id, FilterFun, Filter, Name} end, Running),
case List =/= persistent_term:get(?TRACE_FILTER, undefined) of
true -> persistent_term:put(?TRACE_FILTER, List);
false -> ok
end
end.
filter_cli_handler(Names) ->
lists:filter(fun(Name) ->
nomatch =:= re:run(Name, "^CLI-+.", [])
end, Names).

62
apps/emqx/src/emqx_trace/emqx_trace_formatter.erl Normal file
View File

@ -0,0 +1,62 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_trace_formatter).
-export([format/2]).
%%%-----------------------------------------------------------------
%%% API
-spec format(LogEvent, Config) -> unicode:chardata() when
LogEvent :: logger:log_event(),
Config :: logger:config().
format(#{level := trace, event := Event, meta := Meta, msg := Msg},
#{payload_encode := PEncode}) ->
Time = calendar:system_time_to_rfc3339(erlang:system_time(second)),
ClientId = to_iolist(maps:get(clientid, Meta, "")),
Peername = maps:get(peername, Meta, ""),
MetaBin = format_meta(Meta, PEncode),
[Time, " [", Event, "] ", ClientId, "@", Peername, " msg: ", Msg, MetaBin, "\n"];
format(Event, Config) ->
emqx_logger_textfmt:format(Event, Config).
format_meta(Meta0, Encode) ->
Packet = format_packet(maps:get(packet, Meta0, undefined), Encode),
Payload = format_payload(maps:get(payload, Meta0, undefined), Encode),
Meta1 = maps:without([msg, clientid, peername, packet, payload], Meta0),
case Meta1 =:= #{} of
true -> [Packet, Payload];
false -> [Packet, ", ", map_to_iolist(Meta1), Payload]
end.
format_packet(undefined, _) -> "";
format_packet(Packet, Encode) -> [", packet: ", emqx_packet:format(Packet, Encode)].
format_payload(undefined, _) -> "";
format_payload(Payload, text) -> [", payload: ", io_lib:format("~ts", [Payload])];
format_payload(Payload, hex) -> [", payload(hex): ", emqx_packet:encode_hex(Payload)];
format_payload(_, hidden) -> ", payload=******".
to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom);
to_iolist(Int) when is_integer(Int) -> integer_to_list(Int);
to_iolist(Float) when is_float(Float) -> float_to_list(Float, [{decimals, 2}]);
to_iolist(SubMap) when is_map(SubMap) -> ["[", map_to_iolist(SubMap), "]"];
to_iolist(Char) -> emqx_logger_textfmt:try_format_unicode(Char).
map_to_iolist(Map) ->
lists:join(",",
lists:map(fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end,
maps:to_list(Map))).

94
apps/emqx/src/emqx_trace/emqx_trace_handler.erl
View File

@ -25,6 +25,7 @@
-export([ running/0
, install/3
, install/4
, install/5
, uninstall/1
, uninstall/2
]).
@ -36,6 +37,7 @@
]).
-export([handler_id/2]).
-export([payload_encode/0]).
-type tracer() :: #{
name := binary(),
@ -77,22 +79,18 @@ install(Type, Filter, Level, LogFile) ->
-spec install(tracer(), logger:level() | all, string()) -> ok | {error, term()}.
install(Who, all, LogFile) ->
install(Who, debug, LogFile);
install(Who, Level, LogFile) ->
PrimaryLevel = emqx_logger:get_primary_log_level(),
try logger:compare_levels(Level, PrimaryLevel) of
lt ->
{error,
io_lib:format(
"Cannot trace at a log level (~s) "
"lower than the primary log level (~s)",
[Level, PrimaryLevel]
)};
_GtOrEq ->
install_handler(Who, Level, LogFile)
catch
error:badarg ->
{error, {invalid_log_level, Level}}
end.
install(Who = #{name := Name, type := Type}, Level, LogFile) ->
HandlerId = handler_id(Name, Type),
Config = #{
level => Level,
formatter => formatter(Who),
filter_default => stop,
filters => filters(Who),
config => ?CONFIG(LogFile)
},
Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
show_prompts(Res, Who, "start_trace"),
Res.
-spec uninstall(Type :: clientid | topic | ip_address,
Name :: binary() | list()) -> ok | {error, term()}.
@ -121,83 +119,59 @@ uninstall(HandlerId) ->
running() ->
lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)).
-spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore.
-spec filter_clientid(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop.
filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log;
filter_clientid(_Log, _ExpectId) -> ignore.
filter_clientid(_Log, _ExpectId) -> stop.
-spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore.
-spec filter_topic(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop.
filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) ->
case emqx_topic:match(Topic, TopicFilter) of
true -> Log;
false -> ignore
false -> stop
end;
filter_topic(_Log, _ExpectId) -> ignore.
filter_topic(_Log, _ExpectId) -> stop.
-spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore.
-spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop.
filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) ->
case lists:prefix(IP, Peername) of
true -> Log;
false -> ignore
false -> stop
end;
filter_ip_address(_Log, _ExpectId) -> ignore.
install_handler(Who = #{name := Name, type := Type}, Level, LogFile) ->
HandlerId = handler_id(Name, Type),
Config = #{
level => Level,
formatter => formatter(Who),
filter_default => stop,
filters => filters(Who),
config => ?CONFIG(LogFile)
},
Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
show_prompts(Res, Who, "start_trace"),
Res.
filter_ip_address(_Log, _ExpectId) -> stop.
filters(#{type := clientid, filter := Filter, name := Name}) ->
[{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}];
[{clientid, {fun ?MODULE:filter_clientid/2, {Filter, Name}}}];
filters(#{type := topic, filter := Filter, name := Name}) ->
[{topic, {fun ?MODULE:filter_topic/2, {ensure_bin(Filter), Name}}}];
filters(#{type := ip_address, filter := Filter, name := Name}) ->
[{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}].
formatter(#{type := Type}) ->
{logger_formatter,
formatter(#{type := _Type}) ->
{emqx_trace_formatter,
#{
template => template(Type),
single_line => false,
%% template is for ?SLOG message not ?TRACE.
template => [time," [",level,"] ", msg,"\n"],
single_line => true,
max_size => unlimited,
depth => unlimited
depth => unlimited,
payload_encode => payload_encode()
}
}.
%% Don't log clientid since clientid only supports exact match, all client ids are the same.
%% if clientid is not latin characters. the logger_formatter restricts the output must be `~tp`
%% (actually should use `~ts`), the utf8 characters clientid will become very difficult to read.
template(clientid) ->
[time, " [", level, "] ", {peername, [peername, " "], []}, msg, "\n"];
%% TODO better format when clientid is utf8.
template(_) ->
[time, " [", level, "] ",
{clientid,
[{peername, [clientid, "@", peername, " "], [clientid, " "]}],
[{peername, [peername, " "], []}]
},
msg, "\n"
].
filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) ->
Init = #{id => Id, level => Level, dst => Dst},
case Filters of
[{Type, {_FilterFun, {Filter, Name}}}] when
[{Type, {FilterFun, {Filter, Name}}}] when
Type =:= topic orelse
Type =:= clientid orelse
Type =:= ip_address ->
[Init#{type => Type, filter => Filter, name => Name} | Acc];
[Init#{type => Type, filter => Filter, name => Name, filter_fun => FilterFun} | Acc];
_ ->
Acc
end.
payload_encode() -> emqx_config:get([trace, payload_encode], text).
handler_id(Name, Type) ->
try
do_handler_id(Name, Type)

15
apps/emqx/src/emqx_ws_connection.erl
View File

@ -347,7 +347,6 @@ websocket_handle({binary, Data}, State) when is_list(Data) ->
websocket_handle({binary, iolist_to_binary(Data)}, State);
websocket_handle({binary, Data}, State) ->
?SLOG(debug, #{msg => "RECV_data", data => Data, transport => websocket}),
State2 = ensure_stats_timer(State),
{Packets, State3} = parse_incoming(Data, [], State2),
LenMsg = erlang:length(Packets),
@ -432,11 +431,11 @@ websocket_info(Info, State) ->
websocket_close({_, ReasonCode, _Payload}, State) when is_integer(ReasonCode) ->
websocket_close(ReasonCode, State);
websocket_close(Reason, State) ->
?SLOG(debug, #{msg => "websocket_closed", reason => Reason}),
?TRACE("SOCKET", "websocket_closed", #{reason => Reason}),
handle_info({sock_closed, Reason}, State).
terminate(Reason, _Req, #state{channel = Channel}) ->
?SLOG(debug, #{msg => "terminated", reason => Reason}),
?TRACE("SOCKET", "websocket_terminated", #{reason => Reason}),
emqx_channel:terminate(Reason, Channel);
terminate(_Reason, _Req, _UnExpectedState) ->
@ -480,7 +479,7 @@ handle_info({connack, ConnAck}, State) ->
return(enqueue(ConnAck, State));
handle_info({close, Reason}, State) ->
?SLOG(debug, #{msg => "force_socket_close", reason => Reason}),
?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}),
return(enqueue({close, Reason}, State));
handle_info({event, connected}, State = #state{channel = Channel}) ->
@ -550,7 +549,7 @@ check_limiter(Needs,
{ok, Limiter2} ->
WhenOk(Data, Msgs, State#state{limiter = Limiter2});
{pause, Time, Limiter2} ->
?SLOG(warning, #{msg => "pause time dueto rate limit",
?SLOG(warning, #{msg => "pause_time_due_to_rate_limit",
needs => Needs,
time_in_ms => Time}),
@ -586,7 +585,7 @@ retry_limiter(#state{limiter = Limiter} = State) ->
, limiter_timer = undefined
});
{pause, Time, Limiter2} ->
?SLOG(warning, #{msg => "pause time dueto rate limit",
?SLOG(warning, #{msg => "pause_time_due_to_rate_limit",
types => Types,
time_in_ms => Time}),
@ -663,7 +662,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
handle_incoming(Packet, State = #state{listener = {Type, Listener}})
when is_record(Packet, mqtt_packet) ->
?SLOG(debug, #{msg => "RECV", packet => emqx_packet:format(Packet)}),
?TRACE("WS-MQTT", "mqtt_packet_received", #{packet => Packet}),
ok = inc_incoming_stats(Packet),
NState = case emqx_pd:get_counter(incoming_pubs) >
get_active_n(Type, Listener) of
@ -727,7 +726,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
ok = emqx_metrics:inc('delivery.dropped.too_large'),
ok = emqx_metrics:inc('delivery.dropped'),
<<>>;
Data -> ?SLOG(debug, #{msg => "SEND", packet => Packet}),
Data -> ?TRACE("WS-MQTT", "mqtt_packet_sent", #{packet => Packet}),
ok = inc_outgoing_stats(Packet),
Data
catch

16
apps/emqx/test/emqx_banned_SUITE.erl
View File

@ -39,9 +39,13 @@ t_add_delete(_) ->
by = <<"banned suite">>,
reason = <<"test">>,
at = erlang:system_time(second),
until = erlang:system_time(second) + 1000
until = erlang:system_time(second) + 1
},
{ok, _} = emqx_banned:create(Banned),
{error, {already_exist, Banned}} = emqx_banned:create(Banned),
?assertEqual(1, emqx_banned:info(size)),
{error, {already_exist, Banned}} =
emqx_banned:create(Banned#banned{until = erlang:system_time(second) + 100}),
?assertEqual(1, emqx_banned:info(size)),
ok = emqx_banned:delete({clientid, <<"TestClient">>}),
@ -68,10 +72,14 @@ t_check(_) ->
username => <<"user">>,
peerhost => {127,0,0,1}
},
ClientInfo5 = #{},
ClientInfo6 = #{clientid => <<"client1">>},
?assert(emqx_banned:check(ClientInfo1)),
?assert(emqx_banned:check(ClientInfo2)),
?assert(emqx_banned:check(ClientInfo3)),
?assertNot(emqx_banned:check(ClientInfo4)),
?assertNot(emqx_banned:check(ClientInfo5)),
?assertNot(emqx_banned:check(ClientInfo6)),
ok = emqx_banned:delete({clientid, <<"BannedClient">>}),
ok = emqx_banned:delete({username, <<"BannedUser">>}),
ok = emqx_banned:delete({peerhost, {192,168,0,1}}),
@ -83,8 +91,10 @@ t_check(_) ->
t_unused(_) ->
{ok, Banned} = emqx_banned:start_link(),
{ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient">>},
until = erlang:system_time(second)}),
{ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient1">>},
until = erlang:system_time(second)}),
{ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient2">>},
until = erlang:system_time(second) - 1}),
?assertEqual(ignored, gen_server:call(Banned, unexpected_req)),
?assertEqual(ok, gen_server:cast(Banned, unexpected_msg)),
?assertEqual(ok, Banned ! ok),

107
apps/emqx/test/emqx_trace_handler_SUITE.erl
View File

@ -39,32 +39,29 @@ end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
init_per_testcase(t_trace_clientid, Config) ->
init(),
Config;
init_per_testcase(_Case, Config) ->
ok = emqx_logger:set_log_level(debug),
_ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()],
init(),
Config.
end_per_testcase(_Case, _Config) ->
ok = emqx_logger:set_log_level(warning),
terminate(),
ok.
t_trace_clientid(_Config) ->
%% Start tracing
emqx_logger:set_log_level(error),
{error, _} = emqx_trace_handler:install(clientid, <<"client">>, debug, "tmp/client.log"),
emqx_logger:set_log_level(debug),
%% add list clientid
ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"),
ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"),
ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"),
{error, {invalid_log_level, bad_level}} =
emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"),
ok = emqx_trace_handler:install("CLI-client1", clientid, "client", debug, "tmp/client.log"),
ok = emqx_trace_handler:install("CLI-client2", clientid, <<"client2">>, all, "tmp/client2.log"),
ok = emqx_trace_handler:install("CLI-client3", clientid, <<"client3">>, all, "tmp/client3.log"),
{error, {handler_not_added, {file_error, ".", eisdir}}} =
emqx_trace_handler:install(clientid, <<"client5">>, debug, "."),
ok = filesync(<<"client">>, clientid),
ok = filesync(<<"client2">>, clientid),
ok = filesync(<<"client3">>, clientid),
emqx_trace:check(),
ok = filesync(<<"CLI-client1">>, clientid),
ok = filesync(<<"CLI-client2">>, clientid),
ok = filesync(<<"CLI-client3">>, clientid),
%% Verify the tracing file exits
?assert(filelib:is_regular("tmp/client.log")),
@ -72,11 +69,11 @@ t_trace_clientid(_Config) ->
?assert(filelib:is_regular("tmp/client3.log")),
%% Get current traces
?assertMatch([#{type := clientid, filter := "client", name := <<"client">>,
?assertMatch([#{type := clientid, filter := <<"client">>, name := <<"CLI-client1">>,
level := debug, dst := "tmp/client.log"},
#{type := clientid, filter := "client2", name := <<"client2">>
#{type := clientid, filter := <<"client2">>, name := <<"CLI-client2">>
, level := debug, dst := "tmp/client2.log"},
#{type := clientid, filter := "client3", name := <<"client3">>,
#{type := clientid, filter := <<"client3">>, name := <<"CLI-client3">>,
level := debug, dst := "tmp/client3.log"}
], emqx_trace_handler:running()),
@ -85,9 +82,9 @@ t_trace_clientid(_Config) ->
emqtt:connect(T),
emqtt:publish(T, <<"a/b/c">>, <<"hi">>),
emqtt:ping(T),
ok = filesync(<<"client">>, clientid),
ok = filesync(<<"client2">>, clientid),
ok = filesync(<<"client3">>, clientid),
ok = filesync(<<"CLI-client1">>, clientid),
ok = filesync(<<"CLI-client2">>, clientid),
ok = filesync(<<"CLI-client3">>, clientid),
%% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log".
{ok, Bin} = file:read_file("tmp/client.log"),
@ -98,25 +95,24 @@ t_trace_clientid(_Config) ->
?assert(filelib:file_size("tmp/client2.log") == 0),
%% Stop tracing
ok = emqx_trace_handler:uninstall(clientid, <<"client">>),
ok = emqx_trace_handler:uninstall(clientid, <<"client2">>),
ok = emqx_trace_handler:uninstall(clientid, <<"client3">>),
ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client1">>),
ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client2">>),
ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client3">>),
emqtt:disconnect(T),
?assertEqual([], emqx_trace_handler:running()).
t_trace_clientid_utf8(_) ->
emqx_logger:set_log_level(debug),
Utf8Id = <<"client 漢字編碼"/utf8>>,
ok = emqx_trace_handler:install(clientid, Utf8Id, debug, "tmp/client-utf8.log"),
ok = emqx_trace_handler:install("CLI-UTF8", clientid, Utf8Id, debug, "tmp/client-utf8.log"),
emqx_trace:check(),
{ok, T} = emqtt:start_link([{clientid, Utf8Id}]),
emqtt:connect(T),
[begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)],
emqtt:ping(T),
ok = filesync(Utf8Id, clientid),
ok = emqx_trace_handler:uninstall(clientid, Utf8Id),
ok = filesync("CLI-UTF8", clientid),
ok = emqx_trace_handler:uninstall(clientid, "CLI-UTF8"),
emqtt:disconnect(T),
?assertEqual([], emqx_trace_handler:running()),
ok.
@ -126,11 +122,11 @@ t_trace_topic(_Config) ->
emqtt:connect(T),
%% Start tracing
emqx_logger:set_log_level(debug),
ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"),
ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"),
ok = filesync(<<"x/#">>, topic),
ok = filesync(<<"y/#">>, topic),
ok = emqx_trace_handler:install("CLI-TOPIC-1", topic, <<"x/#">>, all, "tmp/topic_trace_x.log"),
ok = emqx_trace_handler:install("CLI-TOPIC-2", topic, <<"y/#">>, all, "tmp/topic_trace_y.log"),
emqx_trace:check(),
ok = filesync("CLI-TOPIC-1", topic),
ok = filesync("CLI-TOPIC-2", topic),
%% Verify the tracing file exits
?assert(filelib:is_regular("tmp/topic_trace_x.log")),
@ -138,9 +134,9 @@ t_trace_topic(_Config) ->
%% Get current traces
?assertMatch([#{type := topic, filter := <<"x/#">>,
level := debug, dst := "tmp/topic_trace_x.log", name := <<"x/#">>},
level := debug, dst := "tmp/topic_trace_x.log", name := <<"CLI-TOPIC-1">>},
#{type := topic, filter := <<"y/#">>,
name := <<"y/#">>, level := debug, dst := "tmp/topic_trace_y.log"}
name := <<"CLI-TOPIC-2">>, level := debug, dst := "tmp/topic_trace_y.log"}
],
emqx_trace_handler:running()),
@ -149,8 +145,8 @@ t_trace_topic(_Config) ->
emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
emqtt:subscribe(T, <<"x/y/z">>),
emqtt:unsubscribe(T, <<"x/y/z">>),
ok = filesync(<<"x/#">>, topic),
ok = filesync(<<"y/#">>, topic),
ok = filesync("CLI-TOPIC-1", topic),
ok = filesync("CLI-TOPIC-2", topic),
{ok, Bin} = file:read_file("tmp/topic_trace_x.log"),
?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@ -161,8 +157,8 @@ t_trace_topic(_Config) ->
?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0),
%% Stop tracing
ok = emqx_trace_handler:uninstall(topic, <<"x/#">>),
ok = emqx_trace_handler:uninstall(topic, <<"y/#">>),
ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-1">>),
ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-2">>),
{error, _Reason} = emqx_trace_handler:uninstall(topic, <<"z/#">>),
?assertEqual([], emqx_trace_handler:running()),
emqtt:disconnect(T).
@ -172,10 +168,12 @@ t_trace_ip_address(_Config) ->
emqtt:connect(T),
%% Start tracing
ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"),
ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"),
ok = filesync(<<"127.0.0.1">>, ip_address),
ok = filesync(<<"192.168.1.1">>, ip_address),
ok = emqx_trace_handler:install("CLI-IP-1", ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"),
ok = emqx_trace_handler:install("CLI-IP-2", ip_address,
"192.168.1.1", all, "tmp/ip_trace_y.log"),
emqx_trace:check(),
ok = filesync(<<"CLI-IP-1">>, ip_address),
ok = filesync(<<"CLI-IP-2">>, ip_address),
%% Verify the tracing file exits
?assert(filelib:is_regular("tmp/ip_trace_x.log")),
@ -183,10 +181,10 @@ t_trace_ip_address(_Config) ->
%% Get current traces
?assertMatch([#{type := ip_address, filter := "127.0.0.1",
name := <<"127.0.0.1">>,
name := <<"CLI-IP-1">>,
level := debug, dst := "tmp/ip_trace_x.log"},
#{type := ip_address, filter := "192.168.1.1",
name := <<"192.168.1.1">>,
name := <<"CLI-IP-2">>,
level := debug, dst := "tmp/ip_trace_y.log"}
],
emqx_trace_handler:running()),
@ -196,8 +194,8 @@ t_trace_ip_address(_Config) ->
emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
emqtt:subscribe(T, <<"x/y/z">>),
emqtt:unsubscribe(T, <<"x/y/z">>),
ok = filesync(<<"127.0.0.1">>, ip_address),
ok = filesync(<<"192.168.1.1">>, ip_address),
ok = filesync(<<"CLI-IP-1">>, ip_address),
ok = filesync(<<"CLI-IP-2">>, ip_address),
{ok, Bin} = file:read_file("tmp/ip_trace_x.log"),
?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@ -208,8 +206,8 @@ t_trace_ip_address(_Config) ->
?assert(filelib:file_size("tmp/ip_trace_y.log") =:= 0),
%% Stop tracing
ok = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.1">>),
ok = emqx_trace_handler:uninstall(ip_address, <<"192.168.1.1">>),
ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-1">>),
ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-2">>),
{error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>),
emqtt:disconnect(T),
?assertEqual([], emqx_trace_handler:running()).
@ -221,7 +219,12 @@ filesync(Name, Type) ->
%% sometime the handler process is not started yet.
filesync(_Name, _Type, 0) -> ok;
filesync(Name, Type, Retry) ->
filesync(Name0, Type, Retry) ->
Name =
case is_binary(Name0) of
true -> Name0;
false -> list_to_binary(Name0)
end,
try
Handler = binary_to_atom(<<"trace_",
(atom_to_binary(Type))/binary, "_", Name/binary>>),
@ -231,3 +234,9 @@ filesync(Name, Type, Retry) ->
ct:sleep(100),
filesync(Name, Type, Retry - 1)
end.
init() ->
emqx_trace:start_link().
terminate() ->
catch ok = gen_server:stop(emqx_trace, normal, 5000).

6
apps/emqx_authn/src/emqx_authn_api.erl
View File

@ -726,7 +726,7 @@ with_chain(ListenerID, Fun) ->
create_authenticator(ConfKeyPath, ChainName, Config) ->
case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of
{ok, #{post_config_update := #{emqx_authentication := #{id := ID}},
raw_config := AuthenticatorsConfig}} ->
raw_config := AuthenticatorsConfig}} ->
{ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig),
{200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))};
{error, {_PrePostConfigUpdate, emqx_authentication, Reason}} ->
@ -872,7 +872,7 @@ fill_defaults(Configs) when is_list(Configs) ->
fill_defaults(Config) ->
emqx_authn:check_config(Config, #{only_fill_defaults => true}).
convert_certs(#{ssl := SSLOpts} = Config) ->
convert_certs(#{ssl := #{enable := true} = SSLOpts} = Config) ->
NSSLOpts = lists:foldl(fun(K, Acc) ->
case maps:get(K, Acc, undefined) of
undefined -> Acc;
@ -979,7 +979,7 @@ authenticator_examples() ->
mechanism => <<"password-based">>,
backend => <<"http">>,
method => <<"post">>,
url => <<"http://127.0.0.2:8080">>,
url => <<"http://127.0.0.1:18083">>,
headers => #{
<<"content-type">> => <<"application/json">>
},

2
apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
View File

@ -106,7 +106,7 @@ authenticate(#{password := Password} = Credential,
resource_id := ResourceId,
password_hash_algorithm := Algorithm}) ->
Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential),
case emqx_resource:query(ResourceId, {sql, Query, Params}) of
case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Query, Params}) of
{ok, _Columns, []} -> ignore;
{ok, Columns, [Row | _]} ->
NColumns = [Name || #column{name = Name} <- Columns],

2
apps/emqx_authn/test/emqx_authn_api_SUITE.erl
View File

@ -67,7 +67,7 @@ init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx_authn, emqx_dashboard]),
emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authn]),
ok.
set_special_configs(emqx_dashboard) ->

5
apps/emqx_authn/test/emqx_authn_http_SUITE.erl
View File

@ -153,9 +153,8 @@ t_destroy(_Config) ->
?GLOBAL),
% Authenticator should not be usable anymore
?assertException(
error,
_,
?assertMatch(
ignore,
emqx_authn_http:authenticate(
Credentials,
State)).

5
apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl
View File

@ -146,9 +146,8 @@ t_destroy(_Config) ->
?GLOBAL),
% Authenticator should not be usable anymore
?assertException(
error,
_,
?assertMatch(
ignore,
emqx_authn_mongodb:authenticate(
#{username => <<"plain">>,
password => <<"plain">>

5
apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
View File

@ -157,9 +157,8 @@ t_destroy(_Config) ->
?GLOBAL),
% Authenticator should not be usable anymore
?assertException(
error,
_,
?assertMatch(
ignore,
emqx_authn_mysql:authenticate(
#{username => <<"plain">>,
password => <<"plain">>

9
apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
View File

@ -158,9 +158,8 @@ t_destroy(_Config) ->
?GLOBAL),
% Authenticator should not be usable anymore
?assertException(
error,
_,
?assertMatch(
ignore,
emqx_authn_pgsql:authenticate(
#{username => <<"plain">>,
password => <<"plain">>
@ -440,12 +439,12 @@ create_user(Values) ->
q(Sql) ->
emqx_resource:query(
?PGSQL_RESOURCE,
{sql, Sql}).
{query, Sql}).
q(Sql, Params) ->
emqx_resource:query(
?PGSQL_RESOURCE,
{sql, Sql, Params}).
{query, Sql, Params}).
drop_seeds() ->
{ok, _, _} = q("DROP TABLE IF EXISTS users"),

5
apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
View File

@ -164,9 +164,8 @@ t_destroy(_Config) ->
?GLOBAL),
% Authenticator should not be usable anymore
?assertException(
error,
_,
?assertMatch(
ignore,
emqx_authn_redis:authenticate(
#{username => <<"plain">>,
password => <<"plain">>

36
apps/emqx_authz/src/emqx_authz.erl
View File

@ -31,9 +31,7 @@
, lookup/0
, lookup/1
, move/2
, move/3
, update/2
, update/3
, authorize/5
]).
@ -112,28 +110,19 @@ lookup(Type) ->
{Source, _Front, _Rear} = take(Type),
Source.
move(Type, Cmd) ->
move(Type, Cmd, #{}).
move(Type, #{<<"before">> := Before}, Opts) ->
emqx:update_config( ?CONF_KEY_PATH
, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts);
move(Type, #{<<"after">> := After}, Opts) ->
emqx:update_config( ?CONF_KEY_PATH
, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts);
move(Type, Position, Opts) ->
emqx:update_config( ?CONF_KEY_PATH
, {?CMD_MOVE, type(Type), Position}, Opts).
move(Type, #{<<"before">> := Before}) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))});
move(Type, #{<<"after">> := After}) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))});
move(Type, Position) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}).
update({?CMD_REPLACE, Type}, Sources) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources});
update({?CMD_DELETE, Type}, Sources) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_DELETE, type(Type)}, Sources});
update(Cmd, Sources) ->
update(Cmd, Sources, #{}).
update({?CMD_REPLACE, Type}, Sources, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources}, Opts);
update({?CMD_DELETE, Type}, Sources, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {{?CMD_DELETE, type(Type)}, Sources}, Opts);
update(Cmd, Sources, Opts) ->
emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}, Opts).
emqx_authz_utils:update_config(?CONF_KEY_PATH, {Cmd, Sources}).
do_update({?CMD_MOVE, Type, ?CMD_MOVE_TOP}, Conf) when is_list(Conf) ->
{Source, Front, Rear} = take(Type, Conf),
@ -167,7 +156,8 @@ do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := Enable} = Source}, Conf)
NConf;
{error, _} = Error -> Error
end;
do_update({{?CMD_REPLACE, Type}, Source}, Conf) when is_map(Source), is_list(Conf) ->
do_update({{?CMD_REPLACE, Type}, Source}, Conf)
when is_map(Source), is_list(Conf) ->
{_Old, Front, Rear} = take(Type, Conf),
NConf = Front ++ [Source | Rear],
ok = check_dup_types(NConf),

12
apps/emqx_authz/src/emqx_authz_api_schema.erl
View File

@ -182,8 +182,7 @@ definitions() ->
mongo_type => #{type => string,
enum => [<<"rs">>],
example => <<"rs">>},
servers => #{type => array,
items => #{type => string,example => <<"127.0.0.1:27017">>}},
servers => #{type => string, example => <<"127.0.0.1:27017, 127.0.0.2:27017">>},
replica_set_name => #{type => string},
pool_size => #{type => integer},
username => #{type => string},
@ -240,8 +239,7 @@ definitions() ->
mongo_type => #{type => string,
enum => [<<"sharded">>],
example => <<"sharded">>},
servers => #{type => array,
items => #{type => string,example => <<"127.0.0.1:27017">>}},
servers => #{type => string,example => <<"127.0.0.1:27017, 127.0.0.2:27017">>},
pool_size => #{type => integer},
username => #{type => string},
password => #{type => string},
@ -401,8 +399,7 @@ definitions() ->
type => string,
example => <<"HGETALL mqtt_authz">>
},
servers => #{type => array,
items => #{type => string,example => <<"127.0.0.1:3306">>}},
servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>},
redis_type => #{type => string,
enum => [<<"sentinel">>],
example => <<"sentinel">>},
@ -438,8 +435,7 @@ definitions() ->
type => string,
example => <<"HGETALL mqtt_authz">>
},
servers => #{type => array,
items => #{type => string, example => <<"127.0.0.1:3306">>}},
servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>},
redis_type => #{type => string,
enum => [<<"cluster">>],
example => <<"cluster">>},

7
apps/emqx_authz/src/emqx_authz_api_settings.erl
View File

@ -54,8 +54,9 @@ settings(get, _Params) ->
settings(put, #{body := #{<<"no_match">> := NoMatch,
<<"deny_action">> := DenyAction,
<<"cache">> := Cache}}) ->
{ok, _} = emqx:update_config([authorization, no_match], NoMatch),
{ok, _} = emqx:update_config([authorization, deny_action], DenyAction),
{ok, _} = emqx:update_config([authorization, cache], Cache),
{ok, _} = emqx_authz_utils:update_config([authorization, no_match], NoMatch),
{ok, _} = emqx_authz_utils:update_config(
[authorization, deny_action], DenyAction),
{ok, _} = emqx_authz_utils:update_config([authorization, cache], Cache),
ok = emqx_authz_cache:drain_cache(),
{200, authorization_settings()}.

4
apps/emqx_authz/src/emqx_authz_http.erl
View File

@ -46,10 +46,10 @@ init(Source) ->
end.
destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id).
ok = emqx_resource:remove_local(Id).
dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_http, Source).
emqx_resource:create_dry_run_local(emqx_connector_http, Source).
authorize(Client, PubSub, Topic,
#{type := http,

4
apps/emqx_authz/src/emqx_authz_mongodb.erl
View File

@ -46,10 +46,10 @@ init(Source) ->
end.
dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_mongo, Source).
emqx_resource:create_dry_run_local(emqx_connector_mongo, Source).
destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id).
ok = emqx_resource:remove_local(Id).
authorize(Client, PubSub, Topic,
#{collection := Collection,

4
apps/emqx_authz/src/emqx_authz_mysql.erl
View File

@ -48,10 +48,10 @@ init(#{query := SQL} = Source) ->
end.
dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_mysql, Source).
emqx_resource:create_dry_run_local(emqx_connector_mysql, Source).
destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id).
ok = emqx_resource:remove_local(Id).
authorize(Client, PubSub, Topic,
#{annotations := #{id := ResourceID,

6
apps/emqx_authz/src/emqx_authz_postgresql.erl
View File

@ -48,10 +48,10 @@ init(#{query := SQL} = Source) ->
end.
destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id).
ok = emqx_resource:remove_local(Id).
dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_pgsql, Source).
emqx_resource:create_dry_run_local(emqx_connector_pgsql, Source).
parse_query(Sql) ->
case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of
@ -73,7 +73,7 @@ authorize(Client, PubSub, Topic,
query := {Query, Params}
}
}) ->
case emqx_resource:query(ResourceID, {sql, Query, replvar(Params, Client)}) of
case emqx_resource:query(ResourceID, {prepared_query, ResourceID, Query, replvar(Params, Client)}) of
{ok, _Columns, []} -> nomatch;
{ok, Columns, Rows} ->
do_authorize(Client, PubSub, Topic, Columns, Rows);

4
apps/emqx_authz/src/emqx_authz_redis.erl
View File

@ -46,10 +46,10 @@ init(Source) ->
end.
destroy(#{annotations := #{id := Id}}) ->
ok = emqx_resource:remove(Id).
ok = emqx_resource:remove_local(Id).
dry_run(Source) ->
emqx_resource:create_dry_run(emqx_connector_redis, Source).
emqx_resource:create_dry_run_local(emqx_connector_redis, Source).
authorize(Client, PubSub, Topic,
#{cmd := CMD,

16
apps/emqx_authz/src/emqx_authz_utils.erl
View File

@ -18,9 +18,11 @@
-include_lib("emqx/include/emqx_placeholder.hrl").
-export([cleanup_resources/0,
make_resource_id/1,
create_resource/2]).
-export([ cleanup_resources/0
, make_resource_id/1
, create_resource/2
, update_config/2
]).
-define(RESOURCE_GROUP, <<"emqx_authz">>).
@ -30,7 +32,7 @@
create_resource(Module, Config) ->
ResourceID = make_resource_id(Module),
case emqx_resource:create(ResourceID, Module, Config) of
case emqx_resource:create_local(ResourceID, Module, Config) of
{ok, already_created} -> {ok, ResourceID};
{ok, _} -> {ok, ResourceID};
{error, Reason} -> {error, Reason}
@ -38,13 +40,17 @@ create_resource(Module, Config) ->
cleanup_resources() ->
lists:foreach(
fun emqx_resource:remove/1,
fun emqx_resource:remove_local/1,
emqx_resource:list_group_instances(?RESOURCE_GROUP)).
make_resource_id(Name) ->
NameBin = bin(Name),
emqx_resource:generate_id(?RESOURCE_GROUP, NameBin).
update_config(Path, ConfigRequest) ->
emqx_conf:update(Path, ConfigRequest, #{rawconf_with_defaults => true,
override_to => cluster}).
%%------------------------------------------------------------------------------
%% Internal functions
%%------------------------------------------------------------------------------

8
apps/emqx_authz/test/emqx_authz_SUITE.erl
View File

@ -31,10 +31,9 @@ groups() ->
init_per_suite(Config) ->
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, remove, fun(_) -> ok end),
meck:expect(emqx_resource, create_dry_run, fun(_, _) -> ok end),
meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, remove_local, fun(_) -> ok end),
meck:expect(emqx_resource, create_dry_run_local, fun(_, _) -> ok end),
ok = emqx_common_test_helpers:start_apps(
[emqx_connector, emqx_conf, emqx_authz],
@ -105,6 +104,7 @@ set_special_configs(_App) ->
<<"query">> => <<"abcb">>
}).
-define(SOURCE5, #{<<"type">> => <<"redis">>,
<<"redis_type">> => <<"single">>,
<<"enable">> => true,
<<"server">> => <<"127.0.0.1:27017">>,
<<"pool_size">> => 1,

9
apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl
View File

@ -70,9 +70,7 @@
}).
-define(SOURCE5, #{<<"type">> => <<"redis">>,
<<"enable">> => true,
<<"servers">> => [<<"127.0.0.1:6379">>,
<<"127.0.0.1:6380">>
],
<<"servers">> => <<"127.0.0.1:6379, 127.0.0.1:6380">>,
<<"pool_size">> => 1,
<<"database">> => 0,
<<"password">> => <<"ee">>,
@ -99,13 +97,12 @@ groups() ->
init_per_suite(Config) ->
meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, create_dry_run,
meck:expect(emqx_resource, create_dry_run_local,
fun(emqx_connector_mysql, _) -> ok;
(T, C) -> meck:passthrough([T, C])
end),
meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, health_check, fun(_) -> ok end),
meck:expect(emqx_resource, remove, fun(_) -> ok end ),
meck:expect(emqx_resource, remove_local, fun(_) -> ok end ),
ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authz, emqx_dashboard],

4
apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
View File

@ -228,12 +228,12 @@ raw_pgsql_authz_config() ->
q(Sql) ->
emqx_resource:query(
?PGSQL_RESOURCE,
{sql, Sql}).
{query, Sql}).
insert(Sql, Params) ->
{ok, _} = emqx_resource:query(
?PGSQL_RESOURCE,
{sql, Sql, Params}),
{query, Sql, Params}),
ok.
init_table() ->

11
apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl
View File

@ -80,8 +80,15 @@ format(Rule = #{topic := Topic}) when is_map(Rule) ->
}.
update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE ->
{ok, _} = emqx:update_config([auto_subscribe, topics], Topics),
update_hook();
case emqx_conf:update([auto_subscribe, topics],
Topics,
#{rawconf_with_defaults => true, override_to => cluster}) of
{ok, #{raw_config := NewTopics}} ->
ok = update_hook(),
{ok, NewTopics};
{error, Reason} ->
{error, Reason}
end;
update_(_Topics) ->
{error, quota_exceeded}.

8
apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl
View File

@ -22,6 +22,7 @@
-export([auto_subscribe/2]).
-define(INTERNAL_ERROR, 'INTERNAL_ERROR').
-define(EXCEED_LIMIT, 'EXCEED_LIMIT').
-define(BAD_REQUEST, 'BAD_REQUEST').
@ -90,6 +91,9 @@ auto_subscribe(put, #{body := Params}) ->
Message = list_to_binary(io_lib:format("Max auto subscribe topic count is ~p",
[emqx_auto_subscribe:max_limit()])),
{409, #{code => ?EXCEED_LIMIT, message => Message}};
ok ->
{200, emqx_auto_subscribe:list()}
{error, Reason} ->
Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])),
{500, #{code => ?INTERNAL_ERROR, message => Message}};
{ok, NewTopics} ->
{200, NewTopics}
end.

9
apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl
View File

@ -85,7 +85,7 @@ init_per_suite(Config) ->
}
]
}">>),
emqx_common_test_helpers:start_apps([emqx_dashboard, ?APP], fun set_special_configs/1),
emqx_common_test_helpers:start_apps([emqx_dashboard, emqx_conf, ?APP], fun set_special_configs/1),
Config.
set_special_configs(emqx_dashboard) ->
@ -113,15 +113,17 @@ topic_config(T) ->
end_per_suite(_) ->
application:unload(emqx_management),
application:unload(emqx_conf),
application:unload(?APP),
meck:unload(emqx_resource),
meck:unload(emqx_schema),
emqx_common_test_helpers:stop_apps([emqx_dashboard, ?APP]).
emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_conf, ?APP]).
t_auto_subscribe(_) ->
emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]),
{ok, Client} = emqtt:start_link(#{username => ?CLIENT_USERNAME, clientid => ?CLIENT_ID}),
{ok, _} = emqtt:connect(Client),
timer:sleep(100),
timer:sleep(200),
?assertEqual(check_subs(length(?TOPICS)), ok),
emqtt:disconnect(Client),
ok.
@ -148,6 +150,7 @@ t_update(_) ->
check_subs(Count) ->
Subs = ets:tab2list(emqx_suboption),
ct:pal("---> ~p ~p ~n", [Subs, Count]),
?assert(length(Subs) >= Count),
check_subs((Subs), ?ENSURE_TOPICS).

4
apps/emqx_bridge/etc/emqx_bridge.conf
View File

@ -34,8 +34,8 @@
# direction = egress
# ## NOTE: we cannot use placehodler variables in the `scheme://host:port` part of the url
# url = "http://localhost:9901/messages/${topic}"
# request_timeout = "30s"
# connect_timeout = "30s"
# request_timeout = "15s"
# connect_timeout = "15s"
# max_retries = 3
# retry_interval = "10s"
# pool_type = "random"

79
apps/emqx_bridge/src/emqx_bridge.erl
View File

@ -35,15 +35,19 @@
]).
-export([ load/0
, lookup/1
, lookup/2
, lookup/3
, list/0
, list_bridges_by_connector/1
, create/2
, create/3
, recreate/2
, recreate/3
, create_dry_run/2
, remove/1
, remove/3
, update/2
, update/3
, start/2
, stop/2
@ -80,17 +84,36 @@ unload_hook() ->
on_message_publish(Message = #message{topic = Topic, flags = Flags}) ->
case maps:get(sys, Flags, false) of
false ->
lists:foreach(fun (Id) ->
send_message(Id, emqx_rule_events:eventmsg_publish(Message))
end, get_matched_bridges(Topic));
Msg = emqx_rule_events:eventmsg_publish(Message),
send_to_matched_egress_bridges(Topic, Msg);
true -> ok
end,
{ok, Message}.
send_to_matched_egress_bridges(Topic, Msg) ->
lists:foreach(fun (Id) ->
try send_message(Id, Msg) of
ok -> ok;
Error -> ?SLOG(error, #{msg => "send_message_to_bridge_failed",
bridge => Id, error => Error})
catch Err:Reason:ST ->
?SLOG(error, #{msg => "send_message_to_bridge_crash",
bridge => Id, error => Err, reason => Reason,
stacktrace => ST})
end
end, get_matched_bridges(Topic)).
send_message(BridgeId, Message) ->
{BridgeType, BridgeName} = parse_bridge_id(BridgeId),
ResId = emqx_bridge:resource_id(BridgeType, BridgeName),
emqx_resource:query(ResId, {send_message, Message}).
case emqx:get_config([bridges, BridgeType, BridgeName], not_found) of
not_found ->
{error, {bridge_not_found, BridgeId}};
#{enable := true} ->
emqx_resource:query(ResId, {send_message, Message});
#{enable := false} ->
{error, {bridge_stopped, BridgeId}}
end.
config_key_path() ->
[bridges].
@ -169,6 +192,10 @@ list_bridges_by_connector(ConnectorId) ->
[B || B = #{raw_config := #{<<"connector">> := Id}} <- list(),
ConnectorId =:= Id].
lookup(Id) ->
{Type, Name} = parse_bridge_id(Id),
lookup(Type, Name).
lookup(Type, Name) ->
RawConf = emqx:get_raw_config([bridges, Type, Name], #{}),
lookup(Type, Name, RawConf).
@ -188,16 +215,24 @@ stop(Type, Name) ->
restart(Type, Name) ->
emqx_resource:restart(resource_id(Type, Name)).
create(BridgeId, Conf) ->
{BridgeType, BridgeName} = parse_bridge_id(BridgeId),
create(BridgeType, BridgeName, Conf).
create(Type, Name, Conf) ->
?SLOG(info, #{msg => "create bridge", type => Type, name => Name,
config => Conf}),
case emqx_resource:create_local(resource_id(Type, Name), emqx_bridge:resource_type(Type),
parse_confs(Type, Name, Conf), #{force_create => true}) of
parse_confs(Type, Name, Conf), #{async_create => true}) of
{ok, already_created} -> maybe_disable_bridge(Type, Name, Conf);
{ok, _} -> maybe_disable_bridge(Type, Name, Conf);
{error, Reason} -> {error, Reason}
end.
update(BridgeId, {OldConf, Conf}) ->
{BridgeType, BridgeName} = parse_bridge_id(BridgeId),
update(BridgeType, BridgeName, {OldConf, Conf}).
update(Type, Name, {OldConf, Conf}) ->
%% TODO: sometimes its not necessary to restart the bridge connection.
%%
@ -214,23 +249,27 @@ update(Type, Name, {OldConf, Conf}) ->
case recreate(Type, Name, Conf) of
{ok, _} -> maybe_disable_bridge(Type, Name, Conf);
{error, not_found} ->
?SLOG(warning, #{ msg => "updating a non-exist bridge, create a new one"
?SLOG(warning, #{ msg => "updating_a_non-exist_bridge_need_create_a_new_one"
, type => Type, name => Name, config => Conf}),
create(Type, Name, Conf);
{error, Reason} -> {update_bridge_failed, Reason}
{error, Reason} -> {error, {update_bridge_failed, Reason}}
end;
true ->
%% we don't need to recreate the bridge if this config change is only to
%% toggole the config 'bridge.{type}.{name}.enable'
ok
case maps:get(enable, Conf, true) of
false -> stop(Type, Name);
true -> start(Type, Name)
end
end.
recreate(Type, Name) ->
recreate(Type, Name, emqx:get_raw_config([bridges, Type, Name])).
recreate(Type, Name, emqx:get_config([bridges, Type, Name])).
recreate(Type, Name, Conf) ->
emqx_resource:recreate_local(resource_id(Type, Name),
emqx_bridge:resource_type(Type), parse_confs(Type, Name, Conf), []).
emqx_bridge:resource_type(Type), parse_confs(Type, Name, Conf),
#{async_create => true}).
create_dry_run(Type, Conf) ->
Conf0 = Conf#{<<"ingress">> => #{<<"remote_topic">> => <<"t">>}},
@ -241,8 +280,12 @@ create_dry_run(Type, Conf) ->
Error
end.
remove(BridgeId) ->
{BridgeType, BridgeName} = parse_bridge_id(BridgeId),
remove(BridgeType, BridgeName, #{}).
remove(Type, Name, _Conf) ->
?SLOG(info, #{msg => "remove bridge", type => Type, name => Name}),
?SLOG(info, #{msg => "remove_bridge", type => Type, name => Name}),
case emqx_resource:remove_local(resource_id(Type, Name)) of
ok -> ok;
{error, not_found} -> ok;
@ -276,6 +319,8 @@ get_matched_bridges(Topic) ->
end, Acc0, Conf)
end, [], Bridges).
get_matched_bridge_id(#{enable := false}, _Topic, _BType, _BName, Acc) ->
Acc;
get_matched_bridge_id(#{local_topic := Filter}, Topic, BType, BName, Acc) ->
case emqx_topic:match(Topic, Filter) of
true -> [bridge_id(BType, BName) | Acc];
@ -306,21 +351,21 @@ parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf)
{Type, ConnName} ->
ConnectorConfs = emqx:get_config([connectors, Type, ConnName]),
make_resource_confs(Direction, ConnectorConfs,
maps:without([connector, direction], Conf), Name);
maps:without([connector, direction], Conf), Type, Name);
{_ConnType, _ConnName} ->
error({cannot_use_connector_with_different_type, ConnId})
end;
parse_confs(_Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf)
parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf)
when is_map(ConnectorConfs) ->
make_resource_confs(Direction, ConnectorConfs,
maps:without([connector, direction], Conf), Name).
maps:without([connector, direction], Conf), Type, Name).
make_resource_confs(ingress, ConnectorConfs, BridgeConf, Name) ->
BName = bin(Name),
make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) ->
BName = bridge_id(Type, Name),
ConnectorConfs#{
ingress => BridgeConf#{hookpoint => <<"$bridges/", BName/binary>>}
};
make_resource_confs(egress, ConnectorConfs, BridgeConf, _Name) ->
make_resource_confs(egress, ConnectorConfs, BridgeConf, _Type, _Name) ->
ConnectorConfs#{
egress => BridgeConf
}.

17
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -158,8 +158,8 @@ method_example(_Type, _Direction, put) ->
info_example_basic(http, _) ->
#{
url => <<"http://localhost:9901/messages/${topic}">>,
request_timeout => <<"30s">>,
connect_timeout => <<"30s">>,
request_timeout => <<"15s">>,
connect_timeout => <<"15s">>,
max_retries => 3,
retry_interval => <<"10s">>,
pool_type => <<"random">>,
@ -276,7 +276,7 @@ schema("/bridges/:id/operation/:operation") ->
'/bridges'(post, #{body := #{<<"type">> := BridgeType} = Conf0}) ->
Conf = filter_out_request_body(Conf0),
BridgeName = maps:get(<<"name">>, Conf, emqx_misc:gen_id()),
BridgeName = emqx_misc:gen_id(),
case emqx_bridge:lookup(BridgeType, BridgeName) of
{ok, _} ->
{400, error_msg('ALREADY_EXISTS', <<"bridge already exists">>)};
@ -356,9 +356,8 @@ operation_to_conf_req(<<"restart">>) -> restart;
operation_to_conf_req(_) -> invalid.
ensure_bridge_created(BridgeType, BridgeName, Conf) ->
Conf1 = maps:without([<<"type">>, <<"name">>], Conf),
case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName],
Conf1, #{override_to => cluster}) of
Conf, #{override_to => cluster}) of
{ok, _} -> ok;
{error, Reason} ->
{error, error_msg('BAD_ARG', Reason)}
@ -411,12 +410,12 @@ aggregate_metrics(AllMetrics) ->
format_resp(#{id := Id, raw_config := RawConf,
resource_data := #{status := Status, metrics := Metrics}}) ->
{Type, Name} = emqx_bridge:parse_bridge_id(Id),
{Type, BridgeName} = emqx_bridge:parse_bridge_id(Id),
IsConnected = fun(started) -> connected; (_) -> disconnected end,
RawConf#{
id => Id,
type => Type,
name => Name,
name => maps:get(<<"name">>, RawConf, BridgeName),
node => node(),
status => IsConnected(Status),
metrics => Metrics
@ -431,8 +430,8 @@ rpc_multicall(Func, Args) ->
end.
filter_out_request_body(Conf) ->
ExtraConfs = [<<"id">>, <<"status">>, <<"node_status">>, <<"node_metrics">>,
<<"metrics">>, <<"node">>],
ExtraConfs = [<<"id">>, <<"type">>, <<"status">>, <<"node_status">>,
<<"node_metrics">>, <<"metrics">>, <<"node">>],
maps:without(ExtraConfs, Conf).
rpc_call(Node, Fun, Args) ->

11
apps/emqx_bridge/src/emqx_bridge_http_schema.erl
View File

@ -59,7 +59,7 @@ Template with variables is allowed.
"""
})}
, {request_timeout, mk(emqx_schema:duration_ms(),
#{ default => <<"30s">>
#{ default => <<"15s">>
, desc =>"""
How long will the HTTP request timeout.
"""
@ -68,7 +68,6 @@ How long will the HTTP request timeout.
fields("post") ->
[ type_field()
, name_field()
] ++ fields("bridge");
fields("put") ->
@ -84,9 +83,14 @@ basic_config() ->
#{ desc => "Enable or disable this bridge"
, default => true
})}
, {name,
mk(binary(),
#{ desc => "Bridge name, used as a human-readable description of the bridge."
})}
, {direction,
mk(egress,
#{ desc => "The direction of this bridge, MUST be egress"
, default => egress
})}
]
++ proplists:delete(base_url, emqx_connector_http:fields(config)).
@ -98,8 +102,5 @@ id_field() ->
type_field() ->
{type, mk(http, #{desc => "The Bridge Type"})}.
name_field() ->
{name, mk(binary(), #{desc => "The Bridge Name"})}.
method() ->
enum([post, put, get, delete]).

8
apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl
View File

@ -24,11 +24,9 @@ fields("egress") ->
fields("post_ingress") ->
[ type_field()
, name_field()
] ++ proplists:delete(enable, fields("ingress"));
fields("post_egress") ->
[ type_field()
, name_field()
] ++ proplists:delete(enable, fields("egress"));
fields("put_ingress") ->
@ -49,9 +47,3 @@ id_field() ->
type_field() ->
{type, mk(mqtt, #{desc => "The Bridge Type"})}.
name_field() ->
{name, mk(binary(),
#{ desc => "The Bridge Name"
, example => "some_bridge_name"
})}.

7
apps/emqx_bridge/src/emqx_bridge_schema.erl
View File

@ -43,9 +43,13 @@ http_schema(Method) ->
common_bridge_fields() ->
[ {enable,
mk(boolean(),
#{ desc =>"Enable or disable this bridge"
#{ desc => "Enable or disable this bridge"
, default => true
})}
, {name,
mk(binary(),
#{ desc => "Bridge name, used as a human-readable description of the bridge."
})}
, {connector,
mk(binary(),
#{ nullable => false
@ -71,6 +75,7 @@ metrics_status_fields() ->
direction_field(Dir, Desc) ->
{direction, mk(Dir,
#{ nullable => false
, default => egress
, desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.<br>"
++ Desc
})}.

191
apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
View File

@ -23,12 +23,13 @@
-define(CONF_DEFAULT, <<"bridges: {}">>).
-define(BRIDGE_TYPE, <<"http">>).
-define(BRIDGE_NAME, <<"test_bridge">>).
-define(BRIDGE_ID, <<"http:test_bridge">>).
-define(URL(PORT, PATH), list_to_binary(
io_lib:format("http://localhost:~s/~s",
[integer_to_list(PORT), PATH]))).
-define(HTTP_BRIDGE(URL),
-define(HTTP_BRIDGE(URL, TYPE, NAME),
#{
<<"type">> => TYPE,
<<"name">> => NAME,
<<"url">> => URL,
<<"local_topic">> => <<"emqx_http/#">>,
<<"method">> => <<"post">>,
@ -47,7 +48,7 @@ groups() ->
[].
suite() ->
[{timetrap,{seconds,30}}].
[{timetrap,{seconds,60}}].
init_per_suite(Config) ->
ok = emqx_config:put([emqx_dashboard], #{
@ -84,7 +85,7 @@ start_http_server(HandleFun) ->
spawn_link(fun() ->
{Port, Sock} = listen_on_random_port(),
Parent ! {port, Port},
loop(Sock, HandleFun)
loop(Sock, HandleFun, Parent)
end),
receive
{port, Port} -> Port
@ -95,40 +96,49 @@ start_http_server(HandleFun) ->
listen_on_random_port() ->
Min = 1024, Max = 65000,
Port = rand:uniform(Max - Min) + Min,
case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}]) of
case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}, binary]) of
{ok, Sock} -> {Port, Sock};
{error, eaddrinuse} -> listen_on_random_port()
end.
loop(Sock, HandleFun) ->
loop(Sock, HandleFun, Parent) ->
{ok, Conn} = gen_tcp:accept(Sock),
Handler = spawn(fun () -> HandleFun(Conn) end),
Handler = spawn(fun () -> HandleFun(Conn, Parent) end),
gen_tcp:controlling_process(Conn, Handler),
loop(Sock, HandleFun).
loop(Sock, HandleFun, Parent).
make_response(CodeStr, Str) ->
B = iolist_to_binary(Str),
iolist_to_binary(
io_lib:fwrite(
"HTTP/1.0 ~s\nContent-Type: text/html\nContent-Length: ~p\n\n~s",
"HTTP/1.0 ~s\r\nContent-Type: text/html\r\nContent-Length: ~p\r\n\r\n~s",
[CodeStr, size(B), B])).
handle_fun_200_ok(Conn) ->
handle_fun_200_ok(Conn, Parent) ->
case gen_tcp:recv(Conn, 0) of
{ok, Request} ->
{ok, ReqStr} ->
ct:pal("the http handler got request: ~p", [ReqStr]),
Req = parse_http_request(ReqStr),
Parent ! {http_server, received, Req},
gen_tcp:send(Conn, make_response("200 OK", "Request OK")),
self() ! {http_server, received, Request},
handle_fun_200_ok(Conn);
handle_fun_200_ok(Conn, Parent);
{error, closed} ->
gen_tcp:close(Conn)
end.
parse_http_request(ReqStr0) ->
[Method, ReqStr1] = string:split(ReqStr0, " ", leading),
[Path, ReqStr2] = string:split(ReqStr1, " ", leading),
[_ProtoVsn, ReqStr3] = string:split(ReqStr2, "\r\n", leading),
[_HeaderStr, Body] = string:split(ReqStr3, "\r\n\r\n", leading),
#{method => Method, path => Path, body => Body}.
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_http_crud_apis(_) ->
Port = start_http_server(fun handle_fun_200_ok/1),
Port = start_http_server(fun handle_fun_200_ok/2),
%% assert we there's no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
@ -136,38 +146,39 @@ t_http_crud_apis(_) ->
%% POST /bridges/ will create a bridge
URL1 = ?URL(Port, "path1"),
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?HTTP_BRIDGE(URL1)#{
<<"type">> => ?BRIDGE_TYPE,
<<"name">> => ?BRIDGE_NAME
}),
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
%ct:pal("---bridge: ~p", [Bridge]),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
, <<"node_status">> := [_|_]
, <<"metrics">> := _
, <<"node_metrics">> := [_|_]
, <<"url">> := URL1
}, jsx:decode(Bridge)),
%% create a again returns an error
{ok, 400, RetMsg} = request(post, uri(["bridges"]),
?HTTP_BRIDGE(URL1)#{
<<"type">> => ?BRIDGE_TYPE,
<<"name">> => ?BRIDGE_NAME
}),
?assertMatch(
#{ <<"code">> := _
, <<"message">> := <<"bridge already exists">>
}, jsx:decode(RetMsg)),
#{ <<"id">> := BridgeID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
, <<"node_status">> := [_|_]
, <<"metrics">> := _
, <<"node_metrics">> := [_|_]
, <<"url">> := URL1
} = jsx:decode(Bridge),
%% send an message to emqx and the message should be forwarded to the HTTP server
wait_for_resource_ready(BridgeID, 5),
Body = <<"my msg">>,
emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)),
?assert(
receive
{http_server, received, #{method := <<"POST">>, path := <<"/path1">>,
body := Body}} ->
true;
Msg ->
ct:pal("error: http got unexpected request: ~p", [Msg]),
false
after 100 ->
false
end),
%% update the request-path of the bridge
URL2 = ?URL(Port, "path2"),
{ok, 200, Bridge2} = request(put, uri(["bridges", ?BRIDGE_ID]),
?HTTP_BRIDGE(URL2)),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
{ok, 200, Bridge2} = request(put, uri(["bridges", BridgeID]),
?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
?assertMatch(#{ <<"id">> := BridgeID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
@ -179,7 +190,7 @@ t_http_crud_apis(_) ->
%% list all bridges again, assert Bridge2 is in it
{ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []),
?assertMatch([#{ <<"id">> := ?BRIDGE_ID
?assertMatch([#{ <<"id">> := BridgeID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
@ -190,8 +201,8 @@ t_http_crud_apis(_) ->
}], jsx:decode(Bridge2Str)),
%% get the bridge by id
{ok, 200, Bridge3Str} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
{ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := BridgeID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
@ -201,13 +212,27 @@ t_http_crud_apis(_) ->
, <<"url">> := URL2
}, jsx:decode(Bridge3Str)),
%% send an message to emqx again, check the path has been changed
wait_for_resource_ready(BridgeID, 5),
emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)),
?assert(
receive
{http_server, received, #{path := <<"/path2">>}} ->
true;
Msg2 ->
ct:pal("error: http got unexpected request: ~p", [Msg2]),
false
after 100 ->
false
end),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% update a deleted bridge returns an error
{ok, 404, ErrMsg2} = request(put, uri(["bridges", ?BRIDGE_ID]),
?HTTP_BRIDGE(URL2)),
{ok, 404, ErrMsg2} = request(put, uri(["bridges", BridgeID]),
?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
?assertMatch(
#{ <<"code">> := _
, <<"message">> := <<"bridge not found">>
@ -215,52 +240,51 @@ t_http_crud_apis(_) ->
ok.
t_start_stop_bridges(_) ->
Port = start_http_server(fun handle_fun_200_ok/1),
%% assert we there's no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
Port = start_http_server(fun handle_fun_200_ok/2),
URL1 = ?URL(Port, "abc"),
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?HTTP_BRIDGE(URL1)#{
<<"type">> => ?BRIDGE_TYPE,
<<"name">> => ?BRIDGE_NAME
}),
?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
%ct:pal("the bridge ==== ~p", [Bridge]),
?assertMatch(
#{ <<"id">> := ?BRIDGE_ID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
, <<"node_status">> := [_|_]
, <<"metrics">> := _
, <<"node_metrics">> := [_|_]
, <<"url">> := URL1
}, jsx:decode(Bridge)),
#{ <<"id">> := BridgeID
, <<"type">> := ?BRIDGE_TYPE
, <<"name">> := ?BRIDGE_NAME
, <<"status">> := _
, <<"node_status">> := [_|_]
, <<"metrics">> := _
, <<"node_metrics">> := [_|_]
, <<"url">> := URL1
} = jsx:decode(Bridge),
%% stop it
{ok, 200, <<>>} = request(post, operation_path(stop), <<"">>),
{ok, 200, Bridge2} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
{ok, 200, <<>>} = request(post, operation_path(stop, BridgeID), <<"">>),
{ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := BridgeID
, <<"status">> := <<"disconnected">>
}, jsx:decode(Bridge2)),
%% start again
{ok, 200, <<>>} = request(post, operation_path(start), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
{ok, 200, <<>>} = request(post, operation_path(start, BridgeID), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := BridgeID
, <<"status">> := <<"connected">>
}, jsx:decode(Bridge3)),
%% restart an already started bridge
{ok, 200, <<>>} = request(post, operation_path(restart), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
{ok, 200, <<>>} = request(post, operation_path(restart, BridgeID), <<"">>),
{ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := BridgeID
, <<"status">> := <<"connected">>
}, jsx:decode(Bridge3)),
%% stop it again
{ok, 200, <<>>} = request(post, operation_path(stop), <<"">>),
{ok, 200, <<>>} = request(post, operation_path(stop, BridgeID), <<"">>),
%% restart a stopped bridge
{ok, 200, <<>>} = request(post, operation_path(restart), <<"">>),
{ok, 200, Bridge4} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID
{ok, 200, <<>>} = request(post, operation_path(restart, BridgeID), <<"">>),
{ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []),
?assertMatch(#{ <<"id">> := BridgeID
, <<"status">> := <<"connected">>
}, jsx:decode(Bridge4)),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []).
%%--------------------------------------------------------------------
@ -296,5 +320,16 @@ auth_header_() ->
{ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
{"Authorization", "Bearer " ++ binary_to_list(Token)}.
operation_path(Oper) ->
uri(["bridges", ?BRIDGE_ID, "operation", Oper]).
operation_path(Oper, BridgeID) ->
uri(["bridges", BridgeID, "operation", Oper]).
wait_for_resource_ready(InstId, 0) ->
ct:pal("--- bridge ~p: ~p", [InstId, emqx_bridge:lookup(InstId)]),
ct:fail(wait_resource_timeout);
wait_for_resource_ready(InstId, Retry) ->
case emqx_bridge:lookup(InstId) of
{ok, #{resource_data := #{status := started}}} -> ok;
_ ->
timer:sleep(100),
wait_for_resource_ready(InstId, Retry-1)
end.

6
apps/emqx_conf/src/emqx_cluster_rpc.erl
View File

@ -236,7 +236,7 @@ catch_up(#{node := Node, retry_interval := RetryMs} = State, SkipResult) ->
false -> RetryMs
end;
{aborted, Reason} ->
?SLOG(error, #{msg => "read_next_mfa transaction failed", error => Reason}),
?SLOG(error, #{msg => "read_next_mfa_transaction_failed", error => Reason}),
RetryMs
end.
@ -248,7 +248,7 @@ read_next_mfa(Node) ->
TnxId = max(LatestId - 1, 0),
commit(Node, TnxId),
?SLOG(notice, #{
msg => "New node first catch up and start commit.",
msg => "new_node_first_catch_up_and_start_commit.",
node => Node, tnx_id => TnxId}),
TnxId;
[#cluster_rpc_commit{tnx_id = LastAppliedID}] -> LastAppliedID + 1
@ -277,7 +277,7 @@ do_catch_up(ToTnxId, Node) ->
io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)",
[Node, LastAppliedId, ToTnxId])),
?SLOG(error, #{
msg => "catch up failed!",
msg => "catch_up_failed!",
last_applied_id => LastAppliedId,
to_tnx_id => ToTnxId
}),

2
apps/emqx_conf/src/emqx_conf.erl
View File

@ -144,7 +144,7 @@ multicall(M, F, Args) ->
{retry, TnxId, Res, Nodes} ->
%% The init MFA return ok, but other nodes failed.
%% We return ok and alert an alarm.
?SLOG(error, #{msg => "failed to update config in cluster", nodes => Nodes,
?SLOG(error, #{msg => "failed_to_update_config_in_cluster", nodes => Nodes,
tnx_id => TnxId, mfa => {M, F, Args}}),
Res;
{error, Error} -> %% all MFA return not ok or {ok, term()}.

11
apps/emqx_conf/src/emqx_conf_schema.erl
View File

@ -792,16 +792,7 @@ do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth) ->
}};
do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth) ->
{emqx_logger_textfmt,
#{template =>
[time," [",level,"] ",
{clientid,
[{peername,
[clientid,"@",peername," "],
[clientid, " "]}],
[{peername,
[peername," "],
[]}]},
msg,"\n"],
#{template => [time," [",level,"] ", msg,"\n"],
chars_limit => CharsLimit,
single_line => SingleLine,
time_offset => TimeOffSet,

14
apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl
View File

@ -74,9 +74,19 @@ t_base_test(_Config) ->
?assertEqual(node(), maps:get(initiator, Query)),
?assert(maps:is_key(created_at, Query)),
?assertEqual(ok, receive_msg(3, test)),
?assertEqual({ok, 2, ok}, emqx_cluster_rpc:multicall(M, F, A)),
{atomic, Status} = emqx_cluster_rpc:status(),
?assertEqual(3, length(Status)),
?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 1 end, Status)),
case length(Status) =:= 3 of
true -> ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status));
false ->
%% wait for mnesia to write in.
ct:sleep(42),
{atomic, Status1} = emqx_cluster_rpc:status(),
ct:pal("status: ~p", Status),
ct:pal("status1: ~p", Status1),
?assertEqual(3, length(Status1)),
?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status))
end,
ok.
t_commit_fail_test(_Config) ->

2
apps/emqx_connector/rebar.config
View File

@ -7,7 +7,7 @@
{emqx, {path, "../emqx"}},
{eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
{mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.6.0"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.1"}}},
%% NOTE: mind poolboy version when updating mongodb-erlang version
{mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.11"}}},
%% NOTE: mind poolboy version when updating eredis_cluster version

38
apps/emqx_connector/src/emqx_connector.erl
View File

@ -37,31 +37,26 @@
config_key_path() ->
[connectors].
-dialyzer([{nowarn_function, [post_config_update/5]}, error_handling]).
post_config_update([connectors, Type, Name], '$remove', _, _OldConf, _AppEnvs) ->
ConnId = connector_id(Type, Name),
LinkedBridgeIds = lists:foldl(fun
(#{id := BId, raw_config := #{<<"connector">> := ConnId0}}, Acc)
when ConnId0 == ConnId ->
[BId | Acc];
(_, Acc) -> Acc
end, [], emqx_bridge:list()),
case LinkedBridgeIds of
[] -> ok;
_ -> {error, {dependency_bridges_exist, LinkedBridgeIds}}
try foreach_linked_bridges(ConnId, fun(#{id := BId}) ->
throw({dependency_bridges_exist, BId})
end)
catch throw:Error -> {error, Error}
end;
post_config_update([connectors, Type, Name], _Req, NewConf, _OldConf, _AppEnvs) ->
post_config_update([connectors, Type, Name], _Req, NewConf, OldConf, _AppEnvs) ->
ConnId = connector_id(Type, Name),
lists:foreach(fun
(#{id := BId, raw_config := #{<<"connector">> := ConnId0}}) when ConnId0 == ConnId ->
foreach_linked_bridges(ConnId,
fun(#{id := BId}) ->
{BType, BName} = emqx_bridge:parse_bridge_id(BId),
BridgeConf = emqx:get_config([bridges, BType, BName]),
case emqx_bridge:recreate(BType, BName, BridgeConf#{connector => NewConf}) of
{ok, _} -> ok;
case emqx_bridge:update(BType, BName, {BridgeConf#{connector => OldConf},
BridgeConf#{connector => NewConf}}) of
ok -> ok;
{error, Reason} -> error({update_bridge_error, Reason})
end;
(_) ->
ok
end, emqx_bridge:list()).
end
end).
connector_id(Type0, Name0) ->
Type = bin(Type0),
@ -112,3 +107,10 @@ delete(Type, Name) ->
bin(Bin) when is_binary(Bin) -> Bin;
bin(Str) when is_list(Str) -> list_to_binary(Str);
bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
foreach_linked_bridges(ConnId, Do) ->
lists:foreach(fun
(#{raw_config := #{<<"connector">> := ConnId0}} = Bridge) when ConnId0 == ConnId ->
Do(Bridge);
(_) -> ok
end, emqx_bridge:list()).

21
apps/emqx_connector/src/emqx_connector_api.erl
View File

@ -107,14 +107,14 @@ info_example_basic(mqtt) ->
#{
mode => cluster_shareload,
server => <<"127.0.0.1:1883">>,
reconnect_interval => <<"30s">>,
reconnect_interval => <<"15s">>,
proto_ver => <<"v4">>,
username => <<"foo">>,
password => <<"bar">>,
clientid => <<"foo">>,
clean_start => true,
keepalive => <<"300s">>,
retry_interval => <<"30s">>,
retry_interval => <<"15s">>,
max_inflight => 100,
ssl => #{
enable => false
@ -155,8 +155,7 @@ schema("/connectors") ->
},
post => #{
tags => [<<"connectors">>],
description => <<"Create a new connector by given Id <br>"
"The ID must be of format '{type}:{name}'">>,
description => <<"Create a new connector">>,
summary => <<"Create connector">>,
requestBody => post_request_body_schema(),
responses => #{
@ -212,13 +211,13 @@ schema("/connectors/:id") ->
{200, [format_resp(Conn) || Conn <- emqx_connector:list()]};
'/connectors'(post, #{body := #{<<"type">> := ConnType} = Params}) ->
ConnName = maps:get(<<"name">>, Params, emqx_misc:gen_id()),
ConnName = emqx_misc:gen_id(),
case emqx_connector:lookup(ConnType, ConnName) of
{ok, _} ->
{400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)};
{error, not_found} ->
case emqx_connector:update(ConnType, ConnName,
maps:without([<<"type">>, <<"name">>], Params)) of
filter_out_request_body(Params)) of
{ok, #{raw_config := RawConf}} ->
Id = emqx_connector:connector_id(ConnType, ConnName),
{201, format_resp(Id, RawConf)};
@ -254,6 +253,10 @@ schema("/connectors/:id") ->
{ok, _} ->
case emqx_connector:delete(ConnType, ConnName) of
{ok, _} -> {204};
{error, {post_config_update, _, {dependency_bridges_exist, BridgeID}}} ->
{403, error_msg('DEPENDENCY_EXISTS',
<<"Cannot remove the connector as it's in use by a bridge: ",
BridgeID/binary>>)};
{error, Error} -> {400, error_msg('BAD_ARG', Error)}
end;
{error, not_found} ->
@ -270,16 +273,16 @@ format_resp(#{<<"id">> := Id} = RawConf) ->
format_resp(ConnId, RawConf) ->
NumOfBridges = length(emqx_bridge:list_bridges_by_connector(ConnId)),
{Type, Name} = emqx_connector:parse_connector_id(ConnId),
{Type, ConnName} = emqx_connector:parse_connector_id(ConnId),
RawConf#{
<<"id">> => ConnId,
<<"type">> => Type,
<<"name">> => Name,
<<"name">> => maps:get(<<"name">>, RawConf, ConnName),
<<"num_of_bridges">> => NumOfBridges
}.
filter_out_request_body(Conf) ->
ExtraConfs = [<<"num_of_bridges">>, <<"type">>, <<"name">>],
ExtraConfs = [<<"clientid">>, <<"num_of_bridges">>, <<"type">>],
maps:without(ExtraConfs, Conf).
bin(S) when is_list(S) ->

62
apps/emqx_connector/src/emqx_connector_http.erl
View File

@ -75,7 +75,7 @@ For example: http://localhost:9901/
})}
, {connect_timeout,
sc(emqx_schema:duration_ms(),
#{ default => "30s"
#{ default => "15s"
, desc => "The timeout when connecting to the HTTP server"
})}
, {max_retries,
@ -143,7 +143,7 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
retry_interval := RetryInterval,
pool_type := PoolType,
pool_size := PoolSize} = Config) ->
?SLOG(info, #{msg => "starting http connector",
?SLOG(info, #{msg => "starting_http_connector",
connector => InstId, config => Config}),
{Transport, TransportOpts} = case Scheme of
http ->
@ -181,13 +181,13 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
end.
on_stop(InstId, #{pool_name := PoolName}) ->
?SLOG(info, #{msg => "stopping http connector",
?SLOG(info, #{msg => "stopping_http_connector",
connector => InstId}),
ehttpc_sup:stop_pool(PoolName).
on_query(InstId, {send_message, Msg}, AfterQuery, State) ->
case maps:get(request, State, undefined) of
undefined -> ?SLOG(error, #{msg => "request not found", connector => InstId});
undefined -> ?SLOG(error, #{msg => "request_not_found", connector => InstId});
Request ->
#{method := Method, path := Path, body := Body, headers := Headers,
request_timeout := Timeout} = process_request(Request, Msg),
@ -199,23 +199,32 @@ on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) ->
on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State);
on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery,
#{pool_name := PoolName, base_path := BasePath} = State) ->
?SLOG(debug, #{msg => "http connector received request",
request => Request, connector => InstId,
state => State}),
NRequest = update_path(BasePath, Request),
Name = case KeyOrNum of
undefined -> PoolName;
_ -> {PoolName, KeyOrNum}
end,
Result = ehttpc:request(Name, Method, NRequest, Timeout),
case Result of
?TRACE("QUERY", "http_connector_received",
#{request => Request, connector => InstId, state => State}),
NRequest = formalize_request(Method, BasePath, Request),
case Result = ehttpc:request(case KeyOrNum of
undefined -> PoolName;
_ -> {PoolName, KeyOrNum}
end, Method, NRequest, Timeout) of
{error, Reason} ->
?SLOG(error, #{msg => "http connector do reqeust failed",
?SLOG(error, #{msg => "http_connector_do_reqeust_failed",
request => NRequest, reason => Reason,
connector => InstId}),
emqx_resource:query_failed(AfterQuery);
_ ->
emqx_resource:query_success(AfterQuery)
{ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 ->
emqx_resource:query_success(AfterQuery);
{ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 ->
emqx_resource:query_success(AfterQuery);
{ok, StatusCode, _} ->
?SLOG(error, #{msg => "http connector do reqeust, received error response",
request => NRequest, connector => InstId,
status_code => StatusCode}),
emqx_resource:query_failed(AfterQuery);
{ok, StatusCode, _, _} ->
?SLOG(error, #{msg => "http connector do reqeust, received error response",
request => NRequest, connector => InstId,
status_code => StatusCode}),
emqx_resource:query_failed(AfterQuery)
end,
Result.
@ -268,11 +277,16 @@ process_request(#{
} = Conf, Msg) ->
Conf#{ method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg))
, path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg)
, body => emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg)
, body => process_request_body(BodyTks, Msg)
, headers => maps:to_list(proc_headers(HeadersTks, Msg))
, request_timeout => ReqTimeout
}.
process_request_body([], Msg) ->
emqx_json:encode(Msg);
process_request_body(BodyTks, Msg) ->
emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg).
proc_headers(HeaderTks, Msg) ->
maps:fold(fun(K, V, Acc) ->
Acc#{emqx_plugin_libs_rule:proc_tmpl(K, Msg) =>
@ -296,10 +310,14 @@ check_ssl_opts(URLFrom, Conf) ->
{_, _} -> false
end.
update_path(BasePath, {Path, Headers}) ->
{filename:join(BasePath, Path), Headers};
update_path(BasePath, {Path, Headers, Body}) ->
{filename:join(BasePath, Path), Headers, Body}.
formalize_request(Method, BasePath, {Path, Headers, _Body})
when Method =:= get; Method =:= delete ->
formalize_request(Method, BasePath, {Path, Headers});
formalize_request(_Method, BasePath, {Path, Headers, Body}) ->
{filename:join(BasePath, Path), Headers, Body};
formalize_request(_Method, BasePath, {Path, Headers}) ->
{filename:join(BasePath, Path), Headers}.
bin(Bin) when is_binary(Bin) ->
Bin;

14
apps/emqx_connector/src/emqx_connector_ldap.erl
View File

@ -55,7 +55,7 @@ on_start(InstId, #{servers := Servers0,
pool_size := PoolSize,
auto_reconnect := AutoReconn,
ssl := SSL} = Config) ->
?SLOG(info, #{msg => "starting ldap connector",
?SLOG(info, #{msg => "starting_ldap_connector",
connector => InstId, config => Config}),
Servers = [begin proplists:get_value(host, S) end || S <- Servers0],
SslOpts = case maps:get(enable, SSL) of
@ -81,23 +81,21 @@ on_start(InstId, #{servers := Servers0,
{ok, #{poolname => PoolName}}.
on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping ldap connector",
?SLOG(info, #{msg => "stopping_ldap_connector",
connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) ->
Request = {Base, Filter, Attributes},
?SLOG(debug, #{msg => "ldap connector received request",
request => Request, connector => InstId,
state => State}),
?TRACE("QUERY", "ldap_connector_received",
#{request => Request, connector => InstId, state => State}),
case Result = ecpool:pick_and_do(
PoolName,
{?MODULE, search, [Base, Filter, Attributes]},
no_handover) of
{error, Reason} ->
?SLOG(error, #{msg => "ldap connector do request failed",
request => Request, connector => InstId,
reason => Reason}),
?SLOG(error, #{msg => "ldap_connector_do_request_failed",
request => Request, connector => InstId, reason => Reason}),
emqx_resource:query_failed(AfterQuery);
_ ->
emqx_resource:query_success(AfterQuery)

9
apps/emqx_connector/src/emqx_connector_mongo.erl
View File

@ -129,7 +129,7 @@ on_start(InstId, Config = #{mongo_type := Type,
{ok, #{poolname => PoolName, type => Type}}.
on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping mongodb connector",
?SLOG(info, #{msg => "stopping_mongodb_connector",
connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
@ -138,14 +138,13 @@ on_query(InstId,
AfterQuery,
#{poolname := PoolName} = State) ->
Request = {Action, Collection, Selector, Docs},
?SLOG(debug, #{msg => "mongodb connector received request",
request => Request, connector => InstId,
state => State}),
?TRACE("QUERY", "mongodb_connector_received",
#{request => Request, connector => InstId, state => State}),
case ecpool:pick_and_do(PoolName,
{?MODULE, mongo_query, [Action, Collection, Selector, Docs]},
no_handover) of
{error, Reason} ->
?SLOG(error, #{msg => "mongodb connector do query failed",
?SLOG(error, #{msg => "mongodb_connector_do_query_failed",
request => Request, reason => Reason,
connector => InstId}),
emqx_resource:query_failed(AfterQuery),

42
apps/emqx_connector/src/emqx_connector_mqtt.erl
View File

@ -29,7 +29,7 @@
, bridges/0
]).
-export([on_message_received/2]).
-export([on_message_received/3]).
%% callbacks of behaviour emqx_resource
-export([ on_start/2
@ -68,10 +68,6 @@ fields("put") ->
fields("post") ->
[ {type, mk(mqtt, #{desc => "The Connector Type"})}
, {name, mk(binary(),
#{ desc => "The Connector Name"
, example => <<"my_mqtt_connector">>
})}
] ++ fields("put").
%% ===================================================================
@ -105,26 +101,29 @@ drop_bridge(Name) ->
case supervisor:terminate_child(?MODULE, Name) of
ok ->
supervisor:delete_child(?MODULE, Name);
{error, not_found} ->
ok;
{error, Error} ->
{error, Error}
end.
%% ===================================================================
%% When use this bridge as a data source, ?MODULE:on_message_received/2 will be called
%% When use this bridge as a data source, ?MODULE:on_message_received will be called
%% if the bridge received msgs from the remote broker.
on_message_received(Msg, HookPoint) ->
on_message_received(Msg, HookPoint, InstId) ->
_ = emqx_resource:query(InstId, {message_received, Msg}),
emqx:run_hook(HookPoint, [Msg]).
%% ===================================================================
on_start(InstId, Conf) ->
InstanceId = binary_to_atom(InstId, utf8),
?SLOG(info, #{msg => "starting mqtt connector",
?SLOG(info, #{msg => "starting_mqtt_connector",
connector => InstanceId, config => Conf}),
BasicConf = basic_config(Conf),
BridgeConf = BasicConf#{
name => InstanceId,
clientid => clientid(maps:get(clientid, Conf, InstId)),
subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined)),
clientid => clientid(InstId),
subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined), InstId),
forwards => make_forward_confs(maps:get(egress, Conf, undefined))
},
case ?MODULE:create_bridge(BridgeConf) of
@ -139,19 +138,21 @@ on_start(InstId, Conf) ->
end.
on_stop(_InstId, #{name := InstanceId}) ->
?SLOG(info, #{msg => "stopping mqtt connector",
?SLOG(info, #{msg => "stopping_mqtt_connector",
connector => InstanceId}),
case ?MODULE:drop_bridge(InstanceId) of
ok -> ok;
{error, not_found} -> ok;
{error, Reason} ->
?SLOG(error, #{msg => "stop mqtt connector",
?SLOG(error, #{msg => "stop_mqtt_connector",
connector => InstanceId, reason => Reason})
end.
on_query(_InstId, {message_received, _Msg}, AfterQuery, _State) ->
emqx_resource:query_success(AfterQuery);
on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) ->
?SLOG(debug, #{msg => "send msg to remote node", message => Msg,
connector => InstanceId}),
?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
emqx_resource:query_success(AfterQuery).
@ -167,15 +168,15 @@ ensure_mqtt_worker_started(InstanceId) ->
{error, Reason} -> {error, Reason}
end.
make_sub_confs(EmptyMap) when map_size(EmptyMap) == 0 ->
make_sub_confs(EmptyMap, _) when map_size(EmptyMap) == 0 ->
undefined;
make_sub_confs(undefined) ->
make_sub_confs(undefined, _) ->
undefined;
make_sub_confs(SubRemoteConf) ->
make_sub_confs(SubRemoteConf, InstId) ->
case maps:take(hookpoint, SubRemoteConf) of
error -> SubRemoteConf;
{HookPoint, SubConf} ->
MFA = {?MODULE, on_message_received, [HookPoint]},
MFA = {?MODULE, on_message_received, [HookPoint, InstId]},
SubConf#{on_message_received => MFA}
end.
@ -208,7 +209,7 @@ basic_config(#{
username => User,
password => Password,
clean_start => CleanStart,
keepalive => KeepAlive,
keepalive => ms_to_s(KeepAlive),
retry_interval => RetryIntv,
max_inflight => MaxInflight,
ssl => EnableSsl,
@ -216,5 +217,8 @@ basic_config(#{
if_record_metrics => true
}.
ms_to_s(Ms) ->
erlang:ceil(Ms / 1000).
clientid(Id) ->
iolist_to_binary([Id, ":", atom_to_list(node())]).

9
apps/emqx_connector/src/emqx_connector_mysql.erl
View File

@ -56,7 +56,7 @@ on_start(InstId, #{server := {Host, Port},
auto_reconnect := AutoReconn,
pool_size := PoolSize,
ssl := SSL } = Config) ->
?SLOG(info, #{msg => "starting mysql connector",
?SLOG(info, #{msg => "starting_mysql_connector",
connector => InstId, config => Config}),
SslOpts = case maps:get(enable, SSL) of
true ->
@ -75,7 +75,7 @@ on_start(InstId, #{server := {Host, Port},
{ok, #{poolname => PoolName}}.
on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping mysql connector",
?SLOG(info, #{msg => "stopping_mysql_connector",
connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
@ -84,14 +84,13 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) ->
on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State);
on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) ->
?SLOG(debug, #{msg => "mysql connector received sql query",
connector => InstId, sql => SQL, state => State}),
?TRACE("QUERY", "mysql_connector_received", #{connector => InstId, sql => SQL, state => State}),
case Result = ecpool:pick_and_do(
PoolName,
{mysql, query, [SQL, Params, Timeout]},
no_handover) of
{error, Reason} ->
?SLOG(error, #{msg => "mysql connector do sql query failed",
?SLOG(error, #{msg => "mysql_connector_do_sql_query_failed",
connector => InstId, sql => SQL, reason => Reason}),
emqx_resource:query_failed(AfterQuery);
_ ->

29
apps/emqx_connector/src/emqx_connector_pgsql.erl
View File

@ -32,7 +32,9 @@
-export([connect/1]).
-export([query/3]).
-export([ query/3
, prepared_query/4
]).
-export([do_health_check/1]).
@ -56,7 +58,7 @@ on_start(InstId, #{server := {Host, Port},
auto_reconnect := AutoReconn,
pool_size := PoolSize,
ssl := SSL } = Config) ->
?SLOG(info, #{msg => "starting postgresql connector",
?SLOG(info, #{msg => "starting_postgresql_connector",
connector => InstId, config => Config}),
SslOpts = case maps:get(enable, SSL) of
true ->
@ -65,7 +67,7 @@ on_start(InstId, #{server := {Host, Port},
emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)}];
false ->
[{ssl, false}]
end,
end,
Options = [{host, Host},
{port, Port},
{username, User},
@ -82,15 +84,19 @@ on_stop(InstId, #{poolname := PoolName}) ->
connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
on_query(InstId, {sql, SQL, []}, AfterQuery, State);
on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) ->
?SLOG(debug, #{msg => "postgresql connector received sql query",
connector => InstId, sql => SQL, state => State}),
case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of
on_query(InstId, QueryParams, AfterQuery, #{poolname := PoolName} = State) ->
{Command, Args} = case QueryParams of
{query, SQL} -> {query, [SQL, []]};
{query, SQL, Params} -> {query, [SQL, Params]};
{prepared_query, Name, SQL} -> {prepared_query, [Name, SQL, []]};
{prepared_query, Name, SQL, Params} -> {prepared_query, [Name, SQL, Params]}
end,
?TRACE("QUERY", "postgresql_connector_received",
#{connector => InstId, command => Command, args => Args, state => State}),
case Result = ecpool:pick_and_do(PoolName, {?MODULE, Command, Args}, no_handover) of
{error, Reason} ->
?SLOG(error, #{
msg => "postgresql connector do sql query failed",
msg => "postgresql_connector_do_sql_query_failed",
connector => InstId, sql => SQL, reason => Reason}),
emqx_resource:query_failed(AfterQuery);
_ ->
@ -117,6 +123,9 @@ connect(Opts) ->
query(Conn, SQL, Params) ->
epgsql:equery(Conn, SQL, Params).
prepared_query(Conn, Name, SQL, Params) ->
epgsql:prepared_query2(Conn, Name, SQL, Params).
conn_opts(Opts) ->
conn_opts(Opts, []).
conn_opts([], Acc) ->

49
apps/emqx_connector/src/emqx_connector_redis.erl
View File

@ -20,12 +20,19 @@
-include_lib("emqx/include/logger.hrl").
-type server() :: tuple().
%% {"127.0.0.1", 7000}
%% For eredis:start_link/1~7
-reflect_type([server/0]).
-typerefl_from_string({server/0, ?MODULE, to_server}).
-export([to_server/1]).
-type servers() :: list().
%% [{"127.0.0.1", 7000}, {"127.0.0.2", 7000}]
%% For eredis_cluster
-reflect_type([servers/0]).
-typerefl_from_string({servers/0, ?MODULE, to_servers}).
-export([ to_server/1
, to_servers/1]).
-export([roots/0, fields/1]).
@ -63,14 +70,14 @@ fields(single) ->
redis_fields() ++
emqx_connector_schema_lib:ssl_fields();
fields(cluster) ->
[ {servers, #{type => hoconsc:array(server())}}
[ {servers, #{type => servers()}}
, {redis_type, #{type => hoconsc:enum([cluster]),
default => cluster}}
] ++
redis_fields() ++
emqx_connector_schema_lib:ssl_fields();
fields(sentinel) ->
[ {servers, #{type => hoconsc:array(server())}}
[ {servers, #{type => servers()}}
, {redis_type, #{type => hoconsc:enum([sentinel]),
default => sentinel}}
, {sentinel, #{type => string()}}
@ -87,7 +94,7 @@ on_start(InstId, #{redis_type := Type,
pool_size := PoolSize,
auto_reconnect := AutoReconn,
ssl := SSL } = Config) ->
?SLOG(info, #{msg => "starting redis connector",
?SLOG(info, #{msg => "starting_redis_connector",
connector => InstId, config => Config}),
Servers = case Type of
single -> [{servers, [maps:get(server, Config)]}];
@ -120,20 +127,20 @@ on_start(InstId, #{redis_type := Type,
{ok, #{poolname => PoolName, type => Type}}.
on_stop(InstId, #{poolname := PoolName}) ->
?SLOG(info, #{msg => "stopping redis connector",
?SLOG(info, #{msg => "stopping_redis_connector",
connector => InstId}),
emqx_plugin_libs_pool:stop_pool(PoolName).
on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) ->
?SLOG(debug, #{msg => "redis connector received cmd query",
connector => InstId, sql => Command, state => State}),
?TRACE("QUERY", "redis_connector_received",
#{connector => InstId, sql => Command, state => State}),
Result = case Type of
cluster -> eredis_cluster:q(PoolName, Command);
_ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover)
end,
case Result of
{error, Reason} ->
?SLOG(error, #{msg => "redis connector do cmd query failed",
?SLOG(error, #{msg => "redis_connector_do_cmd_query_failed",
connector => InstId, sql => Command, reason => Reason}),
emqx_resource:query_failed(AfterCommand);
_ ->
@ -181,7 +188,23 @@ redis_fields() ->
].
to_server(Server) ->
case string:tokens(Server, ":") of
[Host, Port] -> {ok, {Host, list_to_integer(Port)}};
_ -> {error, Server}
try {ok, parse_server(Server)}
catch
throw : Error ->
Error
end.
to_servers(Servers) ->
try {ok, lists:map(fun parse_server/1, string:tokens(Servers, ", "))}
catch
throw : _Reason ->
{error, Servers}
end.
parse_server(Server) ->
case string:tokens(Server, ": ") of
[Host, Port] ->
{Host, list_to_integer(Port)};
_ ->
throw({error, Server})
end.

60
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl
View File

@ -158,27 +158,23 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent)
RC =:= ?RC_NO_MATCHING_SUBSCRIBERS ->
Parent ! {batch_ack, PktId}, ok;
handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) ->
?SLOG(warning, #{msg => "publish to remote node falied",
?SLOG(warning, #{msg => "publish_to_remote_node_falied",
packet_id => PktId, reason_code => RC}).
handle_publish(Msg, undefined) ->
?SLOG(error, #{msg => "cannot publish to local broker as"
" 'ingress' is not configured",
?SLOG(error, #{msg => "cannot_publish_to_local_broker_as"
"_'ingress'_is_not_configured",
message => Msg});
handle_publish(Msg, Vars) ->
?SLOG(debug, #{msg => "publish to local broker",
handle_publish(Msg0, Vars) ->
Msg = format_msg_received(Msg0),
?SLOG(debug, #{msg => "publish_to_local_broker",
message => Msg, vars => Vars}),
emqx_metrics:inc('bridge.mqtt.message_received_from_remote', 1),
case Vars of
#{on_message_received := {Mod, Func, Args}} ->
_ = erlang:apply(Mod, Func, [Msg | Args]);
_ -> ok
end,
case maps:get(local_topic, Vars, undefined) of
undefined -> ok;
_Topic ->
emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars))
end.
maybe_publish_to_local_broker(Msg0, Vars).
handle_disconnected(Reason, Parent) ->
Parent ! {disconnected, self(), Reason}.
@ -198,3 +194,45 @@ sub_remote_topics(ClientPid, #{remote_topic := FromTopic, remote_qos := QoS}) ->
process_config(Config) ->
maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config).
maybe_publish_to_local_broker(#{topic := Topic} = Msg, #{remote_topic := SubTopic} = Vars) ->
case maps:get(local_topic, Vars, undefined) of
undefined ->
ok; %% local topic is not set, discard it
_ ->
case emqx_topic:match(Topic, SubTopic) of
true ->
_ = emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)),
ok;
false ->
?SLOG(warning, #{msg => "discard_message_as_topic_not_matched",
message => Msg, subscribed => SubTopic, got_topic => Topic})
end
end.
format_msg_received(#{dup := Dup, payload := Payload, properties := Props,
qos := QoS, retain := Retain, topic := Topic}) ->
#{event => '$bridges/mqtt',
id => emqx_guid:to_hexstr(emqx_guid:gen()),
payload => Payload,
topic => Topic,
qos => QoS,
dup => Dup,
retain => Retain,
pub_props => printable_maps(Props),
timestamp => erlang:system_time(millisecond)
}.
printable_maps(undefined) -> #{};
printable_maps(Headers) ->
maps:fold(
fun ('User-Property', V0, AccIn) when is_list(V0) ->
AccIn#{
'User-Property' => maps:from_list(V0),
'User-Property-Pairs' => [#{
key => Key,
value => Value
} || {Key, Value} <- V0]
};
(K, V0, AccIn) -> AccIn#{K => V0}
end, #{}, Headers).

11
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
View File

@ -61,12 +61,12 @@ make_pub_vars(Mountpoint, Conf) when is_map(Conf) ->
-> exp_msg().
to_remote_msg(#message{flags = Flags0} = Msg, Vars) ->
Retain0 = maps:get(retain, Flags0, false),
MapMsg = maps:put(retain, Retain0, emqx_message:to_map(Msg)),
MapMsg = maps:put(retain, Retain0, emqx_rule_events:eventmsg_publish(Msg)),
to_remote_msg(MapMsg, Vars);
to_remote_msg(MapMsg, #{remote_topic := TopicToken, payload := PayloadToken,
remote_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) ->
Topic = replace_vars_in_str(TopicToken, MapMsg),
Payload = replace_vars_in_str(PayloadToken, MapMsg),
Payload = process_payload(PayloadToken, MapMsg),
QoS = replace_simple_var(QoSToken, MapMsg),
Retain = replace_simple_var(RetainToken, MapMsg),
#mqtt_msg{qos = QoS,
@ -82,13 +82,18 @@ to_broker_msg(#{dup := Dup, properties := Props} = MapMsg,
#{local_topic := TopicToken, payload := PayloadToken,
local_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) ->
Topic = replace_vars_in_str(TopicToken, MapMsg),
Payload = replace_vars_in_str(PayloadToken, MapMsg),
Payload = process_payload(PayloadToken, MapMsg),
QoS = replace_simple_var(QoSToken, MapMsg),
Retain = replace_simple_var(RetainToken, MapMsg),
set_headers(Props,
emqx_message:set_flags(#{dup => Dup, retain => Retain},
emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))).
process_payload([], Msg) ->
emqx_json:encode(Msg);
process_payload(Tks, Msg) ->
replace_vars_in_str(Tks, Msg).
%% Replace a string contains vars to another string in which the placeholders are replace by the
%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be:
%% "a: 1".

15
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
View File

@ -39,7 +39,7 @@ fields("config") ->
fields("connector") ->
[ {mode,
sc(hoconsc:enum([cluster_singleton, cluster_shareload]),
sc(hoconsc:enum([cluster_shareload]),
#{ default => cluster_shareload
, desc => """
The mode of the MQTT Bridge. Can be one of 'cluster_singleton' or 'cluster_shareload'<br>
@ -55,12 +55,17 @@ clientid conflicts between different nodes. And we can only use shared subscript
topic filters for 'remote_topic' of ingress connections.
"""
})}
, {name,
sc(binary(),
#{ nullable => true
, desc => "Connector name, used as a human-readable description of the connector."
})}
, {server,
sc(emqx_schema:ip_port(),
#{ default => "127.0.0.1:1883"
, desc => "The host and port of the remote MQTT broker"
})}
, {reconnect_interval, mk_duration("reconnect interval", #{default => "30s"})}
, {reconnect_interval, mk_duration("reconnect interval", #{default => "15s"})}
, {proto_ver,
sc(hoconsc:enum([v3, v4, v5]),
#{ default => v4
@ -76,17 +81,13 @@ topic filters for 'remote_topic' of ingress connections.
#{ default => "emqx"
, desc => "The password of the MQTT protocol"
})}
, {clientid,
sc(binary(),
#{ desc => "The clientid of the MQTT protocol"
})}
, {clean_start,
sc(boolean(),
#{ default => true
, desc => "The clean-start or the clean-session of the MQTT protocol"
})}
, {keepalive, mk_duration("keepalive", #{default => "300s"})}
, {retry_interval, mk_duration("retry interval", #{default => "30s"})}
, {retry_interval, mk_duration("retry interval", #{default => "15s"})}
, {max_inflight,
sc(integer(),
#{ default => 32

12
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl
View File

@ -188,7 +188,7 @@ callback_mode() -> [state_functions].
%% @doc Config should be a map().
init(#{name := Name} = ConnectOpts) ->
?SLOG(debug, #{msg => "starting bridge worker",
?SLOG(debug, #{msg => "starting_bridge_worker",
name => Name}),
erlang:process_flag(trap_exit, true),
Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})),
@ -335,7 +335,7 @@ common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) ->
NewQ = replayq:append(Q, [Msg]),
{keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}};
common(StateName, Type, Content, #{name := Name} = State) ->
?SLOG(notice, #{msg => "Bridge discarded event",
?SLOG(notice, #{msg => "bridge_discarded_event",
name => Name, type => Type, state_name => StateName,
content => Content}),
{keep_state, State}.
@ -349,7 +349,7 @@ do_connect(#{connect_opts := ConnectOpts,
{ok, State#{connection => Conn}};
{error, Reason} ->
ConnectOpts1 = obfuscate(ConnectOpts),
?SLOG(error, #{msg => "Failed to connect",
?SLOG(error, #{msg => "failed_to_connect",
config => ConnectOpts1, reason => Reason}),
{error, Reason, State}
end.
@ -386,8 +386,8 @@ pop_and_send_loop(#{replayq := Q} = State, N) ->
end.
do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) ->
?SLOG(error, #{msg => "cannot forward messages to remote broker"
" as 'egress' is not configured",
?SLOG(error, #{msg => "cannot_forward_messages_to_remote_broker"
"_as_'egress'_is_not_configured",
messages => Msg});
do_send(#{inflight := Inflight,
connection := Connection,
@ -398,7 +398,7 @@ do_send(#{inflight := Inflight,
emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'),
emqx_connector_mqtt_msg:to_remote_msg(Message, Vars)
end,
?SLOG(debug, #{msg => "publish to remote broker",
?SLOG(debug, #{msg => "publish_to_remote_broker",
message => Msg, vars => Vars}),
case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(Msg)]) of
{ok, Refs} ->

418
apps/emqx_connector/test/emqx_connector_api_SUITE.erl
View File

@ -22,15 +22,15 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-define(CONF_DEFAULT, <<"connectors: {}">>).
%% output functions
-export([ inspect/3
]).
-define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
-define(CONNECTR_TYPE, <<"mqtt">>).
-define(CONNECTR_NAME, <<"test_connector">>).
-define(CONNECTR_ID, <<"mqtt:test_connector">>).
-define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>).
-define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>).
-define(BRIDGE_ID_INGRESS, <<"mqtt:ingress_test_bridge">>).
-define(BRIDGE_ID_EGRESS, <<"mqtt:egress_test_bridge">>).
-define(MQTT_CONNECOTR(Username),
#{
<<"server">> => <<"127.0.0.1:1883">>,
@ -70,6 +70,9 @@
<<"failed">> := FAILED, <<"rate">> := SPEED,
<<"rate_last5m">> := SPEED5M, <<"rate_max">> := SPEEDMAX}).
inspect(Selected, _Envs, _Args) ->
persistent_term:put(?MODULE, #{inspect => Selected}).
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -92,21 +95,38 @@ init_per_suite(Config) ->
%% some testcases (may from other app) already get emqx_connector started
_ = application:stop(emqx_resource),
_ = application:stop(emqx_connector),
ok = emqx_common_test_helpers:start_apps([emqx_connector, emqx_bridge, emqx_dashboard]),
ok = emqx_config:init_load(emqx_connector_schema, ?CONF_DEFAULT),
ok = emqx_common_test_helpers:start_apps([emqx_rule_engine, emqx_connector,
emqx_bridge, emqx_dashboard]),
ok = emqx_config:init_load(emqx_connector_schema, <<"connectors: {}">>),
ok = emqx_config:init_load(emqx_rule_engine_schema, <<"rule_engine {rules {}}">>),
ok = emqx_config:init_load(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
Config.
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx_connector, emqx_bridge, emqx_dashboard]),
emqx_common_test_helpers:stop_apps([emqx_rule_engine, emqx_connector, emqx_bridge, emqx_dashboard]),
ok.
init_per_testcase(_, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
Config.
end_per_testcase(_, _Config) ->
clear_resources(),
ok.
clear_resources() ->
lists:foreach(fun(#{id := Id}) ->
ok = emqx_rule_engine:delete_rule(Id)
end, emqx_rule_engine:get_rules()),
lists:foreach(fun(#{id := Id}) ->
ok = emqx_bridge:remove(Id)
end, emqx_bridge:list()),
lists:foreach(fun(#{<<"id">> := Id}) ->
ok = emqx_connector:delete(Id)
end, emqx_connector:list()).
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
@ -123,32 +143,21 @@ t_mqtt_crud_apis(_) ->
, <<"name">> => ?CONNECTR_NAME
}),
%ct:pal("---connector: ~p", [Connector]),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
, <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?CONNECTR_NAME
, <<"server">> := <<"127.0.0.1:1883">>
, <<"username">> := User1
, <<"password">> := <<"">>
, <<"proto_ver">> := <<"v4">>
, <<"ssl">> := #{<<"enable">> := false}
}, jsx:decode(Connector)),
%% create a again returns an error
{ok, 400, RetMsg} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
?assertMatch(
#{ <<"code">> := _
, <<"message">> := <<"connector already exists">>
}, jsx:decode(RetMsg)),
#{ <<"id">> := ConnctorID
, <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?CONNECTR_NAME
, <<"server">> := <<"127.0.0.1:1883">>
, <<"username">> := User1
, <<"password">> := <<"">>
, <<"proto_ver">> := <<"v4">>
, <<"ssl">> := #{<<"enable">> := false}
} = jsx:decode(Connector),
%% update the request-path of the connector
User2 = <<"user2">>,
{ok, 200, Connector2} = request(put, uri(["connectors", ?CONNECTR_ID]),
{ok, 200, Connector2} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR(User2)),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
?assertMatch(#{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:1883">>
, <<"username">> := User2
, <<"password">> := <<"">>
@ -158,7 +167,7 @@ t_mqtt_crud_apis(_) ->
%% list all connectors again, assert Connector2 is in it
{ok, 200, Connector2Str} = request(get, uri(["connectors"]), []),
?assertMatch([#{ <<"id">> := ?CONNECTR_ID
?assertMatch([#{ <<"id">> := ConnctorID
, <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?CONNECTR_NAME
, <<"server">> := <<"127.0.0.1:1883">>
@ -169,8 +178,8 @@ t_mqtt_crud_apis(_) ->
}], jsx:decode(Connector2Str)),
%% get the connector by id
{ok, 200, Connector3Str} = request(get, uri(["connectors", ?CONNECTR_ID]), []),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
{ok, 200, Connector3Str} = request(get, uri(["connectors", ConnctorID]), []),
?assertMatch(#{ <<"id">> := ConnctorID
, <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?CONNECTR_NAME
, <<"server">> := <<"127.0.0.1:1883">>
@ -181,11 +190,11 @@ t_mqtt_crud_apis(_) ->
}, jsx:decode(Connector3Str)),
%% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
%% update a deleted connector returns an error
{ok, 404, ErrMsg2} = request(put, uri(["connectors", ?CONNECTR_ID]),
{ok, 404, ErrMsg2} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR(User2)),
?assertMatch(
#{ <<"code">> := _
@ -194,10 +203,6 @@ t_mqtt_crud_apis(_) ->
ok.
t_mqtt_conn_bridge_ingress(_) ->
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% then we add a mqtt connector, using POST
User1 = <<"user1">>,
{ok, 201, Connector} = request(post, uri(["connectors"]),
@ -205,28 +210,28 @@ t_mqtt_conn_bridge_ingress(_) ->
, <<"name">> => ?CONNECTR_NAME
}),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
, <<"server">> := <<"127.0.0.1:1883">>
, <<"num_of_bridges">> := 0
, <<"username">> := User1
, <<"password">> := <<"">>
, <<"proto_ver">> := <<"v4">>
, <<"ssl">> := #{<<"enable">> := false}
}, jsx:decode(Connector)),
#{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:1883">>
, <<"num_of_bridges">> := 0
, <<"username">> := User1
, <<"password">> := <<"">>
, <<"proto_ver">> := <<"v4">>
, <<"ssl">> := #{<<"enable">> := false}
} = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_INGRESS(?CONNECTR_ID)#{
?MQTT_BRIDGE_INGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_INGRESS
}),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_INGRESS
, <<"type">> := <<"mqtt">>
, <<"status">> := <<"connected">>
, <<"connector">> := ?CONNECTR_ID
}, jsx:decode(Bridge)),
#{ <<"id">> := BridgeIDIngress
, <<"type">> := <<"mqtt">>
, <<"status">> := <<"connected">>
, <<"connector">> := ConnctorID
} = jsx:decode(Bridge),
%% we now test if the bridge works as expected
@ -236,8 +241,8 @@ t_mqtt_conn_bridge_ingress(_) ->
emqx:subscribe(LocalTopic),
%% PUBLISH a message to the 'remote' broker, as we have only one broker,
%% the remote broker is also the local one.
wait_for_resource_ready(BridgeIDIngress, 5),
emqx:publish(emqx_message:make(RemoteTopic, Payload)),
%% we should receive a message on the local broker, with specified topic
?assert(
receive
@ -252,25 +257,21 @@ t_mqtt_conn_bridge_ingress(_) ->
end),
%% get the connector by id, verify the num_of_bridges now is 1
{ok, 200, Connector1Str} = request(get, uri(["connectors", ?CONNECTR_ID]), []),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
{ok, 200, Connector1Str} = request(get, uri(["connectors", ConnctorID]), []),
?assertMatch(#{ <<"id">> := ConnctorID
, <<"num_of_bridges">> := 1
}, jsx:decode(Connector1Str)),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_INGRESS]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
ok.
t_mqtt_conn_bridge_egress(_) ->
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% then we add a mqtt connector, using POST
User1 = <<"user1">>,
{ok, 201, Connector} = request(post, uri(["connectors"]),
@ -279,29 +280,28 @@ t_mqtt_conn_bridge_egress(_) ->
}),
%ct:pal("---connector: ~p", [Connector]),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
, <<"server">> := <<"127.0.0.1:1883">>
, <<"username">> := User1
, <<"password">> := <<"">>
, <<"proto_ver">> := <<"v4">>
, <<"ssl">> := #{<<"enable">> := false}
}, jsx:decode(Connector)),
#{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:1883">>
, <<"username">> := User1
, <<"password">> := <<"">>
, <<"proto_ver">> := <<"v4">>
, <<"ssl">> := #{<<"enable">> := false}
} = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{
?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
%ct:pal("---bridge: ~p", [Bridge]),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
, <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"connected">>
, <<"connector">> := ?CONNECTR_ID
}, jsx:decode(Bridge)),
#{ <<"id">> := BridgeIDEgress
, <<"type">> := ?CONNECTR_TYPE
, <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"connected">>
, <<"connector">> := ConnctorID
} = jsx:decode(Bridge),
%% we now test if the bridge works as expected
LocalTopic = <<"local_topic/1">>,
@ -310,6 +310,7 @@ t_mqtt_conn_bridge_egress(_) ->
emqx:subscribe(RemoteTopic),
%% PUBLISH a message to the 'local' broker, as we have only one broker,
%% the remote broker is also the local one.
wait_for_resource_ready(BridgeIDEgress, 5),
emqx:publish(emqx_message:make(LocalTopic, Payload)),
%% we should receive a message on the "remote" broker, with specified topic
@ -326,19 +327,19 @@ t_mqtt_conn_bridge_egress(_) ->
end),
%% verify the metrics of the bridge
{ok, 200, BridgeStr} = request(get, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
{ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
?assertMatch(#{ <<"id">> := BridgeIDEgress
, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)
, <<"node_metrics">> :=
[#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}]
}, jsx:decode(BridgeStr)),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
ok.
@ -346,10 +347,6 @@ t_mqtt_conn_bridge_egress(_) ->
%% - update a connector should also update all of the the bridges
%% - cannot delete a connector that is used by at least one bridge
t_mqtt_conn_update(_) ->
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% then we add a mqtt connector, using POST
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)
@ -358,44 +355,41 @@ t_mqtt_conn_update(_) ->
}),
%ct:pal("---connector: ~p", [Connector]),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
, <<"server">> := <<"127.0.0.1:1883">>
}, jsx:decode(Connector)),
#{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:1883">>
} = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{
?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
, <<"type">> := <<"mqtt">>
, <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"connected">>
, <<"connector">> := ?CONNECTR_ID
}, jsx:decode(Bridge)),
#{ <<"id">> := BridgeIDEgress
, <<"type">> := <<"mqtt">>
, <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"connected">>
, <<"connector">> := ConnctorID
} = jsx:decode(Bridge),
wait_for_resource_ready(BridgeIDEgress, 2),
%% then we try to update 'server' of the connector, to an unavailable IP address
%% the update should fail because of 'unreachable' or 'connrefused'
{ok, 400, _ErrorMsg} = request(put, uri(["connectors", ?CONNECTR_ID]),
{ok, 400, _ErrorMsg} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR2(<<"127.0.0.1:2603">>)),
%% we fix the 'server' parameter to a normal one, it should work
{ok, 200, _} = request(put, uri(["connectors", ?CONNECTR_ID]),
{ok, 200, _} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR2(<<"127.0.0.1 : 1883">>)),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []).
t_mqtt_conn_update2(_) ->
%% assert we there's no connectors and no bridges at first
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% then we add a mqtt connector, using POST
%% but this connector is point to a unreachable server "2603"
{ok, 201, Connector} = request(post, uri(["connectors"]),
@ -404,38 +398,71 @@ t_mqtt_conn_update2(_) ->
, <<"name">> => ?CONNECTR_NAME
}),
?assertMatch(#{ <<"id">> := ?CONNECTR_ID
, <<"server">> := <<"127.0.0.1:2603">>
}, jsx:decode(Connector)),
#{ <<"id">> := ConnctorID
, <<"server">> := <<"127.0.0.1:2603">>
} = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{
?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
, <<"type">> := <<"mqtt">>
, <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"disconnected">>
, <<"connector">> := ?CONNECTR_ID
}, jsx:decode(Bridge)),
#{ <<"id">> := BridgeIDEgress
, <<"type">> := <<"mqtt">>
, <<"name">> := ?BRIDGE_NAME_EGRESS
, <<"status">> := <<"disconnected">>
, <<"connector">> := ConnctorID
} = jsx:decode(Bridge),
%% We try to fix the 'server' parameter, to another unavailable server..
%% The update should success: we don't check the connectivity of the new config
%% if the resource is now disconnected.
{ok, 200, _} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR2(<<"127.0.0.1:2604">>)),
%% we fix the 'server' parameter to a normal one, it should work
{ok, 200, _} = request(put, uri(["connectors", ?CONNECTR_ID]),
{ok, 200, _} = request(put, uri(["connectors", ConnctorID]),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)),
{ok, 200, BridgeStr} = request(get, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
{ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
?assertMatch(#{ <<"id">> := BridgeIDEgress
, <<"status">> := <<"connected">>
}, jsx:decode(BridgeStr)),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
%% delete the connector
{ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
{ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []).
t_mqtt_conn_update3(_) ->
%% we add a mqtt connector, using POST
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)
#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
#{ <<"id">> := ConnctorID } = jsx:decode(Connector),
%% ... and a MQTT bridge, using POST
%% we bind this bridge to the connector created just now
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
#{ <<"id">> := BridgeIDEgress
, <<"connector">> := ConnctorID
} = jsx:decode(Bridge),
wait_for_resource_ready(BridgeIDEgress, 2),
%% delete the connector should fail because it is in use by a bridge
{ok, 403, _} = request(delete, uri(["connectors", ConnctorID]), []),
%% delete the bridge
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
%% the connector now can be deleted without problems
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
t_mqtt_conn_testing(_) ->
%% APIs for testing the connectivity
%% then we add a mqtt connector, using POST
@ -450,6 +477,153 @@ t_mqtt_conn_testing(_) ->
<<"name">> => ?BRIDGE_NAME_EGRESS
}).
t_ingress_mqtt_bridge_with_rules(_) ->
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
#{ <<"id">> := ConnctorID } = jsx:decode(Connector),
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_INGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_INGRESS
}),
#{ <<"id">> := BridgeIDIngress } = jsx:decode(Bridge),
{ok, 201, Rule} = request(post, uri(["rules"]),
#{<<"name">> => <<"A rule get messages from a source mqtt bridge">>,
<<"enable">> => true,
<<"outputs">> => [#{<<"function">> => "emqx_connector_api_SUITE:inspect"}],
<<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">>
}),
#{<<"id">> := RuleId} = jsx:decode(Rule),
%% we now test if the bridge works as expected
RemoteTopic = <<"remote_topic/1">>,
LocalTopic = <<"local_topic/", RemoteTopic/binary>>,
Payload = <<"hello">>,
emqx:subscribe(LocalTopic),
%% PUBLISH a message to the 'remote' broker, as we have only one broker,
%% the remote broker is also the local one.
wait_for_resource_ready(BridgeIDIngress, 5),
emqx:publish(emqx_message:make(RemoteTopic, Payload)),
%% we should receive a message on the local broker, with specified topic
?assert(
receive
{deliver, LocalTopic, #message{payload = Payload}} ->
ct:pal("local broker got message: ~p on topic ~p", [Payload, LocalTopic]),
true;
Msg ->
ct:pal("Msg: ~p", [Msg]),
false
after 100 ->
false
end),
%% and also the rule should be matched, with matched + 1:
{ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
#{ <<"id">> := RuleId
, <<"metrics">> := #{<<"matched">> := 1}
} = jsx:decode(Rule1),
%% we also check if the outputs of the rule is triggered
?assertMatch(#{inspect := #{
event := '$bridges/mqtt',
id := MsgId,
payload := Payload,
topic := RemoteTopic,
qos := 0,
dup := false,
retain := false,
pub_props := #{},
timestamp := _
}} when is_binary(MsgId), persistent_term:get(?MODULE)),
{ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
t_egress_mqtt_bridge_with_rules(_) ->
{ok, 201, Connector} = request(post, uri(["connectors"]),
?MQTT_CONNECOTR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE
, <<"name">> => ?CONNECTR_NAME
}),
#{ <<"id">> := ConnctorID } = jsx:decode(Connector),
{ok, 201, Bridge} = request(post, uri(["bridges"]),
?MQTT_BRIDGE_EGRESS(ConnctorID)#{
<<"type">> => ?CONNECTR_TYPE,
<<"name">> => ?BRIDGE_NAME_EGRESS
}),
#{ <<"id">> := BridgeIDEgress } = jsx:decode(Bridge),
{ok, 201, Rule} = request(post, uri(["rules"]),
#{<<"name">> => <<"A rule send messages to a sink mqtt bridge">>,
<<"enable">> => true,
<<"outputs">> => [BridgeIDEgress],
<<"sql">> => <<"SELECT * from \"t/1\"">>
}),
#{<<"id">> := RuleId} = jsx:decode(Rule),
%% we now test if the bridge works as expected
LocalTopic = <<"local_topic/1">>,
RemoteTopic = <<"remote_topic/", LocalTopic/binary>>,
Payload = <<"hello">>,
emqx:subscribe(RemoteTopic),
%% PUBLISH a message to the 'local' broker, as we have only one broker,
%% the remote broker is also the local one.
wait_for_resource_ready(BridgeIDEgress, 5),
emqx:publish(emqx_message:make(LocalTopic, Payload)),
%% we should receive a message on the "remote" broker, with specified topic
?assert(
receive
{deliver, RemoteTopic, #message{payload = Payload}} ->
ct:pal("local broker got message: ~p on topic ~p", [Payload, RemoteTopic]),
true;
Msg ->
ct:pal("Msg: ~p", [Msg]),
false
after 100 ->
false
end),
emqx:unsubscribe(RemoteTopic),
%% PUBLISH a message to the rule.
Payload2 = <<"hi">>,
RuleTopic = <<"t/1">>,
RemoteTopic2 = <<"remote_topic/", RuleTopic/binary>>,
emqx:subscribe(RemoteTopic2),
wait_for_resource_ready(BridgeIDEgress, 5),
emqx:publish(emqx_message:make(RuleTopic, Payload2)),
{ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
#{ <<"id">> := RuleId
, <<"metrics">> := #{<<"matched">> := 1}
} = jsx:decode(Rule1),
%% we should receive a message on the "remote" broker, with specified topic
?assert(
receive
{deliver, RemoteTopic2, #message{payload = Payload2}} ->
ct:pal("local broker got message: ~p on topic ~p", [Payload2, RemoteTopic2]),
true;
Msg ->
ct:pal("Msg: ~p", [Msg]),
false
after 100 ->
false
end),
%% verify the metrics of the bridge
{ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
?assertMatch(#{ <<"id">> := BridgeIDEgress
, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)
, <<"node_metrics">> :=
[#{<<"node">> := _, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)}]
}, jsx:decode(BridgeStr)),
{ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
{ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
{ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
%%--------------------------------------------------------------------
%% HTTP Request
%%--------------------------------------------------------------------
@ -483,3 +657,13 @@ auth_header_() ->
{ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
{"Authorization", "Bearer " ++ binary_to_list(Token)}.
wait_for_resource_ready(InstId, 0) ->
ct:pal("--- bridge ~p: ~p", [InstId, emqx_bridge:lookup(InstId)]),
ct:fail(wait_resource_timeout);
wait_for_resource_ready(InstId, Retry) ->
case emqx_bridge:lookup(InstId) of
{ok, #{resource_data := #{status := started}}} -> ok;
_ ->
timer:sleep(100),
wait_for_resource_ready(InstId, Retry-1)
end.

4
apps/emqx_dashboard/src/emqx_dashboard.erl
View File

@ -151,9 +151,9 @@ authorize(Req) ->
ok ->
ok;
{error, token_timeout} ->
return_unauthorized(<<"TOKEN_TIME_OUT">>, <<"POST '/login', get new token">>);
{401, 'TOKEN_TIME_OUT', <<"Token expired, get new token by POST /login">>};
{error, not_found} ->
return_unauthorized(<<"BAD_TOKEN">>, <<"POST '/login'">>)
{401, 'BAD_TOKEN', <<"Get a token by POST /login">>}
end;
_ ->
return_unauthorized(<<"AUTHORIZATION_HEADER_ERROR">>,

4
apps/emqx_dashboard/src/emqx_dashboard_api.erl
View File

@ -123,7 +123,7 @@ schema("/users/:username") ->
#{in => path, example => <<"admin">>})}],
'requestBody' => [
{ description
, mk(emqx_schema:unicode_binary(),
, mk(binary(),
#{desc => <<"User description">>, example => <<"administrator">>})}
],
responses => #{
@ -176,7 +176,7 @@ schema("/users/:username/change_pwd") ->
fields(user) ->
[
{description,
mk(emqx_schema:unicode_binary(),
mk(binary(),
#{desc => <<"User description">>, example => "administrator"})},
{username,
mk(binary(),

5
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -312,6 +312,9 @@ responses(Responses, Module) ->
response(Status, Bin, {Acc, RefsAcc, Module}) when is_binary(Bin) ->
{Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module};
%% Support swagger raw object(file download).
response(Status, #{content := _} = Content, {Acc, RefsAcc, Module}) ->
{Acc#{integer_to_binary(Status) => Content}, RefsAcc, Module};
response(Status, ?REF(StructName), {Acc, RefsAcc, Module}) ->
response(Status, ?R_REF(Module, StructName), {Acc, RefsAcc, Module});
response(Status, ?R_REF(_Mod, _Name) = RRef, {Acc, RefsAcc, Module}) ->
@ -423,8 +426,10 @@ typename_to_spec("duration_ms()", _Mod) -> #{type => string, example => <<"32s">
typename_to_spec("percent()", _Mod) -> #{type => number, example => <<"12%">>};
typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>};
typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>};
typename_to_spec("ip_ports()", _Mod) -> #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>};
typename_to_spec("server()", Mod) -> typename_to_spec("ip_port()", Mod);
typename_to_spec("servers()", Mod) -> typename_to_spec("ip_ports()", Mod);
typename_to_spec("connect_timeout()", Mod) -> typename_to_spec("timeout()", Mod);
typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, example => infinity},
#{type => integer, example => 100}], example => infinity};

93
apps/emqx_gateway/src/coap/emqx_coap_impl.erl
View File

@ -16,9 +16,16 @@
-module(emqx_coap_impl).
-behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/include/emqx_gateway.hrl").
-behaviour(emqx_gateway_impl).
-import(emqx_gateway_utils,
[ normalize_config/1
, start_listeners/4
, stop_listeners/2
]).
%% APIs
-export([ reg/0
@ -30,8 +37,6 @@
, on_gateway_unload/2
]).
-include_lib("emqx/include/logger.hrl").
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
@ -51,12 +56,20 @@ unreg() ->
on_gateway_load(_Gateway = #{name := GwName,
config := Config
}, Ctx) ->
Listeners = emqx_gateway_utils:normalize_config(Config),
ListenerPids = lists:map(fun(Lis) ->
start_listener(GwName, Ctx, Lis)
end, Listeners),
{ok, ListenerPids, #{ctx => Ctx}}.
Listeners = normalize_config(Config),
ModCfg = #{frame_mod => emqx_coap_frame,
chann_mod => emqx_coap_channel
},
case start_listeners(
Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
{ok, ListenerPids, #{ctx => Ctx}};
{error, {Reason, Listener}} ->
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway),
@ -76,63 +89,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
on_gateway_unload(_Gateway = #{ name := GwName,
config := Config
}, _GwState) ->
Listeners = emqx_gateway_utils:normalize_config(Config),
lists:foreach(fun(Lis) ->
stop_listener(GwName, Lis)
end, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{ctx => Ctx,
listener => {GwName, Type, LisName},
frame_mod => emqx_coap_frame,
chann_mod => emqx_coap_channel
},
MFA = {emqx_gateway_conn, start_link, [NCfg]},
do_start_listener(Type, Name, ListenOn, SocketOpts, MFA).
do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_udp(Name, ListenOn, SocketOpts, MFA);
do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_dtls(Name, ListenOn, SocketOpts, MFA).
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.
Listeners = normalize_config(Config),
stop_listeners(GwName, Listeners).

16
apps/emqx_gateway/src/emqx_gateway_api_clients.erl
View File

@ -532,7 +532,21 @@ params_client_searching_in_qs() ->
, {lte_connected_at,
mk(binary(),
M#{desc => <<"Match the client socket connected datatime less than "
" a certain value">>})}
"a certain value">>})}
, {endpoint_name,
mk(binary(),
M#{desc => <<"Match the lwm2m client's endpoint name">>})}
, {like_endpoint_name,
mk(binary(),
M#{desc => <<"Use sub-string to match lwm2m client's endpoint name">>})}
, {gte_lifetime,
mk(binary(),
M#{desc => <<"Match the lwm2m client registered lifetime greater "
"than a certain value">>})}
, {lte_lifetime,
mk(binary(),
M#{desc => <<"Match the lwm2m client registered lifetime less than "
"a certain value">>})}
].
params_paging() ->

2
apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
View File

@ -580,7 +580,7 @@ common_listener_opts() ->
#{ nullable => {true, recursively}
, desc => <<"The authenticatior for this listener">>
})}
].
] ++ emqx_gateway_schema:proxy_protocol_opts().
%%--------------------------------------------------------------------
%% examples

56
apps/emqx_gateway/src/emqx_gateway_cli.erl
View File

@ -28,6 +28,8 @@
%, 'gateway-banned'/1
]).
-elvis([{elvis_style, function_naming_convention, disable}]).
-spec load() -> ok.
load() ->
Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)],
@ -50,18 +52,24 @@ is_cmd(Fun) ->
%% Cmds
gateway(["list"]) ->
lists:foreach(fun(#{name := Name} = Gateway) ->
%% TODO: More infos: listeners?, connected?
Status = maps:get(status, Gateway, stopped),
print("Gateway(name=~ts, status=~ts)~n", [Name, Status])
end, emqx_gateway:list());
lists:foreach(
fun (#{name := Name, status := unloaded}) ->
print("Gateway(name=~ts, status=unloaded)\n", [Name]);
(#{name := Name, status := stopped, stopped_at := StoppedAt}) ->
print("Gateway(name=~ts, status=stopped, stopped_at=~ts)\n",
[Name, StoppedAt]);
(#{name := Name, status := running, current_connections := ConnCnt,
started_at := StartedAt}) ->
print("Gateway(name=~ts, status=running, clients=~w, started_at=~ts)\n",
[Name, ConnCnt, StartedAt])
end, emqx_gateway_http:gateways(all));
gateway(["lookup", Name]) ->
case emqx_gateway:lookup(atom(Name)) of
undefined ->
print("undefined~n");
print("undefined\n");
Info ->
print("~p~n", [Info])
print("~p\n", [Info])
end;
gateway(["load", Name, Conf]) ->
@ -70,17 +78,17 @@ gateway(["load", Name, Conf]) ->
emqx_json:decode(Conf, [return_maps])
) of
{ok, _} ->
print("ok~n");
print("ok\n");
{error, Reason} ->
print("Error: ~p~n", [Reason])
print("Error: ~p\n", [Reason])
end;
gateway(["unload", Name]) ->
case emqx_gateway_conf:unload_gateway(bin(Name)) of
ok ->
print("ok~n");
print("ok\n");
{error, Reason} ->
print("Error: ~p~n", [Reason])
print("Error: ~p\n", [Reason])
end;
gateway(["stop", Name]) ->
@ -89,9 +97,9 @@ gateway(["stop", Name]) ->
#{<<"enable">> => <<"false">>}
) of
{ok, _} ->
print("ok~n");
print("ok\n");
{error, Reason} ->
print("Error: ~p~n", [Reason])
print("Error: ~p\n", [Reason])
end;
gateway(["start", Name]) ->
@ -100,9 +108,9 @@ gateway(["start", Name]) ->
#{<<"enable">> => <<"true">>}
) of
{ok, _} ->
print("ok~n");
print("ok\n");
{error, Reason} ->
print("Error: ~p~n", [Reason])
print("Error: ~p\n", [Reason])
end;
gateway(_) ->
@ -123,7 +131,7 @@ gateway(_) ->
'gateway-registry'(["list"]) ->
lists:foreach(
fun({Name, #{cbkmod := CbMod}}) ->
print("Registered Name: ~ts, Callback Module: ~ts~n", [Name, CbMod])
print("Registered Name: ~ts, Callback Module: ~ts\n", [Name, CbMod])
end,
emqx_gateway_registry:list());
@ -137,15 +145,15 @@ gateway(_) ->
InfoTab = emqx_gateway_cm:tabname(info, Name),
case ets:info(InfoTab) of
undefined ->
print("Bad Gateway Name.~n");
print("Bad Gateway Name.\n");
_ ->
dump(InfoTab, client)
dump(InfoTab, client)
end;
'gateway-clients'(["lookup", Name, ClientId]) ->
ChanTab = emqx_gateway_cm:tabname(chan, Name),
case ets:lookup(ChanTab, bin(ClientId)) of
[] -> print("Not Found.~n");
[] -> print("Not Found.\n");
[Chann] ->
InfoTab = emqx_gateway_cm:tabname(info, Name),
[ChannInfo] = ets:lookup(InfoTab, Chann),
@ -154,8 +162,8 @@ gateway(_) ->
'gateway-clients'(["kick", Name, ClientId]) ->
case emqx_gateway_cm:kick_session(Name, bin(ClientId)) of
ok -> print("ok~n");
_ -> print("Not Found.~n")
ok -> print("ok\n");
_ -> print("Not Found.\n")
end;
'gateway-clients'(_) ->
@ -171,11 +179,11 @@ gateway(_) ->
Tab = emqx_gateway_metrics:tabname(Name),
case ets:info(Tab) of
undefined ->
print("Bad Gateway Name.~n");
print("Bad Gateway Name.\n");
_ ->
lists:foreach(
fun({K, V}) ->
print("~-30s: ~w~n", [K, V])
print("~-30s: ~w\n", [K, V])
end, lists:sort(ets:tab2list(Tab)))
end;
@ -232,7 +240,7 @@ print_record({client, {_, Infos, Stats}}) ->
print("Client(~ts, username=~ts, peername=~ts, "
"clean_start=~ts, keepalive=~w, "
"subscriptions=~w, delivered_msgs=~w, "
"connected=~ts, created_at=~w, connected_at=~w)~n",
"connected=~ts, created_at=~w, connected_at=~w)\n",
[format(K, maps:get(K, Info)) || K <- InfoKeys]).
print(S) -> emqx_ctl:print(S).

2
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -50,6 +50,8 @@
-export([namespace/0, roots/0 , fields/1]).
-export([proxy_protocol_opts/0]).
namespace() -> gateway.
roots() -> [gateway].

133
apps/emqx_gateway/src/emqx_gateway_utils.erl
View File

@ -18,6 +18,7 @@
-module(emqx_gateway_utils).
-include("emqx_gateway.hrl").
-include_lib("emqx/include/logger.hrl").
-export([ childspec/2
, childspec/3
@ -26,6 +27,12 @@
, find_sup_child/2
]).
-export([ start_listeners/4
, start_listener/4
, stop_listeners/2
, stop_listener/2
]).
-export([ apply/2
, format_listenon/1
, parse_listenon/1
@ -89,9 +96,15 @@ childspec(Id, Type, Mod, Args) ->
-spec supervisor_ret(supervisor:startchild_ret())
-> {ok, pid()}
| {error, supervisor:startchild_err()}.
supervisor_ret({ok, Pid, _Info}) -> {ok, Pid};
supervisor_ret({error, {Reason, _Child}}) -> {error, Reason};
supervisor_ret(Ret) -> Ret.
supervisor_ret({ok, Pid, _Info}) ->
{ok, Pid};
supervisor_ret({error, {Reason, Child}}) ->
case element(1, Child) == child of
true -> {error, Reason};
_ -> {error, {Reason, Child}}
end;
supervisor_ret(Ret) ->
Ret.
-spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id())
-> false
@ -102,6 +115,120 @@ find_sup_child(Sup, ChildId) ->
{_Id, Pid, _Type, _Mods} -> {ok, Pid}
end.
%% @doc start listeners. close all listeners if someone failed
-spec start_listeners(Listeners :: list(),
GwName :: atom(),
Ctx :: map(),
ModCfg)
-> {ok, [pid()]}
| {error, term()}
when ModCfg :: #{frame_mod := atom(), chann_mod := atom()}.
start_listeners(Listeners, GwName, Ctx, ModCfg) ->
start_listeners(Listeners, GwName, Ctx, ModCfg, []).
start_listeners([], _, _, _, Acc) ->
{ok, lists:map(fun({listener, {_, Pid}}) -> Pid end, Acc)};
start_listeners([L | Ls], GwName, Ctx, ModCfg, Acc) ->
case start_listener(GwName, Ctx, L, ModCfg) of
{ok, {ListenerId, ListenOn, Pid}} ->
NAcc = Acc ++ [{listener, {{ListenerId, ListenOn}, Pid}}],
start_listeners(Ls, GwName, Ctx, ModCfg, NAcc);
{error, Reason} ->
lists:foreach(fun({listener, {{ListenerId, ListenOn}, _}}) ->
esockd:close({ListenerId, ListenOn})
end, Acc),
{error, {Reason, L}}
end.
-spec start_listener(GwName :: atom(),
Ctx :: emqx_gateway_ctx:context(),
Listener :: tuple(),
ModCfg :: map())
-> {ok, {ListenerId :: atom(), esockd:listen_on(), pid()}}
| {error, term()}.
start_listener(GwName, Ctx,
{Type, LisName, ListenOn, SocketOpts, Cfg}, ModCfg) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = maps:merge(Cfg, ModCfg),
case start_listener(GwName, Ctx, Type,
LisName, ListenOn, SocketOpts, NCfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
{ok, {ListenerId, ListenOn, Pid}};
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
emqx_gateway_utils:supervisor_ret({error, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{ ctx => Ctx
, listener => {GwName, Type, LisName}
},
NSocketOpts = merge_default(Type, SocketOpts),
MFA = {emqx_gateway_conn, start_link, [NCfg]},
do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA).
merge_default(Udp, Options) ->
{Key, Default} = case Udp of
udp ->
{udp_options, default_udp_options()};
dtls ->
{udp_options, default_udp_options()};
tcp ->
{tcp_options, default_tcp_options()};
ssl ->
{tcp_options, default_tcp_options()}
end,
case lists:keytake(Key, 1, Options) of
{value, {Key, TcpOpts}, Options1} ->
[{Key, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{Key, Default} | Options]
end.
do_start_listener(Type, Name, ListenOn, SocketOpts, MFA)
when Type == tcp;
Type == ssl ->
esockd:open(Name, ListenOn, SocketOpts, MFA);
do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_udp(Name, ListenOn, SocketOpts, MFA);
do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_dtls(Name, ListenOn, SocketOpts, MFA).
-spec stop_listeners(GwName :: atom(), Listeners :: list()) -> ok.
stop_listeners(GwName, Listeners) ->
lists:foreach(fun(L) -> stop_listener(GwName, L) end, Listeners).
-spec stop_listener(GwName :: atom(), Listener :: tuple()) -> ok.
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.
apply({M, F, A}, A2) when is_atom(M),
is_atom(M),
is_list(A),

208
apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl
View File

@ -19,6 +19,14 @@
-behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
-import(emqx_gateway_utils,
[ normalize_config/1
, start_listeners/4
, stop_listeners/2
]).
%% APIs
-export([ reg/0
, unreg/0
@ -29,8 +37,6 @@
, on_gateway_unload/2
]).
-include_lib("emqx/include/logger.hrl").
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
@ -47,6 +53,73 @@ unreg() ->
%% emqx_gateway_registry callbacks
%%--------------------------------------------------------------------
on_gateway_load(_Gateway = #{ name := GwName,
config := Config
}, Ctx) ->
%% XXX: How to monitor it ?
%% Start grpc client pool & client channel
PoolName = pool_name(GwName),
PoolSize = emqx_vm:schedulers() * 2,
{ok, PoolSup} = emqx_pool_sup:start_link(
PoolName, hash, PoolSize,
{emqx_exproto_gcli, start_link, []}),
_ = start_grpc_client_channel(GwName,
maps:get(handler, Config, undefined)
),
%% XXX: How to monitor it ?
_ = start_grpc_server(GwName, maps:get(server, Config, undefined)),
NConfig = maps:without(
[server, handler],
Config#{pool_name => PoolName}
),
Listeners = emqx_gateway_utils:normalize_config(
NConfig#{handler => GwName}
),
ModCfg = #{frame_mod => emqx_exproto_frame,
chann_mod => emqx_exproto_channel
},
case start_listeners(
Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
{ok, ListenerPids, _GwState = #{ctx => Ctx, pool => PoolSup}};
{error, {Reason, Listener}} ->
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway),
try
%% XXX: 1. How hot-upgrade the changes ???
%% XXX: 2. Check the New confs first before destroy old instance ???
on_gateway_unload(Gateway, GwState),
on_gateway_load(Gateway#{config => Config}, Ctx)
catch
Class : Reason : Stk ->
logger:error("Failed to update ~ts; "
"reason: {~0p, ~0p} stacktrace: ~0p",
[GwName, Class, Reason, Stk]),
{error, {Class, Reason}}
end.
on_gateway_unload(_Gateway = #{ name := GwName,
config := Config
}, _GwState = #{pool := PoolSup}) ->
Listeners = emqx_gateway_utils:normalize_config(Config),
%% Stop funcs???
exit(PoolSup, kill),
stop_grpc_server(GwName),
stop_grpc_client_channel(GwName),
stop_listeners(GwName, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_grpc_server(_GwName, undefined) ->
undefined;
start_grpc_server(GwName, Options = #{bind := ListenOn}) ->
@ -103,140 +176,9 @@ stop_grpc_client_channel(GwName) ->
_ = grpc_client_sup:stop_channel_pool(GwName),
ok.
on_gateway_load(_Gateway = #{ name := GwName,
config := Config
}, Ctx) ->
%% XXX: How to monitor it ?
%% Start grpc client pool & client channel
PoolName = pool_name(GwName),
PoolSize = emqx_vm:schedulers() * 2,
{ok, PoolSup} = emqx_pool_sup:start_link(
PoolName, hash, PoolSize,
{emqx_exproto_gcli, start_link, []}),
_ = start_grpc_client_channel(GwName,
maps:get(handler, Config, undefined)
),
%% XXX: How to monitor it ?
_ = start_grpc_server(GwName, maps:get(server, Config, undefined)),
NConfig = maps:without(
[server, handler],
Config#{pool_name => PoolName}
),
Listeners = emqx_gateway_utils:normalize_config(
NConfig#{handler => GwName}
),
ListenerPids = lists:map(fun(Lis) ->
start_listener(GwName, Ctx, Lis)
end, Listeners),
{ok, ListenerPids, _GwState = #{ctx => Ctx, pool => PoolSup}}.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway),
try
%% XXX: 1. How hot-upgrade the changes ???
%% XXX: 2. Check the New confs first before destroy old instance ???
on_gateway_unload(Gateway, GwState),
on_gateway_load(Gateway#{config => Config}, Ctx)
catch
Class : Reason : Stk ->
logger:error("Failed to update ~ts; "
"reason: {~0p, ~0p} stacktrace: ~0p",
[GwName, Class, Reason, Stk]),
{error, {Class, Reason}}
end.
on_gateway_unload(_Gateway = #{ name := GwName,
config := Config
}, _GwState = #{pool := PoolSup}) ->
Listeners = emqx_gateway_utils:normalize_config(Config),
%% Stop funcs???
exit(PoolSup, kill),
stop_grpc_server(GwName),
stop_grpc_client_channel(GwName),
lists:foreach(fun(Lis) ->
stop_listener(GwName, Lis)
end, Listeners).
pool_name(GwName) ->
list_to_atom(lists:concat([GwName, "_gcli_pool"])).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{
ctx => Ctx,
listener => {GwName, Type, LisName},
frame_mod => emqx_exproto_frame,
chann_mod => emqx_exproto_channel
},
MFA = {emqx_gateway_conn, start_link, [NCfg]},
NSockOpts = merge_default_by_type(Type, SocketOpts),
do_start_listener(Type, Name, ListenOn, NSockOpts, MFA).
do_start_listener(Type, Name, ListenOn, Opts, MFA)
when Type == tcp;
Type == ssl ->
esockd:open(Name, ListenOn, Opts, MFA);
do_start_listener(udp, Name, ListenOn, Opts, MFA) ->
esockd:open_udp(Name, ListenOn, Opts, MFA);
do_start_listener(dtls, Name, ListenOn, Opts, MFA) ->
esockd:open_dtls(Name, ListenOn, Opts, MFA).
merge_default_by_type(Type, Options) when Type =:= tcp;
Type =:= ssl ->
Default = emqx_gateway_utils:default_tcp_options(),
case lists:keytake(tcp_options, 1, Options) of
{value, {tcp_options, TcpOpts}, Options1} ->
[{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{tcp_options, Default} | Options]
end;
merge_default_by_type(Type, Options) when Type =:= udp;
Type =:= dtls ->
Default = emqx_gateway_utils:default_udp_options(),
case lists:keytake(udp_options, 1, Options) of
{value, {udp_options, TcpOpts}, Options1} ->
[{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{udp_options, Default} | Options]
end.
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.

93
apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl
View File

@ -19,6 +19,8 @@
-behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
%% APIs
-export([ reg/0
, unreg/0
@ -29,8 +31,6 @@
, on_gateway_unload/2
]).
-include_lib("emqx/include/logger.hrl").
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
@ -54,10 +54,20 @@ on_gateway_load(_Gateway = #{ name := GwName,
case emqx_lwm2m_xml_object_db:start_link(XmlDir) of
{ok, RegPid} ->
Listeners = emqx_gateway_utils:normalize_config(Config),
ListenerPids = lists:map(fun(Lis) ->
start_listener(GwName, Ctx, Lis)
end, Listeners),
{ok, ListenerPids, _GwState = #{ctx => Ctx, registry => RegPid}};
ModCfg = #{frame_mod => emqx_coap_frame,
chann_mod => emqx_lwm2m_channel
},
case emqx_gateway_utils:start_listeners(
Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
{ok, ListenerPids, #{ctx => Ctx, registry => RegPid}};
{error, {Reason, Listener}} ->
_ = emqx_lwm2m_xml_object_db:stop(),
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end;
{error, Reason} ->
throw({badconf, #{ key => xml_dir
, value => XmlDir
@ -85,73 +95,4 @@ on_gateway_unload(_Gateway = #{ name := GwName,
}, _GwState = #{registry := RegPid}) ->
exit(RegPid, kill),
Listeners = emqx_gateway_utils:normalize_config(Config),
lists:foreach(fun(Lis) ->
stop_listener(GwName, Lis)
end, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{ ctx => Ctx
, listener => {GwName, Type, LisName}
, frame_mod => emqx_coap_frame
, chann_mod => emqx_lwm2m_channel
},
NSocketOpts = merge_default(SocketOpts),
MFA = {emqx_gateway_conn, start_link, [NCfg]},
do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA).
merge_default(Options) ->
Default = emqx_gateway_utils:default_udp_options(),
case lists:keytake(udp_options, 1, Options) of
{value, {udp_options, TcpOpts}, Options1} ->
[{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{udp_options, Default} | Options]
end.
do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_udp(Name, ListenOn, SocketOpts, MFA);
do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) ->
esockd:open_dtls(Name, ListenOn, SocketOpts, MFA).
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.
emqx_gateway_utils:stop_listeners(GwName, Listeners).

96
apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl
View File

@ -19,6 +19,14 @@
-behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
-import(emqx_gateway_utils,
[ normalize_config/1
, start_listeners/4
, stop_listeners/2
]).
%% APIs
-export([ reg/0
, unreg/0
@ -29,8 +37,6 @@
, on_gateway_unload/2
]).
-include_lib("emqx/include/logger.hrl").
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
@ -70,12 +76,23 @@ on_gateway_load(_Gateway = #{ name := GwName,
[broadcast, predefined],
Config#{registry => emqx_sn_registry:lookup_name(RegistrySvr)}
),
Listeners = emqx_gateway_utils:normalize_config(NConfig),
ListenerPids = lists:map(fun(Lis) ->
start_listener(GwName, Ctx, Lis)
end, Listeners),
{ok, ListenerPids, _InstaState = #{ctx => Ctx}}.
ModCfg = #{frame_mod => emqx_sn_frame,
chann_mod => emqx_sn_channel
},
case start_listeners(
Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
{ok, ListenerPids, _GwState = #{ctx => Ctx}};
{error, {Reason, Listener}} ->
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway),
@ -95,68 +112,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
on_gateway_unload(_Gateway = #{ name := GwName,
config := Config
}, _GwState) ->
Listeners = emqx_gateway_utils:normalize_config(Config),
lists:foreach(fun(Lis) ->
stop_listener(GwName, Lis)
end, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{
ctx => Ctx,
listene => {GwName, Type, LisName},
frame_mod => emqx_sn_frame,
chann_mod => emqx_sn_channel
},
esockd:open_udp(Name, ListenOn, merge_default(SocketOpts),
{emqx_gateway_conn, start_link, [NCfg]}).
merge_default(Options) ->
Default = emqx_gateway_utils:default_udp_options(),
case lists:keytake(udp_options, 1, Options) of
{value, {udp_options, TcpOpts}, Options1} ->
[{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{udp_options, Default} | Options]
end.
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.
Listeners = normalize_config(Config),
stop_listeners(GwName, Listeners).

104
apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl
View File

@ -18,6 +18,15 @@
-behaviour(emqx_gateway_impl).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_gateway/include/emqx_gateway.hrl").
-import(emqx_gateway_utils,
[ normalize_config/1
, start_listeners/4
, stop_listeners/2
]).
%% APIs
-export([ reg/0
, unreg/0
@ -28,9 +37,6 @@
, on_gateway_unload/2
]).
-include_lib("emqx_gateway/include/emqx_gateway.hrl").
-include_lib("emqx/include/logger.hrl").
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
@ -52,15 +58,22 @@ unreg() ->
on_gateway_load(_Gateway = #{ name := GwName,
config := Config
}, Ctx) ->
%% Step1. Fold the config to listeners
Listeners = emqx_gateway_utils:normalize_config(Config),
%% Step2. Start listeners or escokd:specs
ListenerPids = lists:map(fun(Lis) ->
start_listener(GwName, Ctx, Lis)
end, Listeners),
%% FIXME: How to throw an exception to interrupt the restart logic ?
%% FIXME: Assign ctx to GwState
{ok, ListenerPids, _GwState = #{ctx => Ctx}}.
Listeners = normalize_config(Config),
ModCfg = #{frame_mod => emqx_stomp_frame,
chann_mod => emqx_stomp_channel
},
case start_listeners(
Listeners, GwName, Ctx, ModCfg) of
{ok, ListenerPids} ->
%% FIXME: How to throw an exception to interrupt the restart logic ?
%% FIXME: Assign ctx to GwState
{ok, ListenerPids, _GwState = #{ctx => Ctx}};
{error, {Reason, Listener}} ->
throw({badconf, #{ key => listeners
, vallue => Listener
, reason => Reason
}})
end.
on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
GwName = maps:get(name, Gateway),
@ -80,68 +93,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
on_gateway_unload(_Gateway = #{ name := GwName,
config := Config
}, _GwState) ->
Listeners = emqx_gateway_utils:normalize_config(Config),
lists:foreach(fun(Lis) ->
stop_listener(GwName, Lis)
end, Listeners).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------
start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
{ok, Pid} ->
console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
[GwName, Type, LisName, ListenOnStr]),
Pid;
{error, Reason} ->
?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason]),
throw({badconf, Reason})
end.
start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
NCfg = Cfg#{
ctx => Ctx,
listener => {GwName, Type, LisName}, %% Used for authn
frame_mod => emqx_stomp_frame,
chann_mod => emqx_stomp_channel
},
esockd:open(Name, ListenOn, merge_default(SocketOpts),
{emqx_gateway_conn, start_link, [NCfg]}).
merge_default(Options) ->
Default = emqx_gateway_utils:default_tcp_options(),
case lists:keytake(tcp_options, 1, Options) of
{value, {tcp_options, TcpOpts}, Options1} ->
[{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)}
| Options1];
false ->
[{tcp_options, Default} | Options]
end.
stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
case StopRet of
ok ->
console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
[GwName, Type, LisName, ListenOnStr]);
{error, Reason} ->
?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
[GwName, Type, LisName, ListenOnStr, Reason])
end,
StopRet.
stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
esockd:close(Name, ListenOn).
-ifndef(TEST).
console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
-else.
console_print(_Fmt, _Args) -> ok.
-endif.
Listeners = normalize_config(Config),
stop_listeners(GwName, Listeners).

150
apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl Normal file
View File

@ -0,0 +1,150 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_gateway_cli_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-define(GP(S), begin S, receive {fmt, P} -> P; O -> O end end).
%% this parses to #{}, will not cause config cleanup
%% so we will need call emqx_config:erase
-define(CONF_DEFAULT, <<"
gateway {}
">>).
%%--------------------------------------------------------------------
%% Setup
%%--------------------------------------------------------------------
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Conf) ->
emqx_config:erase(gateway),
emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]),
Conf.
end_per_suite(Conf) ->
emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]),
Conf.
init_per_testcase(_, Conf) ->
Self = self(),
ok = meck:new(emqx_ctl, [passthrough, no_history, no_link]),
ok = meck:expect(emqx_ctl, usage,
fun(L) -> emqx_ctl:format_usage(L) end),
ok = meck:expect(emqx_ctl, print,
fun(Fmt) ->
Self ! {fmt, emqx_ctl:format(Fmt)}
end),
ok = meck:expect(emqx_ctl, print,
fun(Fmt, Args) ->
Self ! {fmt, emqx_ctl:format(Fmt, Args)}
end),
Conf.
end_per_testcase(_, _) ->
meck:unload([emqx_ctl]),
ok.
%%--------------------------------------------------------------------
%% Cases
%%--------------------------------------------------------------------
%% TODO:
t_load_unload(_) ->
ok.
t_gateway_registry_usage(_) ->
?assertEqual(
["gateway-registry list # List all registered gateways\n"],
emqx_gateway_cli:'gateway-registry'(usage)).
t_gateway_registry_list(_) ->
emqx_gateway_cli:'gateway-registry'(["list"]),
?assertEqual(
"Registered Name: coap, Callback Module: emqx_coap_impl\n"
"Registered Name: exproto, Callback Module: emqx_exproto_impl\n"
"Registered Name: lwm2m, Callback Module: emqx_lwm2m_impl\n"
"Registered Name: mqttsn, Callback Module: emqx_sn_impl\n"
"Registered Name: stomp, Callback Module: emqx_stomp_impl\n"
, acc_print()).
t_gateway_usage(_) ->
?assertEqual(
["gateway list # List all gateway\n",
"gateway lookup <Name> # Lookup a gateway detailed informations\n",
"gateway load <Name> <JsonConf> # Load a gateway with config\n",
"gateway unload <Name> # Unload the gateway\n",
"gateway stop <Name> # Stop the gateway\n",
"gateway start <Name> # Start the gateway\n"],
emqx_gateway_cli:gateway(usage)
).
t_gateway_list(_) ->
emqx_gateway_cli:gateway(["list"]),
?assertEqual(
"Gateway(name=coap, status=unloaded)\n"
"Gateway(name=exproto, status=unloaded)\n"
"Gateway(name=lwm2m, status=unloaded)\n"
"Gateway(name=mqttsn, status=unloaded)\n"
"Gateway(name=stomp, status=unloaded)\n"
, acc_print()).
t_gateway_load(_) ->
ok.
t_gateway_unload(_) ->
ok.
t_gateway_start(_) ->
ok.
t_gateway_stop(_) ->
ok.
t_gateway_clients_usage(_) ->
ok.
t_gateway_clients_list(_) ->
ok.
t_gateway_clients_lookup(_) ->
ok.
t_gateway_clients_kick(_) ->
ok.
t_gateway_metrcis_usage(_) ->
ok.
t_gateway_metrcis(_) ->
ok.
acc_print() ->
lists:concat(lists:reverse(acc_print([]))).
acc_print(Acc) ->
receive
{fmt, S} -> acc_print([S|Acc])
after 200 ->
Acc
end.

39
apps/emqx_machine/src/emqx_machine_boot.erl
View File

@ -96,7 +96,6 @@ reboot_apps() ->
, emqx_resource
, emqx_rule_engine
, emqx_bridge
, emqx_bridge_mqtt
, emqx_plugin_libs
, emqx_management
, emqx_retainer
@ -112,17 +111,17 @@ sorted_reboot_apps() ->
app_deps(App) ->
case application:get_key(App, applications) of
undefined -> [];
undefined -> undefined;
{ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List)
end.
sorted_reboot_apps(Apps) ->
G = digraph:new(),
try
lists:foreach(fun({App, Deps}) -> add_app(G, App, Deps) end, Apps),
NoDepApps = add_apps_to_digraph(G, Apps),
case digraph_utils:topsort(G) of
Sorted when is_list(Sorted) ->
Sorted;
Sorted ++ (NoDepApps -- Sorted);
false ->
Loops = find_loops(G),
error({circular_application_dependency, Loops})
@ -131,23 +130,33 @@ sorted_reboot_apps(Apps) ->
digraph:delete(G)
end.
add_app(G, App, undefined) ->
%% Build a dependency graph from the provided application list.
%% Return top-sort result of the apps.
%% Isolated apps without which are not dependency of any other apps are
%% put to the end of the list in the original order.
add_apps_to_digraph(G, Apps) ->
lists:foldl(fun
({App, undefined}, Acc) ->
?SLOG(debug, #{msg => "app_is_not_loaded", app => App}),
Acc;
({App, []}, Acc) ->
Acc ++ [App]; %% use '++' to keep the original order
({App, Deps}, Acc) ->
add_app_deps_to_digraph(G, App, Deps),
Acc
end, [], Apps).
add_app_deps_to_digraph(G, App, undefined) ->
?SLOG(debug, #{msg => "app_is_not_loaded", app => App}),
%% not loaded
add_app(G, App, []);
% We ALWAYS want to add `emqx_conf', even if no other app declare a
% dependency on it. Otherwise, emqx may fail to load the config
% schemas, especially in the test profile.
add_app(G, App = emqx_conf, []) ->
digraph:add_vertex(G, App),
add_app_deps_to_digraph(G, App, []);
add_app_deps_to_digraph(_G, _App, []) ->
ok;
add_app(_G, _App, []) ->
ok;
add_app(G, App, [Dep | Deps]) ->
add_app_deps_to_digraph(G, App, [Dep | Deps]) ->
digraph:add_vertex(G, App),
digraph:add_vertex(G, Dep),
digraph:add_edge(G, Dep, App), %% dep -> app as dependency
add_app(G, App, Deps).
add_app_deps_to_digraph(G, App, Deps).
find_loops(G) ->
lists:filtermap(

1
apps/emqx_management/src/emqx_mgmt_api.erl
View File

@ -30,6 +30,7 @@
-export([ node_query/5
, cluster_query/4
, select_table_with_count/5
, b2i/1
]).
-export([do_query/6]).

26
apps/emqx_management/src/emqx_mgmt_api_app.erl
View File

@ -91,16 +91,17 @@ fields(app) ->
"""They are useful for accessing public data anonymously,"""
"""and are used to associate API requests.""",
example => <<"MzAyMjk3ODMwMDk0NjIzOTUxNjcwNzQ0NzQ3MTE2NDYyMDI">>})},
{expired_at, hoconsc:mk(emqx_schema:rfc3339_system_time(),
{expired_at, hoconsc:mk(hoconsc:union([undefined, emqx_schema:rfc3339_system_time()]),
#{desc => "No longer valid datetime",
example => <<"2021-12-05T02:01:34.186Z">>,
nullable => true
nullable => true,
default => undefined
})},
{created_at, hoconsc:mk(emqx_schema:rfc3339_system_time(),
#{desc => "ApiKey create datetime",
example => <<"2021-12-01T00:00:00.000Z">>
})},
{desc, hoconsc:mk(emqx_schema:unicode_binary(),
{desc, hoconsc:mk(binary(),
#{example => <<"Note">>, nullable => true})},
{enable, hoconsc:mk(boolean(), #{desc => "Enable/Disable", nullable => true})}
];
@ -136,13 +137,19 @@ api_key(post, #{body := App}) ->
#{
<<"name">> := Name,
<<"desc">> := Desc0,
<<"expired_at">> := ExpiredAt,
<<"enable">> := Enable
} = App,
%% undefined is never expired
ExpiredAt0 = maps:get(<<"expired_at">>, App, <<"undefined">>),
ExpiredAt =
case ExpiredAt0 of
<<"undefined">> -> undefined;
_ -> ExpiredAt0
end,
Desc = unicode:characters_to_binary(Desc0, unicode),
case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of
{ok, NewApp} -> {200, format(NewApp)};
{error, Reason} -> {400, Reason}
{error, Reason} -> {400, io_lib:format("~p", [Reason])}
end.
api_key_by_name(get, #{bindings := #{name := Name}}) ->
@ -164,8 +171,13 @@ api_key_by_name(put, #{bindings := #{name := Name}, body := Body}) ->
{error, not_found} -> {404, <<"NOT_FOUND">>}
end.
format(App = #{expired_at := ExpiredAt, created_at := CreateAt}) ->
format(App = #{expired_at := ExpiredAt0, created_at := CreateAt}) ->
ExpiredAt =
case ExpiredAt0 of
undefined -> <<"undefined">>;
_ -> list_to_binary(calendar:system_time_to_rfc3339(ExpiredAt0))
end,
App#{
expired_at => list_to_binary(calendar:system_time_to_rfc3339(ExpiredAt)),
expired_at => ExpiredAt,
created_at => list_to_binary(calendar:system_time_to_rfc3339(CreateAt))
}.

31
apps/emqx_management/src/emqx_mgmt_api_banned.erl
View File

@ -101,7 +101,7 @@ fields(ban) ->
desc => <<"Banned type clientid, username, peerhost">>,
nullable => false,
example => username})},
{who, hoconsc:mk(emqx_schema:unicode_binary(), #{
{who, hoconsc:mk(binary(), #{
desc => <<"Client info as banned type">>,
nullable => false,
example => <<"Badass坏"/utf8>>})},
@ -109,19 +109,17 @@ fields(ban) ->
desc => <<"Commander">>,
nullable => true,
example => <<"mgmt_api">>})},
{reason, hoconsc:mk(emqx_schema:unicode_binary(), #{
{reason, hoconsc:mk(binary(), #{
desc => <<"Banned reason">>,
nullable => true,
example => <<"Too many requests">>})},
{at, hoconsc:mk(binary(), #{
{at, hoconsc:mk(emqx_schema:rfc3339_system_time(), #{
desc => <<"Create banned time, rfc3339, now if not specified">>,
nullable => true,
validator => fun is_rfc3339/1,
example => <<"2021-10-25T21:48:47+08:00">>})},
{until, hoconsc:mk(binary(), #{
{until, hoconsc:mk(emqx_schema:rfc3339_system_time(), #{
desc => <<"Cancel banned time, rfc3339, now + 5 minute if not specified">>,
nullable => true,
validator => fun is_rfc3339/1,
example => <<"2021-10-25T21:53:47+08:00">>})
}
];
@ -130,22 +128,19 @@ fields(meta) ->
emqx_dashboard_swagger:fields(limit) ++
[{count, hoconsc:mk(integer(), #{example => 1})}].
is_rfc3339(Time) ->
try
emqx_banned:to_timestamp(Time),
ok
catch _:_ -> {error, Time}
end.
banned(get, #{query_string := Params}) ->
Response = emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN),
{200, Response};
banned(post, #{body := Body}) ->
case emqx_banned:create(emqx_banned:parse(Body)) of
{ok, Banned} ->
{200, format(Banned)};
{error, {already_exist, Old}} ->
{400, #{code => 'ALREADY_EXISTED', message => format(Old)}}
case emqx_banned:parse(Body) of
{error, Reason} ->
{400, #{code => 'PARAMS_ERROR', message => list_to_binary(Reason)}};
Ban ->
case emqx_banned:create(Ban) of
{ok, Banned} -> {200, format(Banned)};
{error, {already_exist, Old}} ->
{400, #{code => 'ALREADY_EXISTED', message => format(Old)}}
end
end.
delete_banned(delete, #{bindings := Params}) ->

27
apps/emqx_management/src/emqx_mgmt_api_trace.erl
View File

@ -107,9 +107,14 @@ schema("/trace/:name/download") ->
get => #{
description => "Download trace log by name",
parameters => [hoconsc:ref(name)],
%% todo zip file octet-stream
responses => #{
200 => <<"TODO octet-stream">>
200 =>
#{description => "A trace zip file",
content => #{
'application/octet-stream' =>
#{schema => #{type => "string", format => "binary"}}
}
}
}
}
};
@ -124,9 +129,12 @@ schema("/trace/:name/log") ->
hoconsc:ref(position),
hoconsc:ref(node)
],
%% todo response data
responses => #{
200 => <<"TODO">>
200 =>
[
{items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}
| fields(bytes) ++ fields(position)
]
}
}
}.
@ -209,6 +217,7 @@ fields(position) ->
default => 0
})}].
-define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$").
validate_name(Name) ->
@ -296,7 +305,12 @@ download_trace_log(get, #{bindings := #{name := Name}}) ->
ZipFileName = ZipDir ++ binary_to_list(Name) ++ ".zip",
{ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]),
emqx_trace:delete_files_after_send(ZipFileName, Zips),
{200, ZipFile};
Headers = #{
<<"content-type">> => <<"application/x-zip">>,
<<"content-disposition">> =>
iolist_to_binary("attachment; filename=" ++ filename:basename(ZipFile))
},
{200, Headers, {file, ZipFile}};
{error, not_found} -> ?NOT_FOUND(Name)
end.
@ -324,11 +338,10 @@ cluster_call(Mod, Fun, Args, Timeout) ->
BadNodes =/= [] andalso ?LOG(error, "rpc call failed on ~p ~p", [BadNodes, {Mod, Fun, Args}]),
GoodRes.
stream_log_file(get, #{bindings := #{name := Name}, query_string := Query} = T) ->
stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) ->
Node0 = maps:get(<<"node">>, Query, atom_to_binary(node())),
Position = maps:get(<<"position">>, Query, 0),
Bytes = maps:get(<<"bytes">>, Query, 1000),
logger:error("~p", [T]),
case to_node(Node0) of
{ok, Node} ->
case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of

2
apps/emqx_management/src/emqx_mgmt_auth.erl
View File

@ -37,7 +37,7 @@
api_secret_hash = <<>> :: binary() | '_',
enable = true :: boolean() | '_',
desc = <<>> :: binary() | '_',
expired_at = 0 :: integer() | '_',
expired_at = 0 :: integer() | undefined | '_',
created_at = 0 :: integer() | '_'
}).

51
apps/emqx_management/src/emqx_mgmt_cli.erl
View File

@ -18,6 +18,7 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/logger.hrl").
-include("emqx_mgmt.hrl").
@ -386,18 +387,20 @@ trace(["list"]) ->
emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst])
end, emqx_trace_handler:running());
trace(["stop", Operation, ClientId]) ->
case trace_type(Operation) of
{ok, Type} -> trace_off(Type, ClientId);
trace(["stop", Operation, Filter0]) ->
case trace_type(Operation, Filter0) of
{ok, Type, Filter} -> trace_off(Type, Filter);
error -> trace([])
end;
trace(["start", Operation, ClientId, LogFile]) ->
trace(["start", Operation, ClientId, LogFile, "all"]);
trace(["start", Operation, ClientId, LogFile, Level]) ->
case trace_type(Operation) of
{ok, Type} -> trace_on(Type, ClientId, list_to_existing_atom(Level), LogFile);
trace(["start", Operation, Filter0, LogFile, Level]) ->
case trace_type(Operation, Filter0) of
{ok, Type, Filter} ->
trace_on(name(Filter0), Type, Filter,
list_to_existing_atom(Level), LogFile);
error -> trace([])
end;
@ -417,20 +420,23 @@ trace(_) ->
"Stop tracing for a client ip on local node"}
]).
trace_on(Who, Name, Level, LogFile) ->
case emqx_trace_handler:install(Who, Name, Level, LogFile) of
trace_on(Name, Type, Filter, Level, LogFile) ->
case emqx_trace_handler:install(Name, Type, Filter, Level, LogFile) of
ok ->
emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]);
emqx_trace:check(),
emqx_ctl:print("trace ~s ~s successfully~n", [Filter, Name]);
{error, Error} ->
emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Who, Name, Error])
emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Filter, Name, Error])
end.
trace_off(Who, Name) ->
case emqx_trace_handler:uninstall(Who, Name) of
trace_off(Type, Filter) ->
?TRACE("CLI", "trace_stopping", #{Type => Filter}),
case emqx_trace_handler:uninstall(Type, name(Filter)) of
ok ->
emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]);
emqx_trace:check(),
emqx_ctl:print("stop tracing ~s ~s successfully~n", [Type, Filter]);
{error, Error} ->
emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Name, Error])
emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Type, Filter, Error])
end.
%%--------------------------------------------------------------------
@ -459,9 +465,9 @@ traces(["delete", Name]) ->
traces(["start", Name, Operation, Filter]) ->
traces(["start", Name, Operation, Filter, "900"]);
traces(["start", Name, Operation, Filter, DurationS]) ->
case trace_type(Operation) of
{ok, Type} -> trace_cluster_on(Name, Type, Filter, DurationS);
traces(["start", Name, Operation, Filter0, DurationS]) ->
case trace_type(Operation, Filter0) of
{ok, Type, Filter} -> trace_cluster_on(Name, Type, Filter, DurationS);
error -> traces([])
end;
@ -503,10 +509,10 @@ trace_cluster_off(Name) ->
{error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error])
end.
trace_type("client") -> {ok, clientid};
trace_type("topic") -> {ok, topic};
trace_type("ip_address") -> {ok, ip_address};
trace_type(_) -> error.
trace_type("client", ClientId) -> {ok, clientid, list_to_binary(ClientId)};
trace_type("topic", Topic) -> {ok, topic, list_to_binary(Topic)};
trace_type("ip_address", IP) -> {ok, ip_address, IP};
trace_type(_, _) -> error.
%%--------------------------------------------------------------------
%% @doc Listeners Command
@ -716,3 +722,6 @@ format_listen_on({Addr, Port}) when is_list(Addr) ->
io_lib:format("~ts:~w", [Addr, Port]);
format_listen_on({Addr, Port}) when is_tuple(Addr) ->
io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]).
name(Filter) ->
iolist_to_binary(["CLI-", Filter]).

19
apps/emqx_management/test/emqx_mgmt_auth_api_SUITE.erl
View File

@ -23,7 +23,7 @@
all() -> [{group, parallel}, {group, sequence}].
suite() -> [{timetrap, {minutes, 1}}].
groups() -> [
{parallel, [parallel], [t_create, t_update, t_delete, t_authorize]},
{parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]},
{sequence, [], [t_create_failed]}
].
@ -137,7 +137,15 @@ t_authorize(_Config) ->
},
?assertMatch({ok, #{<<"api_key">> := _, <<"enable">> := true}}, update_app(Name, Expired)),
?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)),
ok.
t_create_unexpired_app(_Config) ->
Name1 = <<"EMQX-UNEXPIRED-API-KEY-1">>,
Name2 = <<"EMQX-UNEXPIRED-API-KEY-2">>,
{ok, Create1} = create_unexpired_app(Name1, #{}),
?assertMatch(#{<<"expired_at">> := <<"undefined">>}, Create1),
{ok, Create2} = create_unexpired_app(Name2, #{expired_at => <<"undefined">>}),
?assertMatch(#{<<"expired_at">> := <<"undefined">>}, Create2),
ok.
@ -170,6 +178,15 @@ create_app(Name) ->
Error -> Error
end.
create_unexpired_app(Name, Params) ->
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["api_key"]),
App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params),
case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of
{ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
Error -> Error
end.
delete_app(Name) ->
DeletePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]),
emqx_mgmt_api_test_util:request_api(delete, DeletePath).

144
apps/emqx_management/test/emqx_mgmt_banned_api_SUITE.erl Normal file
View File

@ -0,0 +1,144 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_mgmt_banned_api_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_mgmt_api_test_util:init_suite(),
Config.
end_per_suite(_) ->
emqx_mgmt_api_test_util:end_suite().
t_create(_Config) ->
Now = erlang:system_time(second),
At = emqx_banned:to_rfc3339(Now),
Until = emqx_banned:to_rfc3339(Now + 3),
ClientId = <<"TestClient测试"/utf8>>,
By = <<"banned suite测试组"/utf8>>,
Reason = <<"test测试"/utf8>>,
As = <<"clientid">>,
ClientIdBanned = #{
as => As,
who => ClientId,
by => By,
reason => Reason,
at => At,
until => Until
},
{ok, ClientIdBannedRes} = create_banned(ClientIdBanned),
?assertEqual(#{<<"as">> => As,
<<"at">> => At,
<<"by">> => By,
<<"reason">> => Reason,
<<"until">> => Until,
<<"who">> => ClientId
}, ClientIdBannedRes),
PeerHost = <<"192.168.2.13">>,
PeerHostBanned = #{
as => <<"peerhost">>,
who => PeerHost,
by => By,
reason => Reason,
at => At,
until => Until
},
{ok, PeerHostBannedRes} = create_banned(PeerHostBanned),
?assertEqual(#{<<"as">> => <<"peerhost">>,
<<"at">> => At,
<<"by">> => By,
<<"reason">> => Reason,
<<"until">> => Until,
<<"who">> => PeerHost
}, PeerHostBannedRes),
{ok, #{<<"data">> := List}} = list_banned(),
Bans = lists:sort(lists:map(fun(#{<<"who">> := W, <<"as">> := A}) -> {A, W} end, List)),
?assertEqual([{<<"clientid">>, ClientId}, {<<"peerhost">>, PeerHost}], Bans),
ok.
t_create_failed(_Config) ->
Now = erlang:system_time(second),
At = emqx_banned:to_rfc3339(Now),
Until = emqx_banned:to_rfc3339(Now + 10),
Who = <<"BadHost"/utf8>>,
By = <<"banned suite测试组"/utf8>>,
Reason = <<"test测试"/utf8>>,
As = <<"peerhost">>,
BadPeerHost = #{
as => As,
who => Who,
by => By,
reason => Reason,
at => At,
until => Until
},
BadRequest = {error, {"HTTP/1.1", 400, "Bad Request"}},
?assertEqual(BadRequest, create_banned(BadPeerHost)),
Expired = BadPeerHost#{until => emqx_banned:to_rfc3339(Now - 1),
who => <<"127.0.0.1">>},
?assertEqual(BadRequest, create_banned(Expired)),
ok.
t_delete(_Config) ->
Now = erlang:system_time(second),
At = emqx_banned:to_rfc3339(Now),
Until = emqx_banned:to_rfc3339(Now + 3),
Who = <<"TestClient-"/utf8>>,
By = <<"banned suite 中"/utf8>>,
Reason = <<"test测试"/utf8>>,
As = <<"clientid">>,
Banned = #{
as => clientid,
who => Who,
by => By,
reason => Reason,
at => At,
until => Until
},
{ok, _} = create_banned(Banned),
?assertMatch({ok, _}, delete_banned(binary_to_list(As), binary_to_list(Who))),
?assertMatch({error,{"HTTP/1.1",404,"Not Found"}},
delete_banned(binary_to_list(As), binary_to_list(Who))),
ok.
list_banned() ->
Path = emqx_mgmt_api_test_util:api_path(["banned"]),
case emqx_mgmt_api_test_util:request_api(get, Path) of
{ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])};
Error -> Error
end.
create_banned(Banned) ->
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
Path = emqx_mgmt_api_test_util:api_path(["banned"]),
case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Banned) of
{ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
Error -> Error
end.
delete_banned(As, Who) ->
DeletePath = emqx_mgmt_api_test_util:api_path(["banned", As, Who]),
emqx_mgmt_api_test_util:request_api(delete, DeletePath).
to_rfc3339(Sec) ->
list_to_binary(calendar:system_time_to_rfc3339(Sec)).

4
apps/emqx_modules/src/emqx_delayed.erl
View File

@ -155,7 +155,7 @@ format_delayed(#delayed_message{key = {ExpectTimeStamp, Id}, delayed = Delayed,
},
case WithPayload of
true ->
Result#{payload => base64:encode(Payload)};
Result#{payload => Payload};
_ ->
Result
end.
@ -187,7 +187,7 @@ delete_delayed_message(Id0) ->
mria:dirty_delete(?TAB, {Timestamp, Id})
end.
update_config(Config) ->
{ok, _} = emqx:update_config([delayed], Config).
emqx_conf:update([delayed], Config, #{rawconf_with_defaults => true, override_to => cluster}).
%%--------------------------------------------------------------------
%% gen_server callback

80
apps/emqx_modules/src/emqx_delayed_api.erl
View File

@ -25,12 +25,14 @@
-define(MAX_PAYLOAD_LENGTH, 2048).
-define(PAYLOAD_TOO_LARGE, 'PAYLOAD_TOO_LARGE').
-export([status/2
, delayed_messages/2
, delayed_message/2
]).
-export([ status/2
, delayed_messages/2
, delayed_message/2
]).
-export([paths/0, fields/1, schema/1]).
-export([ paths/0
, fields/1
, schema/1]).
%% for rpc
-export([update_config_/1]).
@ -40,15 +42,21 @@
-define(ALREADY_ENABLED, 'ALREADY_ENABLED').
-define(ALREADY_DISABLED, 'ALREADY_DISABLED').
-define(INTERNAL_ERROR, 'INTERNAL_ERROR').
-define(BAD_REQUEST, 'BAD_REQUEST').
-define(MESSAGE_ID_NOT_FOUND, 'MESSAGE_ID_NOT_FOUND').
-define(MESSAGE_ID_SCHEMA_ERROR, 'MESSAGE_ID_SCHEMA_ERROR').
-define(MAX_PAYLOAD_SIZE, 1048576). %% 1MB = 1024 x 1024
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE).
paths() -> ["/mqtt/delayed", "/mqtt/delayed/messages", "/mqtt/delayed/messages/:msgid"].
paths() ->
[ "/mqtt/delayed"
, "/mqtt/delayed/messages"
, "/mqtt/delayed/messages/:msgid"
].
schema("/mqtt/delayed") ->
#{
@ -157,11 +165,11 @@ delayed_message(get, #{bindings := #{msgid := Id}}) ->
case emqx_delayed:get_delayed_message(Id) of
{ok, Message} ->
Payload = maps:get(payload, Message),
case size(Payload) > ?MAX_PAYLOAD_LENGTH of
case erlang:byte_size(Payload) > ?MAX_PAYLOAD_SIZE of
true ->
{200, Message#{payload => ?PAYLOAD_TOO_LARGE}};
{200, Message};
_ ->
{200, Message#{payload => Payload}}
{200, Message#{payload => base64:encode(Payload)}}
end;
{error, id_schema_error} ->
{400, generate_http_code_map(id_schema_error, Id)};
@ -188,8 +196,7 @@ get_status() ->
update_config(Config) ->
case generate_config(Config) of
{ok, Config} ->
update_config_(Config),
{200, get_status()};
update_config_(Config);
{error, {Code, Message}} ->
{400, #{code => Code, message => Message}}
end.
@ -214,29 +221,28 @@ generate_max_delayed_messages(Config) ->
{ok, Config}.
update_config_(Config) ->
lists:foreach(fun(Node) ->
update_config_(Node, Config)
end, mria_mnesia:running_nodes()).
update_config_(Node, Config) when Node =:= node() ->
_ = emqx_delayed:update_config(Config),
case maps:get(<<"enable">>, Config, undefined) of
undefined ->
ignore;
true ->
emqx_delayed:enable();
false ->
emqx_delayed:disable()
end,
case maps:get(<<"max_delayed_messages">>, Config, undefined) of
undefined ->
ignore;
Max ->
ok = emqx_delayed:set_max_delayed_messages(Max)
end;
update_config_(Node, Config) ->
rpc_call(Node, ?MODULE, ?FUNCTION_NAME, [Node, Config]).
case emqx_delayed:update_config(Config) of
{ok, #{raw_config := NewDelayed}} ->
case maps:get(<<"enable">>, Config, undefined) of
undefined ->
ignore;
true ->
emqx_delayed:enable();
false ->
emqx_delayed:disable()
end,
case maps:get(<<"max_delayed_messages">>, Config, undefined) of
undefined ->
ignore;
Max ->
ok = emqx_delayed:set_max_delayed_messages(Max)
end,
{200, NewDelayed};
{error, Reason} ->
Message = list_to_binary(
io_lib:format("Update config failed ~p", [Reason])),
{500, ?INTERNAL_ERROR, Message}
end.
generate_http_code_map(id_schema_error, Id) ->
#{code => ?MESSAGE_ID_SCHEMA_ERROR, message =>
@ -244,9 +250,3 @@ generate_http_code_map(id_schema_error, Id) ->
generate_http_code_map(not_found, Id) ->
#{code => ?MESSAGE_ID_NOT_FOUND, message =>
iolist_to_binary(io_lib:format("Message ID ~p not found", [Id]))}.
rpc_call(Node, Module, Fun, Args) ->
case rpc:call(Node, Module, Fun, Args) of
{badrpc, Reason} -> {error, Reason};
Result -> Result
end.

11
apps/emqx_modules/src/emqx_event_message.erl
View File

@ -44,8 +44,15 @@ list() ->
update(Params) ->
disable(),
{ok, _} = emqx:update_config([event_message], Params),
enable().
case emqx_conf:update([event_message],
Params,
#{rawconf_with_defaults => true, override_to => cluster}) of
{ok, #{raw_config := NewEventMessage}} ->
enable(),
{ok, NewEventMessage};
{error, Reason} ->
{error, Reason}
end.
enable() ->
lists:foreach(fun({_Topic, false}) -> ok;

9
apps/emqx_modules/src/emqx_event_message_api.erl
View File

@ -53,5 +53,10 @@ event_message(get, _Params) ->
{200, emqx_event_message:list()};
event_message(put, #{body := Body}) ->
_ = emqx_event_message:update(Body),
{200, emqx_event_message:list()}.
case emqx_event_message:update(Body) of
{ok, NewConfig} ->
{200, NewConfig};
{error, Reason} ->
Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])),
{500, 'INTERNAL_ERROR', Message}
end.

Some files were not shown because too many files have changed in this diff Show More