yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #12997 from thalesmg/sync-r57-m-20240508 

sync `release-57` to `master`
This commit is contained in:
Ivan Dyachkov 2024-05-10 10:41:30 +02:00 committed by GitHub
parent 23a1b18459 401f0fa84b
commit 900536712d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
119 changed files with 2780 additions and 675 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/emqx/include/emqx_trace.hrl
View File

@ -35,6 +35,11 @@
end_at :: integer() | undefined | '_'
}).
-record(emqx_trace_format_func_data, {
function :: fun((any()) -> any()),
data :: any()
}).
-define(SHARD, ?COMMON_SHARD).
-define(MAX_SIZE, 30).

12
apps/emqx/src/emqx_broker_helper.erl
View File

@ -110,7 +110,7 @@ reclaim_seq(Topic) ->
stats_fun() ->
safe_update_stats(subscriber_val(), 'subscribers.count', 'subscribers.max'),
safe_update_stats(table_size(?SUBSCRIPTION), 'subscriptions.count', 'subscriptions.max'),
safe_update_stats(subscription_count(), 'subscriptions.count', 'subscriptions.max'),
safe_update_stats(table_size(?SUBOPTION), 'suboptions.count', 'suboptions.max').
safe_update_stats(undefined, _Stat, _MaxStat) ->
@ -118,6 +118,16 @@ safe_update_stats(undefined, _Stat, _MaxStat) ->
safe_update_stats(Val, Stat, MaxStat) when is_integer(Val) ->
emqx_stats:setstat(Stat, MaxStat, Val).
subscription_count() ->
NonPSCount = table_size(?SUBSCRIPTION),
PSCount = emqx_persistent_session_bookkeeper:get_subscription_count(),
case is_integer(NonPSCount) of
true ->
NonPSCount + PSCount;
false ->
PSCount
end.
subscriber_val() ->
sum_subscriber(table_size(?SUBSCRIBER), table_size(?SHARED_SUBSCRIBER)).

48
apps/emqx/src/emqx_channel.erl
View File

@ -1075,7 +1075,7 @@ handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel = ?IS_MQTT_V5) -
Packet = ?DISCONNECT_PACKET(ReasonCode, Props),
{ok, [?REPLY_OUTGOING(Packet), ?REPLY_CLOSE(ReasonName)], Channel};
handle_out(disconnect, {_ReasonCode, ReasonName, _Props}, Channel) ->
{ok, {close, ReasonName}, Channel};
{ok, ?REPLY_CLOSE(ReasonName), Channel};
handle_out(auth, {ReasonCode, Properties}, Channel) ->
{ok, ?AUTH_PACKET(ReasonCode, Properties), Channel};
handle_out(Type, Data, Channel) ->
@ -1406,6 +1406,16 @@ handle_timeout(
{_, Quota2} ->
{ok, clean_timer(TimerName, Channel#channel{quota = Quota2})}
end;
handle_timeout(
_TRef,
connection_expire,
#channel{conn_state = ConnState} = Channel0
) ->
Channel1 = clean_timer(connection_expire, Channel0),
case ConnState of
disconnected -> {ok, Channel1};
_ -> handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel1)
end;
handle_timeout(TRef, Msg, Channel) ->
case emqx_hooks:run_fold('client.timeout', [TRef, Msg], []) of
[] ->
@ -1810,18 +1820,23 @@ log_auth_failure(Reason) ->
%% Merge authentication result into ClientInfo
%% Authentication result may include:
%% 1. `is_superuser': The superuser flag from various backends
%% 2. `acl': ACL rules from JWT, HTTP auth backend
%% 3. `client_attrs': Extra client attributes from JWT, HTTP auth backend
%% 4. Maybe more non-standard fields used by hook callbacks
%% 2. `expire_at`: Authentication validity deadline, the client will be disconnected after this time
%% 3. `acl': ACL rules from JWT, HTTP auth backend
%% 4. `client_attrs': Extra client attributes from JWT, HTTP auth backend
%% 5. Maybe more non-standard fields used by hook callbacks
merge_auth_result(ClientInfo, AuthResult0) when is_map(ClientInfo) andalso is_map(AuthResult0) ->
IsSuperuser = maps:get(is_superuser, AuthResult0, false),
AuthResult = maps:without([client_attrs], AuthResult0),
ExpireAt = maps:get(expire_at, AuthResult0, undefined),
AuthResult = maps:without([client_attrs, expire_at], AuthResult0),
Attrs0 = maps:get(client_attrs, ClientInfo, #{}),
Attrs1 = maps:get(client_attrs, AuthResult0, #{}),
Attrs = maps:merge(Attrs0, Attrs1),
NewClientInfo = maps:merge(
ClientInfo#{client_attrs => Attrs},
AuthResult#{is_superuser => IsSuperuser}
AuthResult#{
is_superuser => IsSuperuser,
auth_expire_at => ExpireAt
}
),
fix_mountpoint(NewClientInfo).
@ -2228,10 +2243,16 @@ ensure_connected(
) ->
NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
ok = run_hooks('client.connected', [ClientInfo, NConnInfo]),
Channel#channel{
schedule_connection_expire(Channel#channel{
conninfo = trim_conninfo(NConnInfo),
conn_state = connected
}.
}).
schedule_connection_expire(Channel = #channel{clientinfo = #{auth_expire_at := undefined}}) ->
Channel;
schedule_connection_expire(Channel = #channel{clientinfo = #{auth_expire_at := ExpireAt}}) ->
Interval = max(0, ExpireAt - erlang:system_time(millisecond)),
ensure_timer(connection_expire, Interval, Channel).
trim_conninfo(ConnInfo) ->
maps:without(
@ -2615,10 +2636,15 @@ disconnect_and_shutdown(
->
NChannel = ensure_disconnected(Reason, Channel),
shutdown(Reason, Reply, ?DISCONNECT_PACKET(reason_code(Reason)), NChannel);
%% mqtt v3/v4 sessions, mqtt v5 other conn_state sessions
disconnect_and_shutdown(Reason, Reply, Channel) ->
%% mqtt v3/v4 connected sessions
disconnect_and_shutdown(Reason, Reply, Channel = #channel{conn_state = ConnState}) when
ConnState =:= connected orelse ConnState =:= reauthenticating
->
NChannel = ensure_disconnected(Reason, Channel),
shutdown(Reason, Reply, NChannel).
shutdown(Reason, Reply, NChannel);
%% other conn_state sessions
disconnect_and_shutdown(Reason, Reply, Channel) ->
shutdown(Reason, Reply, Channel).
-compile({inline, [sp/1, flag/1]}).
sp(true) -> 1;

4
apps/emqx/src/emqx_cm_sup.erl
View File

@ -53,6 +53,7 @@ init([]) ->
RegistryKeeper = child_spec(emqx_cm_registry_keeper, 5000, worker),
Manager = child_spec(emqx_cm, 5000, worker),
DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor),
DSSessionBookkeeper = child_spec(emqx_persistent_session_bookkeeper, 5_000, worker),
Children =
[
Banned,
@ -62,7 +63,8 @@ init([]) ->
Registry,
RegistryKeeper,
Manager,
DSSessionGCSup
DSSessionGCSup,
DSSessionBookkeeper
],
{ok, {SupFlags, Children}}.

4
apps/emqx/src/emqx_listeners.erl
View File

@ -1036,8 +1036,8 @@ to_quicer_listener_opts(Opts) ->
SSLOpts = maps:from_list(ssl_opts(Opts)),
Opts1 = maps:filter(
fun
(cacertfile, undefined) -> fasle;
(password, undefined) -> fasle;
(cacertfile, undefined) -> false;
(password, undefined) -> false;
(_, _) -> true
end,
Opts

2
apps/emqx/src/emqx_logger_jsonfmt.erl
View File

@ -229,7 +229,7 @@ best_effort_json_obj(Map, Config) ->
do_format_msg("~p", [Map], Config)
end.
json(A, _) when is_atom(A) -> atom_to_binary(A, utf8);
json(A, _) when is_atom(A) -> A;
json(I, _) when is_integer(I) -> I;
json(F, _) when is_float(F) -> F;
json(P, C) when is_pid(P) -> json(pid_to_list(P), C);

107
apps/emqx/src/emqx_persistent_session_bookkeeper.erl Normal file
View File

@ -0,0 +1,107 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_persistent_session_bookkeeper).
-behaviour(gen_server).
%% API
-export([
start_link/0,
get_subscription_count/0
]).
%% `gen_server' API
-export([
init/1,
handle_continue/2,
handle_call/3,
handle_cast/2,
handle_info/2
]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
%% call/cast/info events
-record(tally_subs, {}).
-record(get_subscription_count, {}).
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
-spec start_link() -> gen_server:start_ret().
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, _InitOpts = #{}, _Opts = []).
%% @doc Gets a cached view of the cluster-global count of persistent subscriptions.
-spec get_subscription_count() -> non_neg_integer().
get_subscription_count() ->
case emqx_persistent_message:is_persistence_enabled() of
true ->
gen_server:call(?MODULE, #get_subscription_count{}, infinity);
false ->
0
end.
%%------------------------------------------------------------------------------
%% `gen_server' API
%%------------------------------------------------------------------------------
init(_Opts) ->
case emqx_persistent_message:is_persistence_enabled() of
true ->
State = #{subs_count => 0},
{ok, State, {continue, #tally_subs{}}};
false ->
ignore
end.
handle_continue(#tally_subs{}, State0) ->
State = tally_persistent_subscriptions(State0),
ensure_subs_tally_timer(),
{noreply, State}.
handle_call(#get_subscription_count{}, _From, State) ->
#{subs_count := N} = State,
{reply, N, State};
handle_call(_Call, _From, State) ->
{reply, {error, bad_call}, State}.
handle_cast(_Cast, State) ->
{noreply, State}.
handle_info(#tally_subs{}, State0) ->
State = tally_persistent_subscriptions(State0),
ensure_subs_tally_timer(),
{noreply, State};
handle_info(_Info, State) ->
{noreply, State}.
%%------------------------------------------------------------------------------
%% Internal fns
%%------------------------------------------------------------------------------
tally_persistent_subscriptions(State0) ->
N = emqx_persistent_session_ds_state:total_subscription_count(),
State0#{subs_count := N}.
ensure_subs_tally_timer() ->
Timeout = emqx_config:get([session_persistence, subscription_count_refresh_interval]),
_ = erlang:send_after(Timeout, self(), #tally_subs{}),
ok.

28
apps/emqx/src/emqx_persistent_session_ds.erl
View File

@ -658,16 +658,17 @@ replay_batch(Srs0, Session0, ClientInfo) ->
%%--------------------------------------------------------------------
-spec disconnect(session(), emqx_types:conninfo()) -> {shutdown, session()}.
disconnect(Session = #{s := S0}, ConnInfo) ->
S1 = emqx_persistent_session_ds_state:set_last_alive_at(now_ms(), S0),
S2 =
disconnect(Session = #{id := Id, s := S0}, ConnInfo) ->
S1 = maybe_set_offline_info(S0, Id),
S2 = emqx_persistent_session_ds_state:set_last_alive_at(now_ms(), S1),
S3 =
case ConnInfo of
#{expiry_interval := EI} when is_number(EI) ->
emqx_persistent_session_ds_state:set_expiry_interval(EI, S1);
emqx_persistent_session_ds_state:set_expiry_interval(EI, S2);
_ ->
S1
S2
end,
S = emqx_persistent_session_ds_state:commit(S2),
S = emqx_persistent_session_ds_state:commit(S3),
{shutdown, Session#{s => S}}.
-spec terminate(Reason :: term(), session()) -> ok.
@ -702,7 +703,7 @@ list_client_subscriptions(ClientId) ->
maps:fold(
fun(Topic, #{current_state := CS}, Acc) ->
#{subopts := SubOpts} = maps:get(CS, SStates),
Elem = {Topic, SubOpts},
Elem = {Topic, SubOpts#{durable => true}},
[Elem | Acc]
end,
[],
@ -1175,6 +1176,19 @@ try_get_live_session(ClientId) ->
not_found
end.
-spec maybe_set_offline_info(emqx_persistent_session_ds_state:t(), emqx_types:clientid()) ->
emqx_persistent_session_ds_state:t().
maybe_set_offline_info(S, Id) ->
case emqx_cm:lookup_client({clientid, Id}) of
[{_Key, ChannelInfo, Stats}] ->
emqx_persistent_session_ds_state:set_offline_info(
#{chan_info => ChannelInfo, stats => Stats},
S
);
_ ->
S
end.
%%--------------------------------------------------------------------
%% SeqNo tracking
%% --------------------------------------------------------------------

1
apps/emqx/src/emqx_persistent_session_ds.hrl
View File

@ -81,5 +81,6 @@
-define(will_message, will_message).
-define(clientinfo, clientinfo).
-define(protocol, protocol).
-define(offline_info, offline_info).
-endif.

12
apps/emqx/src/emqx_persistent_session_ds_state.erl
View File

@ -35,6 +35,7 @@
-export([get_expiry_interval/1, set_expiry_interval/2]).
-export([get_clientinfo/1, set_clientinfo/2]).
-export([get_will_message/1, set_will_message/2, clear_will_message/1, clear_will_message_now/1]).
-export([set_offline_info/2]).
-export([get_peername/1, set_peername/2]).
-export([get_protocol/1, set_protocol/2]).
-export([new_id/1]).
@ -53,6 +54,7 @@
cold_get_subscription/2,
fold_subscriptions/3,
n_subscriptions/1,
total_subscription_count/0,
put_subscription/3,
del_subscription/2
]).
@ -372,6 +374,10 @@ clear_will_message_now(SessionId) when is_binary(SessionId) ->
clear_will_message(Rec) ->
set_will_message(undefined, Rec).
-spec set_offline_info(_Info :: map(), t()) -> t().
set_offline_info(Info, Rec) ->
set_meta(?offline_info, Info, Rec).
-spec new_id(t()) -> {emqx_persistent_session_ds:subscription_id(), t()}.
new_id(Rec) ->
LastId =
@ -401,6 +407,12 @@ fold_subscriptions(Fun, Acc, Rec) ->
n_subscriptions(Rec) ->
gen_size(?subscriptions, Rec).
-spec total_subscription_count() -> non_neg_integer().
total_subscription_count() ->
mria:async_dirty(?DS_MRIA_SHARD, fun() ->
mnesia:foldl(fun(#kv{}, Acc) -> Acc + 1 end, 0, ?subscription_tab)
end).
-spec put_subscription(
emqx_persistent_session_ds:topic_filter(),
emqx_persistent_session_ds_subs:subscription(),

12
apps/emqx/src/emqx_router_helper.erl
View File

@ -189,7 +189,17 @@ code_change(_OldVsn, State, _Extra) ->
%%--------------------------------------------------------------------
stats_fun() ->
emqx_stats:setstat('topics.count', 'topics.max', emqx_router:stats(n_routes)).
PSRouteCount = persistent_route_count(),
NonPSRouteCount = emqx_router:stats(n_routes),
emqx_stats:setstat('topics.count', 'topics.max', PSRouteCount + NonPSRouteCount).
persistent_route_count() ->
case emqx_persistent_message:is_persistence_enabled() of
true ->
emqx_persistent_session_ds_router:stats(n_routes);
false ->
0
end.
cleanup_routes(Node) ->
emqx_router:cleanup_routes(Node).

8
apps/emqx/src/emqx_schema.erl
View File

@ -1713,6 +1713,14 @@ fields("session_persistence") ->
desc => ?DESC(session_ds_session_gc_batch_size)
}
)},
{"subscription_count_refresh_interval",
sc(
timeout_duration(),
#{
default => <<"5s">>,
importance => ?IMPORTANCE_HIDDEN
}
)},
{"message_retention_period",
sc(
timeout_duration(),

8
apps/emqx/src/emqx_tls_lib.erl
View File

@ -545,13 +545,19 @@ to_client_opts(Type, Opts) ->
{depth, Get(depth)},
{password, ensure_str(Get(password))},
{secure_renegotiate, Get(secure_renegotiate)}
],
] ++ hostname_check(Verify),
Versions
);
false ->
[]
end.
hostname_check(verify_none) ->
[];
hostname_check(verify_peer) ->
%% allow wildcard certificates
[{customize_hostname_check, [{match_fun, public_key:pkix_verify_hostname_match_fun(https)}]}].
resolve_cert_path_for_read_strict(Path) ->
case resolve_cert_path_for_read(Path) of
undefined ->

54
apps/emqx/src/emqx_trace/emqx_trace.erl
View File

@ -31,7 +31,8 @@
log/4,
rendered_action_template/2,
make_rendered_action_template_trace_context/1,
rendered_action_template_with_ctx/2
rendered_action_template_with_ctx/2,
is_rule_trace_active/0
]).
-export([
@ -96,6 +97,16 @@ unsubscribe(Topic, SubOpts) ->
?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}).
rendered_action_template(<<"action:", _/binary>> = ActionID, RenderResult) ->
do_rendered_action_template(ActionID, RenderResult);
rendered_action_template(#{mod := _, func := _} = ActionID, RenderResult) ->
do_rendered_action_template(ActionID, RenderResult);
rendered_action_template(_ActionID, _RenderResult) ->
%% We do nothing if we don't get a valid Action ID. This can happen when
%% called from connectors that are used for actions as well as authz and
%% authn.
ok.
do_rendered_action_template(ActionID, RenderResult) ->
TraceResult = ?TRACE(
"QUERY_RENDER",
"action_template_rendered",
@ -108,23 +119,25 @@ rendered_action_template(<<"action:", _/binary>> = ActionID, RenderResult) ->
#{stop_action_after_render := true} ->
%% We throw an unrecoverable error to stop action before the
%% resource is called/modified
StopMsg = lists:flatten(
ActionIDStr =
case ActionID of
Bin when is_binary(Bin) ->
Bin;
Term ->
ActionIDFormatted = io_lib:format("~tw", [Term]),
unicode:characters_to_binary(ActionIDFormatted)
end,
StopMsg =
io_lib:format(
"Action ~ts stopped after template rendering due to test setting.",
[ActionID]
)
),
[ActionIDStr]
),
MsgBin = unicode:characters_to_binary(StopMsg),
error(?EMQX_TRACE_STOP_ACTION(MsgBin));
_ ->
ok
end,
TraceResult;
rendered_action_template(_ActionID, _RenderResult) ->
%% We do nothing if we don't get a valid Action ID. This can happen when
%% called from connectors that are used for actions as well as authz and
%% authn.
ok.
TraceResult.
%% The following two functions are used for connectors that don't do the
%% rendering in the main process (the one that called on_*query). In this case
@ -165,6 +178,16 @@ rendered_action_template_with_ctx(
logger:set_process_metadata(OldMetaData)
end.
is_rule_trace_active() ->
case logger:get_process_metadata() of
#{rule_id := RID} when is_binary(RID) ->
true;
#{rule_ids := RIDs} when map_size(RIDs) > 0 ->
true;
_ ->
false
end.
log(List, Msg, Meta) ->
log(debug, List, Msg, Meta).
@ -382,7 +405,14 @@ code_change(_, State, _Extra) ->
{ok, State}.
insert_new_trace(Trace) ->
transaction(fun emqx_trace_dl:insert_new_trace/1, [Trace]).
case transaction(fun emqx_trace_dl:insert_new_trace/1, [Trace]) of
{error, _} = Error ->
Error;
Res ->
%% We call this to ensure the trace is active when we return
check(),
Res
end.
update_trace(Traces) ->
Now = now_second(),

29
apps/emqx/src/emqx_trace/emqx_trace_formatter.erl
View File

@ -15,9 +15,11 @@
%%--------------------------------------------------------------------
-module(emqx_trace_formatter).
-include("emqx_mqtt.hrl").
-include("emqx_trace.hrl").
-export([format/2]).
-export([format_meta_map/1]).
-export([evaluate_lazy_values/1]).
%% logger_formatter:config/0 is not exported.
-type config() :: map().
@ -28,18 +30,35 @@
LogEvent :: logger:log_event(),
Config :: config().
format(
#{level := debug, meta := Meta = #{trace_tag := Tag}, msg := Msg},
#{level := debug, meta := Meta0 = #{trace_tag := Tag}, msg := Msg},
#{payload_encode := PEncode}
) ->
Meta1 = evaluate_lazy_values(Meta0),
Time = emqx_utils_calendar:now_to_rfc3339(microsecond),
ClientId = to_iolist(maps:get(clientid, Meta, "")),
Peername = maps:get(peername, Meta, ""),
MetaBin = format_meta(Meta, PEncode),
ClientId = to_iolist(maps:get(clientid, Meta1, "")),
Peername = maps:get(peername, Meta1, ""),
MetaBin = format_meta(Meta1, PEncode),
Msg1 = to_iolist(Msg),
Tag1 = to_iolist(Tag),
[Time, " [", Tag1, "] ", ClientId, "@", Peername, " msg: ", Msg1, ", ", MetaBin, "\n"];
format(Event, Config) ->
emqx_logger_textfmt:format(Event, Config).
emqx_logger_textfmt:format(evaluate_lazy_values(Event), Config).
evaluate_lazy_values(Map) when is_map(Map) ->
maps:map(fun evaluate_lazy_values_kv/2, Map);
evaluate_lazy_values(V) ->
V.
evaluate_lazy_values_kv(_K, #emqx_trace_format_func_data{function = Formatter, data = V}) ->
try
NewV = Formatter(V),
evaluate_lazy_values(NewV)
catch
_:_ ->
V
end;
evaluate_lazy_values_kv(_K, V) ->
evaluate_lazy_values(V).
format_meta_map(Meta) ->
Encode = emqx_trace_handler:payload_encode(),

60
apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl
View File

@ -16,6 +16,7 @@
-module(emqx_trace_json_formatter).
-include("emqx_mqtt.hrl").
-include("emqx_trace.hrl").
-export([format/2]).
@ -30,15 +31,16 @@
LogEvent :: logger:log_event(),
Config :: config().
format(
LogMap,
LogMap0,
#{payload_encode := PEncode}
) ->
LogMap1 = emqx_trace_formatter:evaluate_lazy_values(LogMap0),
%% We just make some basic transformations on the input LogMap and then do
%% an external call to create the JSON text
Time = emqx_utils_calendar:now_to_rfc3339(microsecond),
LogMap1 = LogMap#{time => Time},
LogMap2 = prepare_log_map(LogMap1, PEncode),
[emqx_logger_jsonfmt:best_effort_json(LogMap2, [force_utf8]), "\n"].
LogMap2 = LogMap1#{time => Time},
LogMap3 = prepare_log_map(LogMap2, PEncode),
[emqx_logger_jsonfmt:best_effort_json(LogMap3, [force_utf8]), "\n"].
%%%-----------------------------------------------------------------
%%% Helper Functions
@ -48,21 +50,26 @@ prepare_log_map(LogMap, PEncode) ->
NewKeyValuePairs = [prepare_key_value(K, V, PEncode) || {K, V} <- maps:to_list(LogMap)],
maps:from_list(NewKeyValuePairs).
prepare_key_value(K, {Formatter, V}, PEncode) when is_function(Formatter, 1) ->
%% A cusom formatter is provided with the value
try
NewV = Formatter(V),
prepare_key_value(K, NewV, PEncode)
catch
_:_ ->
{K, V}
end;
prepare_key_value(K, {ok, Status, Headers, Body}, PEncode) when
is_integer(Status), is_list(Headers), is_binary(Body)
prepare_key_value(host, {I1, I2, I3, I4} = IP, _PEncode) when
is_integer(I1),
is_integer(I2),
is_integer(I3),
is_integer(I4)
->
%% This is unlikely anything else then info about a HTTP request so we make
%% it more structured
prepare_key_value(K, #{status => Status, headers => Headers, body => Body}, PEncode);
%% We assume this is an IP address
{host, unicode:characters_to_binary(inet:ntoa(IP))};
prepare_key_value(host, {I1, I2, I3, I4, I5, I6, I7, I8} = IP, _PEncode) when
is_integer(I1),
is_integer(I2),
is_integer(I3),
is_integer(I4),
is_integer(I5),
is_integer(I6),
is_integer(I7),
is_integer(I8)
->
%% We assume this is an IP address
{host, unicode:characters_to_binary(inet:ntoa(IP))};
prepare_key_value(payload = K, V, PEncode) ->
NewV =
try
@ -81,6 +88,21 @@ prepare_key_value(packet = K, V, PEncode) ->
V
end,
{K, NewV};
prepare_key_value(K, {recoverable_error, Msg} = OrgV, PEncode) ->
try
prepare_key_value(
K,
#{
error_type => recoverable_error,
msg => Msg,
additional_info => <<"The operation may be retried.">>
},
PEncode
)
catch
_:_ ->
{K, OrgV}
end;
prepare_key_value(rule_ids = K, V, _PEncode) ->
NewV =
try
@ -137,6 +159,8 @@ format_map_set_to_list(Map) ->
],
lists:sort(Items).
format_action_info(#{mod := _Mod, func := _Func} = FuncCall) ->
FuncCall;
format_action_info(V) ->
[<<"action">>, Type, Name | _] = binary:split(V, <<":">>, [global]),
#{

1
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -1061,6 +1061,7 @@ clientinfo(InitProps) ->
clientid => <<"clientid">>,
username => <<"username">>,
is_superuser => false,
auth_expire_at => undefined,
is_bridge => false,
mountpoint => undefined
},

3
apps/emqx/test/emqx_config_SUITE.erl
View File

@ -475,6 +475,7 @@ zone_global_defaults() ->
message_retention_period => 86400000,
renew_streams_interval => 5000,
session_gc_batch_size => 100,
session_gc_interval => 600000
session_gc_interval => 600000,
subscription_count_refresh_interval => 5000
}
}.

54
apps/emqx/test/emqx_connection_expire_SUITE.erl Normal file
View File

@ -0,0 +1,54 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_connection_expire_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).
%%--------------------------------------------------------------------
%% CT callbacks
%%--------------------------------------------------------------------
init_per_suite(Config) ->
Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}),
[{apps, Apps} | Config].
end_per_suite(Config) ->
emqx_cth_suite:stop(proplists:get_value(apps, Config)).
t_disonnect_by_auth_info(_) ->
_ = process_flag(trap_exit, true),
_ = meck:new(emqx_access_control, [passthrough, no_history]),
_ = meck:expect(emqx_access_control, authenticate, fun(_) ->
{ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
end),
{ok, C} = emqtt:start_link([{proto_ver, v5}]),
{ok, _} = emqtt:connect(C),
receive
{disconnected, ?RC_NOT_AUTHORIZED, #{}} -> ok
after 5000 ->
ct:fail("Client should be disconnected by timeout")
end.

13
apps/emqx/test/emqx_tls_lib_tests.erl
View File

@ -240,7 +240,7 @@ to_client_opts_test() ->
Versions13Only = ['tlsv1.3'],
Options = #{
enable => true,
verify => "Verify",
verify => verify_none,
server_name_indication => "SNI",
ciphers => "Ciphers",
depth => "depth",
@ -249,9 +249,16 @@ to_client_opts_test() ->
secure_renegotiate => "secure_renegotiate",
reuse_sessions => "reuse_sessions"
},
Expected1 = lists:usort(maps:keys(Options) -- [enable]),
Expected0 = lists:usort(maps:keys(Options) -- [enable]),
Expected1 = lists:sort(Expected0 ++ [customize_hostname_check]),
?assertEqual(
Expected1, lists:usort(proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options)))
Expected0, lists:usort(proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options)))
),
?assertEqual(
Expected1,
lists:usort(
proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options#{verify => verify_peer}))
)
),
Expected2 =
lists:usort(

2
apps/emqx_auth/src/emqx_authn/emqx_authn_chains.erl
View File

@ -142,6 +142,8 @@ end).
-type state() :: #{atom() => term()}.
-type extra() :: #{
is_superuser := boolean(),
%% millisecond timestamp
expire_at => pos_integer(),
atom() => term()
}.
-type user_info() :: #{

54
apps/emqx_auth_jwt/src/emqx_authn_jwt.erl
View File

@ -78,6 +78,7 @@ authenticate(
Credential,
#{
verify_claims := VerifyClaims0,
disconnect_after_expire := DisconnectAfterExpire,
jwk := JWK,
acl_claim_name := AclClaimName,
from := From
@ -86,11 +87,12 @@ authenticate(
JWT = maps:get(From, Credential),
JWKs = [JWK],
VerifyClaims = render_expected(VerifyClaims0, Credential),
verify(JWT, JWKs, VerifyClaims, AclClaimName);
verify(JWT, JWKs, VerifyClaims, AclClaimName, DisconnectAfterExpire);
authenticate(
Credential,
#{
verify_claims := VerifyClaims0,
disconnect_after_expire := DisconnectAfterExpire,
jwk_resource := ResourceId,
acl_claim_name := AclClaimName,
from := From
@ -106,7 +108,7 @@ authenticate(
{ok, JWKs} ->
JWT = maps:get(From, Credential),
VerifyClaims = render_expected(VerifyClaims0, Credential),
verify(JWT, JWKs, VerifyClaims, AclClaimName)
verify(JWT, JWKs, VerifyClaims, AclClaimName, DisconnectAfterExpire)
end.
destroy(#{jwk_resource := ResourceId}) ->
@ -125,6 +127,7 @@ create2(#{
secret := Secret0,
secret_base64_encoded := Base64Encoded,
verify_claims := VerifyClaims,
disconnect_after_expire := DisconnectAfterExpire,
acl_claim_name := AclClaimName,
from := From
}) ->
@ -136,6 +139,7 @@ create2(#{
{ok, #{
jwk => JWK,
verify_claims => VerifyClaims,
disconnect_after_expire => DisconnectAfterExpire,
acl_claim_name => AclClaimName,
from => From
}}
@ -145,6 +149,7 @@ create2(#{
algorithm := 'public-key',
public_key := PublicKey,
verify_claims := VerifyClaims,
disconnect_after_expire := DisconnectAfterExpire,
acl_claim_name := AclClaimName,
from := From
}) ->
@ -152,6 +157,7 @@ create2(#{
{ok, #{
jwk => JWK,
verify_claims => VerifyClaims,
disconnect_after_expire => DisconnectAfterExpire,
acl_claim_name => AclClaimName,
from => From
}};
@ -159,6 +165,7 @@ create2(
#{
use_jwks := true,
verify_claims := VerifyClaims,
disconnect_after_expire := DisconnectAfterExpire,
acl_claim_name := AclClaimName,
from := From
} = Config
@ -173,6 +180,7 @@ create2(
{ok, #{
jwk_resource => ResourceId,
verify_claims => VerifyClaims,
disconnect_after_expire => DisconnectAfterExpire,
acl_claim_name => AclClaimName,
from => From
}}.
@ -211,23 +219,12 @@ render_expected([{Name, ExpectedTemplate} | More], Variables) ->
Expected = emqx_authn_utils:render_str(ExpectedTemplate, Variables),
[{Name, Expected} | render_expected(More, Variables)].
verify(undefined, _, _, _) ->
verify(undefined, _, _, _, _) ->
ignore;
verify(JWT, JWKs, VerifyClaims, AclClaimName) ->
verify(JWT, JWKs, VerifyClaims, AclClaimName, DisconnectAfterExpire) ->
case do_verify(JWT, JWKs, VerifyClaims) of
{ok, Extra} ->
IsSuperuser = emqx_authn_utils:is_superuser(Extra),
Attrs = emqx_authn_utils:client_attrs(Extra),
try
ACL = acl(Extra, AclClaimName),
Result = maps:merge(IsSuperuser, maps:merge(ACL, Attrs)),
{ok, Result}
catch
throw:{bad_acl_rule, Reason} ->
%% it's a invalid token, so ok to log
?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{jwt => JWT}),
{error, bad_username_or_password}
end;
extra_to_auth_data(Extra, JWT, AclClaimName, DisconnectAfterExpire);
{error, {missing_claim, Claim}} ->
%% it's a invalid token, so it's ok to log
?TRACE_AUTHN_PROVIDER("missing_jwt_claim", #{jwt => JWT, claim => Claim}),
@ -242,6 +239,28 @@ verify(JWT, JWKs, VerifyClaims, AclClaimName) ->
{error, bad_username_or_password}
end.
extra_to_auth_data(Extra, JWT, AclClaimName, DisconnectAfterExpire) ->
IsSuperuser = emqx_authn_utils:is_superuser(Extra),
Attrs = emqx_authn_utils:client_attrs(Extra),
ExpireAt = expire_at(DisconnectAfterExpire, Extra),
try
ACL = acl(Extra, AclClaimName),
Result = merge_maps([ExpireAt, IsSuperuser, ACL, Attrs]),
{ok, Result}
catch
throw:{bad_acl_rule, Reason} ->
%% it's a invalid token, so ok to log
?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{jwt => JWT}),
{error, bad_username_or_password}
end.
expire_at(false, _Extra) ->
#{};
expire_at(true, #{<<"exp">> := ExpireTime}) ->
#{expire_at => erlang:convert_time_unit(ExpireTime, second, millisecond)};
expire_at(true, #{}) ->
#{}.
acl(Claims, AclClaimName) ->
case Claims of
#{AclClaimName := Rules} ->
@ -379,3 +398,6 @@ parse_rule(Rule) ->
{error, Reason} ->
throw({bad_acl_rule, Reason})
end.
merge_maps([]) -> #{};
merge_maps([Map | Maps]) -> maps:merge(Map, merge_maps(Maps)).

6
apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl
View File

@ -127,6 +127,7 @@ common_fields() ->
desc => ?DESC(acl_claim_name)
}},
{verify_claims, fun verify_claims/1},
{disconnect_after_expire, fun disconnect_after_expire/1},
{from, fun from/1}
] ++ emqx_authn_schema:common_fields().
@ -188,6 +189,11 @@ verify_claims(required) ->
verify_claims(_) ->
undefined.
disconnect_after_expire(type) -> boolean();
disconnect_after_expire(desc) -> ?DESC(?FUNCTION_NAME);
disconnect_after_expire(default) -> true;
disconnect_after_expire(_) -> undefined.
do_check_verify_claims([]) ->
true;
%% _Expected can't be invalid since tuples may come only from converter

20
apps/emqx_auth_jwt/test/emqx_authn_jwt_SUITE.erl
View File

@ -55,7 +55,8 @@ t_hmac_based(_) ->
algorithm => 'hmac-based',
secret => Secret,
secret_base64_encoded => false,
verify_claims => [{<<"username">>, <<"${username}">>}]
verify_claims => [{<<"username">>, <<"${username}">>}],
disconnect_after_expire => false
},
{ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
@ -179,7 +180,8 @@ t_public_key(_) ->
use_jwks => false,
algorithm => 'public-key',
public_key => PublicKey,
verify_claims => []
verify_claims => [],
disconnect_after_expire => false
},
{ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
@ -207,7 +209,8 @@ t_jwt_in_username(_) ->
algorithm => 'hmac-based',
secret => Secret,
secret_base64_encoded => false,
verify_claims => []
verify_claims => [],
disconnect_after_expire => false
},
{ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
@ -229,7 +232,8 @@ t_complex_template(_) ->
algorithm => 'hmac-based',
secret => Secret,
secret_base64_encoded => false,
verify_claims => [{<<"id">>, <<"${username}-${clientid}">>}]
verify_claims => [{<<"id">>, <<"${username}-${clientid}">>}],
disconnect_after_expire => false
},
{ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
@ -269,7 +273,7 @@ t_jwks_renewal(_Config) ->
algorithm => 'public-key',
ssl => #{enable => false},
verify_claims => [],
disconnect_after_expire => false,
use_jwks => true,
endpoint => "https://127.0.0.1:" ++ integer_to_list(?JWKS_PORT + 1) ++ ?JWKS_PATH,
refresh_interval => 1000,
@ -366,7 +370,8 @@ t_verify_claims(_) ->
algorithm => 'hmac-based',
secret => Secret,
secret_base64_encoded => false,
verify_claims => [{<<"foo">>, <<"bar">>}]
verify_claims => [{<<"foo">>, <<"bar">>}],
disconnect_after_expire => false
},
{ok, State0} = emqx_authn_jwt:create(?AUTHN_ID, Config0),
@ -456,7 +461,8 @@ t_verify_claim_clientid(_) ->
algorithm => 'hmac-based',
secret => Secret,
secret_base64_encoded => false,
verify_claims => [{<<"cl">>, <<"${clientid}">>}]
verify_claims => [{<<"cl">>, <<"${clientid}">>}],
disconnect_after_expire => false
},
{ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),

96
apps/emqx_auth_jwt/test/emqx_authn_jwt_expire_SUITE.erl Normal file
View File

@ -0,0 +1,96 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authn_jwt_expire_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-define(PATH, [authentication]).
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_testcase(_, Config) ->
_ = emqx_authn_test_lib:delete_authenticators(?PATH, ?GLOBAL),
Config.
end_per_testcase(_, _Config) ->
_ = emqx_authn_test_lib:delete_authenticators(?PATH, ?GLOBAL),
ok.
init_per_suite(Config) ->
Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_auth, emqx_auth_jwt], #{
work_dir => ?config(priv_dir, Config)
}),
[{apps, Apps} | Config].
end_per_suite(Config) ->
emqx_authn_test_lib:delete_authenticators(?PATH, ?GLOBAL),
ok = emqx_cth_suite:stop(?config(apps, Config)),
ok.
%%--------------------------------------------------------------------
%% CT cases
%%--------------------------------------------------------------------
t_jwt_expire(_Config) ->
_ = process_flag(trap_exit, true),
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, auth_config()}
),
{ok, [#{provider := emqx_authn_jwt}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
Expire = erlang:system_time(second) + 3,
Payload = #{
<<"username">> => <<"myuser">>,
<<"exp">> => Expire
},
JWS = emqx_authn_jwt_SUITE:generate_jws('hmac-based', Payload, <<"secret">>),
{ok, C} = emqtt:start_link([{username, <<"myuser">>}, {password, JWS}, {proto_ver, v5}]),
{ok, _} = emqtt:connect(C),
receive
{disconnected, ?RC_NOT_AUTHORIZED, #{}} ->
?assert(erlang:system_time(second) >= Expire)
after 5000 ->
ct:fail("Client should be disconnected by timeout")
end.
%%--------------------------------------------------------------------
%% Helper functions
%%--------------------------------------------------------------------
auth_config() ->
#{
<<"use_jwks">> => false,
<<"algorithm">> => <<"hmac-based">>,
<<"acl_claim_name">> => <<"acl">>,
<<"secret">> => <<"secret">>,
<<"mechanism">> => <<"jwt">>,
<<"verify_claims">> => #{<<"username">> => <<"${username}">>}
%% Should be enabled by default
%% <<"disconnect_after_expire">> => true
}.

5
apps/emqx_auth_jwt/test/emqx_authz_jwt_SUITE.erl
View File

@ -455,11 +455,12 @@ t_invalid_rule(_Config) ->
authn_config() ->
#{
<<"mechanism">> => <<"jwt">>,
<<"use_jwks">> => <<"false">>,
<<"use_jwks">> => false,
<<"algorithm">> => <<"hmac-based">>,
<<"secret">> => ?SECRET,
<<"secret_base64_encoded">> => <<"false">>,
<<"secret_base64_encoded">> => false,
<<"acl_claim_name">> => <<"acl">>,
<<"disconnect_after_expire">> => false,
<<"verify_claims">> => #{
<<"username">> => ?PH_USERNAME
}

5
apps/emqx_bridge/test/emqx_bridge_testlib.erl
View File

@ -252,11 +252,14 @@ create_rule_and_action_http(BridgeType, RuleTopic, Config) ->
create_rule_and_action_http(BridgeType, RuleTopic, Config, Opts) ->
BridgeName = ?config(bridge_name, Config),
BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
create_rule_and_action(BridgeId, RuleTopic, Opts).
create_rule_and_action(Action, RuleTopic, Opts) ->
SQL = maps:get(sql, Opts, <<"SELECT * FROM \"", RuleTopic/binary, "\"">>),
Params = #{
enable => true,
sql => SQL,
actions => [BridgeId]
actions => [Action]
},
Path = emqx_mgmt_api_test_util:api_path(["rules"]),
AuthHeader = emqx_mgmt_api_test_util:auth_header_(),

8
apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl
View File

@ -29,7 +29,8 @@
on_query_async/4,
on_batch_query/3,
on_batch_query_async/4,
on_get_status/2
on_get_status/2,
on_format_query_result/1
]).
%% callbacks of ecpool
@ -459,6 +460,11 @@ handle_result({error, Error}) ->
handle_result(Res) ->
Res.
on_format_query_result({ok, Result}) ->
#{result => ok, info => Result};
on_format_query_result(Result) ->
Result.
%%--------------------------------------------------------------------
%% utils

10
apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
View File

@ -38,7 +38,8 @@
on_get_channels/1,
on_query/3,
on_batch_query/3,
on_get_status/2
on_get_status/2,
on_format_query_result/1
]).
%% callbacks for ecpool
@ -519,6 +520,13 @@ transform_and_log_clickhouse_result(ClickhouseErrorResult, ResourceID, SQL) ->
to_error_tuple(ClickhouseErrorResult)
end.
on_format_query_result(ok) ->
#{result => ok, message => <<"">>};
on_format_query_result({ok, Message}) ->
#{result => ok, message => Message};
on_format_query_result(Result) ->
Result.
to_recoverable_error({error, Reason}) ->
{error, {recoverable_error, Reason}};
to_recoverable_error(Error) ->

8
apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl
View File

@ -26,7 +26,8 @@
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
-export([
@ -184,6 +185,11 @@ on_batch_query(InstanceId, [{_ChannelId, _} | _] = Query, State) ->
on_batch_query(_InstanceId, Query, _State) ->
{error, {unrecoverable_error, {invalid_request, Query}}}.
on_format_query_result({ok, Result}) ->
#{result => ok, info => Result};
on_format_query_result(Result) ->
Result.
health_check_timeout() ->
2500.

7
apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl
View File

@ -27,6 +27,8 @@
-export([execute/2]).
-endif.
-include_lib("emqx/include/emqx_trace.hrl").
%%%===================================================================
%%% API
%%%===================================================================
@ -107,7 +109,10 @@ do_query(Table, Query0, Templates, TraceRenderedCTX) ->
Query = apply_template(Query0, Templates),
emqx_trace:rendered_action_template_with_ctx(TraceRenderedCTX, #{
table => Table,
query => {fun trace_format_query/1, Query}
query => #emqx_trace_format_func_data{
function = fun trace_format_query/1,
data = Query
}
}),
execute(Query, Table)
catch

6
apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl
View File

@ -23,7 +23,8 @@
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
-export([
@ -286,6 +287,9 @@ on_query_async(
InstanceId, {ChannelId, Msg}, ReplyFunAndArgs, State
).
on_format_query_result(Result) ->
emqx_bridge_http_connector:on_format_query_result(Result).
on_add_channel(
InstanceId,
#{channels := Channels} = State0,

8
apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl
View File

@ -53,7 +53,8 @@
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
-export([reply_delegator/2]).
@ -489,6 +490,11 @@ handle_result({error, Reason} = Result, _Request, QueryMode, ResourceId) ->
handle_result({ok, _} = Result, _Request, _QueryMode, _ResourceId) ->
Result.
on_format_query_result({ok, Info}) ->
#{result => ok, info => Info};
on_format_query_result(Result) ->
Result.
reply_delegator(ReplyFunAndArgs, Response) ->
case Response of
{error, Reason} when

8
apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl
View File

@ -27,7 +27,8 @@
on_batch_query/3,
on_query_async/4,
on_batch_query_async/4,
on_get_status/2
on_get_status/2,
on_format_query_result/1
]).
-export([reply_callback/2]).
@ -453,6 +454,11 @@ do_query(InstId, Channel, Client, Points) ->
end
end.
on_format_query_result({ok, {affected_rows, Rows}}) ->
#{result => ok, affected_rows => Rows};
on_format_query_result(Result) ->
Result.
do_async_query(InstId, Channel, Client, Points, ReplyFunAndArgs) ->
?SLOG(info, #{
msg => "greptimedb_write_point_async",

89
apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl
View File

@ -20,6 +20,7 @@
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/emqx_trace.hrl").
-behaviour(emqx_resource).
@ -35,7 +36,8 @@
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
-export([reply_delegator/3]).
@ -231,6 +233,7 @@ on_start(
host => Host,
port => Port,
connect_timeout => ConnectTimeout,
scheme => Scheme,
request => preprocess_request(maps:get(request, Config, undefined))
},
case start_pool(InstId, PoolOpts) of
@ -358,7 +361,7 @@ on_query(InstId, {Method, Request, Timeout}, State) ->
on_query(
InstId,
{ActionId, KeyOrNum, Method, Request, Timeout, Retry},
#{host := Host} = State
#{host := Host, port := Port, scheme := Scheme} = State
) ->
?TRACE(
"QUERY",
@ -372,7 +375,7 @@ on_query(
}
),
NRequest = formalize_request(Method, Request),
trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout),
trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout),
Worker = resolve_pool_worker(State, KeyOrNum),
Result0 = ehttpc:request(
Worker,
@ -468,7 +471,7 @@ on_query_async(
InstId,
{ActionId, KeyOrNum, Method, Request, Timeout},
ReplyFunAndArgs,
#{host := Host} = State
#{host := Host, port := Port, scheme := Scheme} = State
) ->
Worker = resolve_pool_worker(State, KeyOrNum),
?TRACE(
@ -482,7 +485,7 @@ on_query_async(
}
),
NRequest = formalize_request(Method, Request),
trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout),
trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout),
MaxAttempts = maps:get(max_attempts, State, 3),
Context = #{
attempt => 1,
@ -491,7 +494,8 @@ on_query_async(
key_or_num => KeyOrNum,
method => Method,
request => NRequest,
timeout => Timeout
timeout => Timeout,
trace_metadata => logger:get_process_metadata()
},
ok = ehttpc:request_async(
Worker,
@ -502,17 +506,25 @@ on_query_async(
),
{ok, Worker}.
trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) ->
trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout) ->
case NRequest of
{Path, Headers} ->
emqx_trace:rendered_action_template(
ActionId,
#{
host => Host,
port => Port,
path => Path,
method => Method,
headers => {fun emqx_utils_redact:redact_headers/1, Headers},
timeout => Timeout
headers => #emqx_trace_format_func_data{
function = fun emqx_utils_redact:redact_headers/1,
data = Headers
},
timeout => Timeout,
url => #emqx_trace_format_func_data{
function = fun render_url/1,
data = {Scheme, Host, Port, Path}
}
}
);
{Path, Headers, Body} ->
@ -520,15 +532,42 @@ trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) ->
ActionId,
#{
host => Host,
port => Port,
path => Path,
method => Method,
headers => {fun emqx_utils_redact:redact_headers/1, Headers},
headers => #emqx_trace_format_func_data{
function = fun emqx_utils_redact:redact_headers/1,
data = Headers
},
timeout => Timeout,
body => {fun log_format_body/1, Body}
body => #emqx_trace_format_func_data{
function = fun log_format_body/1,
data = Body
},
url => #emqx_trace_format_func_data{
function = fun render_url/1,
data = {Scheme, Host, Port, Path}
}
}
)
end.
render_url({Scheme, Host, Port, Path}) ->
SchemeStr =
case Scheme of
http ->
<<"http://">>;
https ->
<<"https://">>
end,
unicode:characters_to_binary([
SchemeStr,
Host,
<<":">>,
erlang:integer_to_binary(Port),
Path
]).
log_format_body(Body) ->
unicode:characters_to_binary(Body).
@ -604,6 +643,26 @@ on_get_channel_status(
%% XXX: Reuse the connector status
on_get_status(InstId, State).
on_format_query_result({ok, Status, Headers, Body}) ->
#{
result => ok,
response => #{
status => Status,
headers => Headers,
body => Body
}
};
on_format_query_result({ok, Status, Headers}) ->
#{
result => ok,
response => #{
status => Status,
headers => Headers
}
};
on_format_query_result(Result) ->
Result.
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
@ -809,9 +868,15 @@ to_bin(Str) when is_list(Str) ->
to_bin(Atom) when is_atom(Atom) ->
atom_to_binary(Atom, utf8).
reply_delegator(Context, ReplyFunAndArgs, Result0) ->
reply_delegator(
#{trace_metadata := TraceMetadata} = Context,
ReplyFunAndArgs,
Result0
) ->
spawn(fun() ->
logger:set_process_metadata(TraceMetadata),
Result = transform_result(Result0),
logger:unset_process_metadata(),
maybe_retry(Result, Context, ReplyFunAndArgs)
end).

6
apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
View File

@ -27,7 +27,8 @@
on_batch_query/3,
on_query_async/4,
on_batch_query_async/4,
on_get_status/2
on_get_status/2,
on_format_query_result/1
]).
-export([reply_callback/2]).
@ -209,6 +210,9 @@ on_batch_query_async(
{error, {unrecoverable_error, Reason}}
end.
on_format_query_result(Result) ->
emqx_bridge_http_connector:on_format_query_result(Result).
on_get_status(_InstId, #{client := Client}) ->
case influxdb:is_alive(Client) andalso ok =:= influxdb:check_auth(Client) of
true ->

6
apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl
View File

@ -26,7 +26,8 @@
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
-export([
@ -388,6 +389,9 @@ on_batch_query(
Error
end.
on_format_query_result(Result) ->
emqx_bridge_http_connector:on_format_query_result(Result).
on_add_channel(
InstanceId,
#{iotdb_version := Version, channels := Channels} = OldState0,

8
apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl
View File

@ -39,7 +39,8 @@
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
-export([
@ -318,6 +319,11 @@ handle_result({error, Reason} = Error, Requests, InstanceId) ->
}),
Error.
on_format_query_result({ok, Result}) ->
#{result => ok, info => Result};
on_format_query_result(Result) ->
Result.
parse_template(Config) ->
#{payload_template := PayloadTemplate, partition_key := PartitionKeyTemplate} = Config,
Templates = #{send_message => PayloadTemplate, partition_key => PartitionKeyTemplate},

8
apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl
View File

@ -18,7 +18,8 @@
on_get_status/2,
on_query/3,
on_start/2,
on_stop/2
on_stop/2,
on_format_query_result/1
]).
%%========================================================================================
@ -85,6 +86,11 @@ on_query(InstanceId, {Channel, Message0}, #{channels := Channels, connector_stat
on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) ->
emqx_mongodb:on_query(InstanceId, Request, ConnectorState).
on_format_query_result({{Result, Info}, Documents}) ->
#{result => Result, info => Info, documents => Documents};
on_format_query_result(Result) ->
Result.
on_remove_channel(_InstanceId, #{channels := Channels} = State, ChannelId) ->
NewState = State#{channels => maps:remove(ChannelId, Channels)},
{ok, NewState}.

8
apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl
View File

@ -27,7 +27,8 @@
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
-export([connector_examples/1]).
@ -175,6 +176,11 @@ on_batch_query(
Error
end.
on_format_query_result({ok, StatusCode, BodyMap}) ->
#{result => ok, status_code => StatusCode, body => BodyMap};
on_format_query_result(Result) ->
Result.
on_get_status(_InstanceId, #{server := Server}) ->
Result =
case opentsdb_connectivity(Server) of

8
apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl
View File

@ -20,7 +20,8 @@
on_get_status/2,
on_get_channel_status/3,
on_query/3,
on_query_async/4
on_query_async/4,
on_format_query_result/1
]).
-type pulsar_client_id() :: atom().
@ -234,6 +235,11 @@ on_query_async2(ChannelId, Producers, Message, MessageTmpl, AsyncReplyFn) ->
}),
pulsar:send(Producers, [PulsarMessage], #{callback_fn => AsyncReplyFn}).
on_format_query_result({ok, Info}) ->
#{result => ok, info => Info};
on_format_query_result(Result) ->
Result.
%%-------------------------------------------------------------------------------------
%% Internal fns
%%-------------------------------------------------------------------------------------

8
apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl
View File

@ -20,7 +20,8 @@
on_query/3,
on_batch_query/3,
on_get_status/2,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
%% -------------------------------------------------------------------------------------------------
@ -161,6 +162,11 @@ on_batch_query(
Error
end.
on_format_query_result({ok, Msg}) ->
#{result => ok, message => Msg};
on_format_query_result(Res) ->
Res.
%% -------------------------------------------------------------------------------------------------
%% private helpers
%% -------------------------------------------------------------------------------------------------

3
apps/emqx_bridge_s3/rebar.config
View File

@ -2,5 +2,6 @@
{erl_opts, [debug_info]}.
{deps, [
{emqx_resource, {path, "../../apps/emqx_resource"}}
{emqx_resource, {path, "../../apps/emqx_resource"}},
{emqx_connector_aggregator, {path, "../../apps/emqx_connector_aggregator"}}
]}.

2
apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src
View File

@ -7,6 +7,7 @@
stdlib,
erlcloud,
emqx_resource,
emqx_connector_aggregator,
emqx_s3
]},
{env, [
@ -18,7 +19,6 @@
emqx_bridge_s3_connector_info
]}
]},
{mod, {emqx_bridge_s3_app, []}},
{modules, []},
{links, []}
]}.

212
apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl
View File

@ -1,212 +0,0 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
%% This module takes aggregated records from a buffer and delivers them to S3,
%% wrapped in a configurable container (though currently there's only CSV).
-module(emqx_bridge_s3_aggreg_delivery).
-include_lib("snabbkaffe/include/trace.hrl").
-include("emqx_bridge_s3_aggregator.hrl").
-export([start_link/3]).
%% Internal exports
-export([
init/4,
loop/3
]).
-behaviour(emqx_template).
-export([lookup/2]).
%% Sys
-export([
system_continue/3,
system_terminate/4,
format_status/2
]).
-record(delivery, {
name :: _Name,
container :: emqx_bridge_s3_aggreg_csv:container(),
reader :: emqx_bridge_s3_aggreg_buffer:reader(),
upload :: emqx_s3_upload:t(),
empty :: boolean()
}).
-type state() :: #delivery{}.
%%
start_link(Name, Buffer, Opts) ->
proc_lib:start_link(?MODULE, init, [self(), Name, Buffer, Opts]).
%%
-spec init(pid(), _Name, buffer(), _Opts :: map()) -> no_return().
init(Parent, Name, Buffer, Opts) ->
?tp(s3_aggreg_delivery_started, #{action => Name, buffer => Buffer}),
Reader = open_buffer(Buffer),
Delivery = init_delivery(Name, Reader, Buffer, Opts#{action => Name}),
_ = erlang:process_flag(trap_exit, true),
ok = proc_lib:init_ack({ok, self()}),
loop(Delivery, Parent, []).
init_delivery(Name, Reader, Buffer, Opts = #{container := ContainerOpts}) ->
#delivery{
name = Name,
container = mk_container(ContainerOpts),
reader = Reader,
upload = mk_upload(Buffer, Opts),
empty = true
}.
open_buffer(#buffer{filename = Filename}) ->
case file:open(Filename, [read, binary, raw]) of
{ok, FD} ->
{_Meta, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(FD),
Reader;
{error, Reason} ->
error({buffer_open_failed, Reason})
end.
mk_container(#{type := csv, column_order := OrderOpt}) ->
%% TODO: Deduplicate?
ColumnOrder = lists:map(fun emqx_utils_conv:bin/1, OrderOpt),
emqx_bridge_s3_aggreg_csv:new(#{column_order => ColumnOrder}).
mk_upload(
Buffer,
Opts = #{
bucket := Bucket,
upload_options := UploadOpts,
client_config := Config,
uploader_config := UploaderConfig
}
) ->
Client = emqx_s3_client:create(Bucket, Config),
Key = mk_object_key(Buffer, Opts),
emqx_s3_upload:new(Client, Key, UploadOpts, UploaderConfig).
mk_object_key(Buffer, #{action := Name, key := Template}) ->
emqx_template:render_strict(Template, {?MODULE, {Name, Buffer}}).
%%
-spec loop(state(), pid(), [sys:debug_option()]) -> no_return().
loop(Delivery, Parent, Debug) ->
%% NOTE: This function is mocked in tests.
receive
Msg -> handle_msg(Msg, Delivery, Parent, Debug)
after 0 ->
process_delivery(Delivery, Parent, Debug)
end.
process_delivery(Delivery0 = #delivery{reader = Reader0}, Parent, Debug) ->
case emqx_bridge_s3_aggreg_buffer:read(Reader0) of
{Records = [#{} | _], Reader} ->
Delivery1 = Delivery0#delivery{reader = Reader},
Delivery2 = process_append_records(Records, Delivery1),
Delivery = process_write(Delivery2),
loop(Delivery, Parent, Debug);
{[], Reader} ->
Delivery = Delivery0#delivery{reader = Reader},
loop(Delivery, Parent, Debug);
eof ->
process_complete(Delivery0);
{Unexpected, _Reader} ->
exit({buffer_unexpected_record, Unexpected})
end.
process_append_records(Records, Delivery = #delivery{container = Container0, upload = Upload0}) ->
{Writes, Container} = emqx_bridge_s3_aggreg_csv:fill(Records, Container0),
{ok, Upload} = emqx_s3_upload:append(Writes, Upload0),
Delivery#delivery{
container = Container,
upload = Upload,
empty = false
}.
process_write(Delivery = #delivery{upload = Upload0}) ->
case emqx_s3_upload:write(Upload0) of
{ok, Upload} ->
Delivery#delivery{upload = Upload};
{cont, Upload} ->
process_write(Delivery#delivery{upload = Upload});
{error, Reason} ->
_ = emqx_s3_upload:abort(Upload0),
exit({upload_failed, Reason})
end.
process_complete(#delivery{name = Name, empty = true}) ->
?tp(s3_aggreg_delivery_completed, #{action => Name, upload => empty}),
exit({shutdown, {skipped, empty}});
process_complete(#delivery{name = Name, container = Container, upload = Upload0}) ->
Trailer = emqx_bridge_s3_aggreg_csv:close(Container),
{ok, Upload} = emqx_s3_upload:append(Trailer, Upload0),
case emqx_s3_upload:complete(Upload) of
{ok, Completed} ->
?tp(s3_aggreg_delivery_completed, #{action => Name, upload => Completed}),
ok;
{error, Reason} ->
_ = emqx_s3_upload:abort(Upload),
exit({upload_failed, Reason})
end.
%%
handle_msg({system, From, Msg}, Delivery, Parent, Debug) ->
sys:handle_system_msg(Msg, From, Parent, ?MODULE, Debug, Delivery);
handle_msg({'EXIT', Parent, Reason}, Delivery, Parent, Debug) ->
system_terminate(Reason, Parent, Debug, Delivery);
handle_msg(_Msg, Delivery, Parent, Debug) ->
loop(Parent, Debug, Delivery).
-spec system_continue(pid(), [sys:debug_option()], state()) -> no_return().
system_continue(Parent, Debug, Delivery) ->
loop(Delivery, Parent, Debug).
-spec system_terminate(_Reason, pid(), [sys:debug_option()], state()) -> _.
system_terminate(_Reason, _Parent, _Debug, #delivery{upload = Upload}) ->
emqx_s3_upload:abort(Upload).
-spec format_status(normal, Args :: [term()]) -> _StateFormatted.
format_status(_Normal, [_PDict, _SysState, _Parent, _Debug, Delivery]) ->
Delivery#delivery{
upload = emqx_s3_upload:format(Delivery#delivery.upload)
}.
%%
-spec lookup(emqx_template:accessor(), {_Name, buffer()}) ->
{ok, integer() | string()} | {error, undefined}.
lookup([<<"action">>], {Name, _Buffer}) ->
{ok, mk_fs_safe_string(Name)};
lookup(Accessor, {_Name, Buffer = #buffer{}}) ->
lookup_buffer_var(Accessor, Buffer);
lookup(_Accessor, _Context) ->
{error, undefined}.
lookup_buffer_var([<<"datetime">>, Format], #buffer{since = Since}) ->
{ok, format_timestamp(Since, Format)};
lookup_buffer_var([<<"datetime_until">>, Format], #buffer{until = Until}) ->
{ok, format_timestamp(Until, Format)};
lookup_buffer_var([<<"sequence">>], #buffer{seq = Seq}) ->
{ok, Seq};
lookup_buffer_var([<<"node">>], #buffer{}) ->
{ok, mk_fs_safe_string(atom_to_binary(erlang:node()))};
lookup_buffer_var(_Binding, _Context) ->
{error, undefined}.
format_timestamp(Timestamp, <<"rfc3339utc">>) ->
String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}, {offset, "Z"}]),
mk_fs_safe_string(String);
format_timestamp(Timestamp, <<"rfc3339">>) ->
String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}]),
mk_fs_safe_string(String);
format_timestamp(Timestamp, <<"unix">>) ->
Timestamp.
mk_fs_safe_string(String) ->
unicode:characters_to_binary(string:replace(String, ":", "_", all)).

16
apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl
View File

@ -1,16 +0,0 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_bridge_s3_app).
-behaviour(application).
-export([start/2, stop/1]).
%%
start(_StartType, _StartArgs) ->
emqx_bridge_s3_sup:start_link().
stop(_State) ->
ok.

116
apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl
View File

@ -7,6 +7,8 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/trace.hrl").
-include_lib("emqx_resource/include/emqx_resource.hrl").
-include_lib("emqx/include/emqx_trace.hrl").
-include_lib("emqx_connector_aggregator/include/emqx_connector_aggregator.hrl").
-include("emqx_bridge_s3.hrl").
-behaviour(emqx_resource).
@ -23,6 +25,19 @@
on_get_channel_status/3
]).
-behaviour(emqx_connector_aggreg_delivery).
-export([
init_transfer_state/2,
process_append/2,
process_write/1,
process_complete/1,
process_format_status/1,
process_terminate/1
]).
-behaviour(emqx_template).
-export([lookup/2]).
-type config() :: #{
access_key_id => string(),
secret_access_key => emqx_secret:t(string()),
@ -204,13 +219,14 @@ start_channel(State, #{
key => emqx_bridge_s3_aggreg_upload:mk_key_template(Parameters),
container => Container,
upload_options => emqx_bridge_s3_aggreg_upload:mk_upload_options(Parameters),
callback_module => ?MODULE,
client_config => maps:get(client_config, State),
uploader_config => maps:with([min_part_size, max_part_size], Parameters)
},
_ = emqx_bridge_s3_sup:delete_child({Type, Name}),
{ok, SupPid} = emqx_bridge_s3_sup:start_child(#{
_ = emqx_connector_aggreg_sup:delete_child({Type, Name}),
{ok, SupPid} = emqx_connector_aggreg_sup:start_child(#{
id => {Type, Name},
start => {emqx_bridge_s3_aggreg_upload_sup, start_link, [Name, AggregOpts, DeliveryOpts]},
start => {emqx_connector_aggreg_upload_sup, start_link, [Name, AggregOpts, DeliveryOpts]},
type => supervisor,
restart => permanent
}),
@ -219,7 +235,7 @@ start_channel(State, #{
name => Name,
bucket => Bucket,
supervisor => SupPid,
on_stop => fun() -> emqx_bridge_s3_sup:delete_child({Type, Name}) end
on_stop => fun() -> emqx_connector_aggreg_sup:delete_child({Type, Name}) end
}.
upload_options(Parameters) ->
@ -241,7 +257,7 @@ channel_status(#{type := ?ACTION_UPLOAD}, _State) ->
channel_status(#{type := ?ACTION_AGGREGATED_UPLOAD, name := Name, bucket := Bucket}, State) ->
%% NOTE: This will effectively trigger uploads of buffers yet to be uploaded.
Timestamp = erlang:system_time(second),
ok = emqx_bridge_s3_aggregator:tick(Name, Timestamp),
ok = emqx_connector_aggregator:tick(Name, Timestamp),
ok = check_bucket_accessible(Bucket, State),
ok = check_aggreg_upload_errors(Name),
?status_connected.
@ -263,7 +279,7 @@ check_bucket_accessible(Bucket, #{client_config := Config}) ->
end.
check_aggreg_upload_errors(Name) ->
case emqx_bridge_s3_aggregator:take_error(Name) of
case emqx_connector_aggregator:take_error(Name) of
[Error] ->
%% TODO
%% This approach means that, for example, 3 upload failures will cause
@ -320,7 +336,10 @@ run_simple_upload(
emqx_trace:rendered_action_template(ChannelID, #{
bucket => Bucket,
key => Key,
content => Content
content => #emqx_trace_format_func_data{
function = fun unicode:characters_to_binary/1,
data = Content
}
}),
case emqx_s3_client:put_object(Client, Key, UploadOpts, Content) of
ok ->
@ -336,7 +355,7 @@ run_simple_upload(
run_aggregated_upload(InstId, Records, #{name := Name}) ->
Timestamp = erlang:system_time(second),
case emqx_bridge_s3_aggregator:push_records(Name, Timestamp, Records) of
case emqx_connector_aggregator:push_records(Name, Timestamp, Records) of
ok ->
?tp(s3_bridge_aggreg_push_ok, #{instance_id => InstId, name => Name}),
ok;
@ -372,3 +391,84 @@ render_content(Template, Data) ->
iolist_to_string(IOList) ->
unicode:characters_to_list(IOList).
%% `emqx_connector_aggreg_delivery` APIs
-spec init_transfer_state(buffer_map(), map()) -> emqx_s3_upload:t().
init_transfer_state(BufferMap, Opts) ->
#{
bucket := Bucket,
upload_options := UploadOpts,
client_config := Config,
uploader_config := UploaderConfig
} = Opts,
Client = emqx_s3_client:create(Bucket, Config),
Key = mk_object_key(BufferMap, Opts),
emqx_s3_upload:new(Client, Key, UploadOpts, UploaderConfig).
mk_object_key(BufferMap, #{action := Name, key := Template}) ->
emqx_template:render_strict(Template, {?MODULE, {Name, BufferMap}}).
process_append(Writes, Upload0) ->
{ok, Upload} = emqx_s3_upload:append(Writes, Upload0),
Upload.
process_write(Upload0) ->
case emqx_s3_upload:write(Upload0) of
{ok, Upload} ->
{ok, Upload};
{cont, Upload} ->
process_write(Upload);
{error, Reason} ->
_ = emqx_s3_upload:abort(Upload0),
{error, Reason}
end.
process_complete(Upload) ->
case emqx_s3_upload:complete(Upload) of
{ok, Completed} ->
{ok, Completed};
{error, Reason} ->
_ = emqx_s3_upload:abort(Upload),
exit({upload_failed, Reason})
end.
process_format_status(Upload) ->
emqx_s3_upload:format(Upload).
process_terminate(Upload) ->
emqx_s3_upload:abort(Upload).
%% `emqx_template` APIs
-spec lookup(emqx_template:accessor(), {_Name, buffer_map()}) ->
{ok, integer() | string()} | {error, undefined}.
lookup([<<"action">>], {Name, _Buffer}) ->
{ok, mk_fs_safe_string(Name)};
lookup(Accessor, {_Name, Buffer = #{}}) ->
lookup_buffer_var(Accessor, Buffer);
lookup(_Accessor, _Context) ->
{error, undefined}.
lookup_buffer_var([<<"datetime">>, Format], #{since := Since}) ->
{ok, format_timestamp(Since, Format)};
lookup_buffer_var([<<"datetime_until">>, Format], #{until := Until}) ->
{ok, format_timestamp(Until, Format)};
lookup_buffer_var([<<"sequence">>], #{seq := Seq}) ->
{ok, Seq};
lookup_buffer_var([<<"node">>], #{}) ->
{ok, mk_fs_safe_string(atom_to_binary(erlang:node()))};
lookup_buffer_var(_Binding, _Context) ->
{error, undefined}.
format_timestamp(Timestamp, <<"rfc3339utc">>) ->
String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}, {offset, "Z"}]),
mk_fs_safe_string(String);
format_timestamp(Timestamp, <<"rfc3339">>) ->
String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}]),
mk_fs_safe_string(String);
format_timestamp(Timestamp, <<"unix">>) ->
Timestamp.
mk_fs_safe_string(String) ->
unicode:characters_to_binary(string:replace(String, ":", "_", all)).

24
apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl
View File

@ -170,7 +170,7 @@ t_aggreg_upload(Config) ->
]),
ok = send_messages(BridgeName, MessageEvents),
%% Wait until the delivery is completed.
?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
%% Check the uploaded objects.
_Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
?assertMatch(
@ -217,7 +217,7 @@ t_aggreg_upload_rule(Config) ->
emqx_message:make(?FUNCTION_NAME, T3 = <<"s3/empty">>, P3 = <<>>),
emqx_message:make(?FUNCTION_NAME, <<"not/s3">>, <<"should not be here">>)
]),
?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
%% Check the uploaded objects.
_Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
_CSV = [Header | Rows] = fetch_parse_csv(Bucket, Key),
@ -258,15 +258,15 @@ t_aggreg_upload_restart(Config) ->
{<<"C3">>, T3 = <<"t/42">>, P3 = <<"">>}
]),
ok = send_messages(BridgeName, MessageEvents),
{ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}),
{ok, _} = ?block_until(#{?snk_kind := connector_aggreg_records_written, action := BridgeName}),
%% Restart the bridge.
{ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName),
{ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName),
%% Send some more messages.
ok = send_messages(BridgeName, MessageEvents),
{ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}),
{ok, _} = ?block_until(#{?snk_kind := connector_aggreg_records_written, action := BridgeName}),
%% Wait until the delivery is completed.
{ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
{ok, _} = ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
%% Check there's still only one upload.
_Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
_Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key),
@ -300,18 +300,18 @@ t_aggreg_upload_restart_corrupted(Config) ->
%% Ensure that they span multiple batch queries.
ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages1), 1),
{ok, _} = ?block_until(
#{?snk_kind := s3_aggreg_records_written, action := BridgeName},
#{?snk_kind := connector_aggreg_records_written, action := BridgeName},
infinity,
0
),
%% Find out the buffer file.
{ok, #{filename := Filename}} = ?block_until(
#{?snk_kind := s3_aggreg_buffer_allocated, action := BridgeName}
#{?snk_kind := connector_aggreg_buffer_allocated, action := BridgeName}
),
%% Stop the bridge, corrupt the buffer file, and restart the bridge.
{ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName),
BufferFileSize = filelib:file_size(Filename),
ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, BufferFileSize div 2),
ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, BufferFileSize div 2),
{ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName),
%% Send some more messages.
Messages2 = [
@ -320,7 +320,7 @@ t_aggreg_upload_restart_corrupted(Config) ->
],
ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages2), 0),
%% Wait until the delivery is completed.
{ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
{ok, _} = ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
%% Check that upload contains part of the first batch and all of the second batch.
_Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
CSV = [_Header | Rows] = fetch_parse_csv(Bucket, Key),
@ -362,7 +362,7 @@ t_aggreg_pending_upload_restart(Config) ->
%% Restart the bridge.
{ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName),
%% Wait until the delivery is completed.
{ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
{ok, _} = ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
%% Check that delivery contains all the messages.
_Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
[_Header | Rows] = fetch_parse_csv(Bucket, Key),
@ -392,7 +392,9 @@ t_aggreg_next_rotate(Config) ->
NSent = receive_sender_reports(Senders),
%% Wait for the last delivery to complete.
ok = timer:sleep(round(?CONF_TIME_INTERVAL * 0.5)),
?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}, infinity, 0),
?block_until(
#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}, infinity, 0
),
%% There should be at least 2 time windows of aggregated records.
Uploads = [K || #{key := K} <- emqx_bridge_s3_test_helpers:list_objects(Bucket)],
DTs = [DT || K <- Uploads, [_Action, _Node, DT | _] <- [string:split(K, "/", all)]],

8
apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl
View File

@ -48,11 +48,3 @@ list_pending_uploads(Bucket, Key) ->
{ok, Props} = erlcloud_s3:list_multipart_uploads(Bucket, [{prefix, Key}], [], AwsConfig),
Uploads = proplists:get_value(uploads, Props),
lists:map(fun maps:from_list/1, Uploads).
%% File utilities
truncate_at(Filename, Pos) ->
{ok, FD} = file:open(Filename, [read, write, binary]),
{ok, Pos} = file:position(FD, Pos),
ok = file:truncate(FD),
ok = file:close(FD).

8
apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
View File

@ -39,7 +39,8 @@
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
%% callbacks for ecpool
@ -320,6 +321,11 @@ on_batch_query(ResourceId, BatchRequests, State) ->
),
do_query(ResourceId, BatchRequests, ?SYNC_QUERY_MODE, State).
on_format_query_result({ok, Rows}) ->
#{result => ok, rows => Rows};
on_format_query_result(Result) ->
Result.
on_get_status(_InstanceId, #{pool_name := PoolName} = _State) ->
Health = emqx_resource_pool:health_check_workers(
PoolName,

8
apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
View File

@ -28,7 +28,8 @@
on_add_channel/4,
on_remove_channel/3,
on_get_channels/1,
on_get_channel_status/3
on_get_channel_status/3,
on_format_query_result/1
]).
-export([connector_examples/1]).
@ -215,6 +216,11 @@ on_batch_query(InstanceId, BatchReq, State) ->
?SLOG(error, LogMeta#{msg => "invalid_request"}),
{error, {unrecoverable_error, invalid_request}}.
on_format_query_result({ok, ResultMap}) ->
#{result => ok, info => ResultMap};
on_format_query_result(Result) ->
Result.
on_get_status(_InstanceId, #{pool_name := PoolName} = State) ->
case
emqx_resource_pool:health_check_workers(

94
apps/emqx_connector_aggregator/BSL.txt Normal file
View File

@ -0,0 +1,94 @@
Business Source License 1.1
Licensor: Hangzhou EMQ Technologies Co., Ltd.
Licensed Work: EMQX Enterprise Edition
The Licensed Work is (c) 2024
Hangzhou EMQ Technologies Co., Ltd.
Additional Use Grant: Students and educators are granted right to copy,
modify, and create derivative work for research
or education.
Change Date: 2028-05-06
Change License: Apache License, Version 2.0
For information about alternative licensing arrangements for the Software,
please contact Licensor: https://www.emqx.com/en/contact
Notice
The Business Source License (this document, or the “License”) is not an Open
Source license. However, the Licensed Work will eventually be made available
under an Open Source License, as stated in this License.
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
“Business Source License” is a trademark of MariaDB Corporation Ab.
-----------------------------------------------------------------------------
Business Source License 1.1
Terms
The Licensor hereby grants you the right to copy, modify, create derivative
works, redistribute, and make non-production use of the Licensed Work. The
Licensor may make an Additional Use Grant, above, permitting limited
production use.
Effective on the Change Date, or the fourth anniversary of the first publicly
available distribution of a specific version of the Licensed Work under this
License, whichever comes first, the Licensor hereby grants you rights under
the terms of the Change License, and the rights granted in the paragraph
above terminate.
If your use of the Licensed Work does not comply with the requirements
currently in effect as described in this License, you must purchase a
commercial license from the Licensor, its affiliated entities, or authorized
resellers, or you must refrain from using the Licensed Work.
All copies of the original and modified Licensed Work, and derivative works
of the Licensed Work, are subject to this License. This License applies
separately for each version of the Licensed Work and the Change Date may vary
for each version of the Licensed Work released by Licensor.
You must conspicuously display this License on each original or modified copy
of the Licensed Work. If you receive the Licensed Work in original or
modified form from a third party, the terms and conditions set forth in this
License apply to your use of that work.
Any use of the Licensed Work in violation of this License will automatically
terminate your rights under this License for the current and all other
versions of the Licensed Work.
This License does not grant you any right in any trademark or logo of
Licensor or its affiliates (provided that you may use a trademark or logo of
Licensor as expressly required by this License).
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
TITLE.
MariaDB hereby grants you permission to use this License’s text to license
your works, and to refer to it using the trademark “Business Source License”,
as long as you comply with the Covenants of Licensor below.
Covenants of Licensor
In consideration of the right to use this License’s text and the “Business
Source License” name and trademark, Licensor covenants to MariaDB, and to all
other recipients of the licensed work to be provided by Licensor:
1. To specify as the Change License the GPL Version 2.0 or any later version,
or a license that is compatible with GPL Version 2.0 or a later version,
where “compatible” means that software provided under the Change License can
be included in a program with software provided under GPL Version 2.0 or a
later version. Licensor may specify additional Change Licenses without
limitation.
2. To either: (a) specify an additional grant of rights to use that does not
impose any additional restriction on the right granted in this License, as
the Additional Use Grant; or (b) insert the text “None”.
3. To specify a Change Date.
4. Not to modify this License in any other way.

11
apps/emqx_connector_aggregator/README.md Normal file
View File

@ -0,0 +1,11 @@
# EMQX Connector Aggregator
This application provides common logic for connector and action implementations of EMQX to aggregate multiple incoming messsages into a container file before sending it to a blob storage backend.
## Contributing
Please see our [contributing.md](../../CONTRIBUTING.md).
## License
EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt).

12
apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl → apps/emqx_connector_aggregator/include/emqx_connector_aggregator.hrl
View File

@ -3,8 +3,8 @@
%%--------------------------------------------------------------------
-record(buffer, {
since :: emqx_bridge_s3_aggregator:timestamp(),
until :: emqx_bridge_s3_aggregator:timestamp(),
since :: emqx_connector_aggregator:timestamp(),
until :: emqx_connector_aggregator:timestamp(),
seq :: non_neg_integer(),
filename :: file:filename(),
fd :: file:io_device() | undefined,
@ -13,3 +13,11 @@
}).
-type buffer() :: #buffer{}.
-type buffer_map() :: #{
since := emqx_connector_aggregator:timestamp(),
until := emqx_connector_aggregator:timestamp(),
seq := non_neg_integer(),
filename := file:filename(),
max_records := pos_integer() | undefined
}.

7
apps/emqx_connector_aggregator/rebar.config Normal file
View File

@ -0,0 +1,7 @@
%% -*- mode: erlang; -*-
{deps, [
{emqx, {path, "../../apps/emqx"}}
]}.
{project_plugins, [erlfmt]}.

25
apps/emqx_connector_aggregator/src/emqx_connector_aggreg_app.erl Normal file
View File

@ -0,0 +1,25 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_connector_aggreg_app).
-behaviour(application).
-export([start/2, stop/1]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
%%------------------------------------------------------------------------------
%% `application` API
%%------------------------------------------------------------------------------
start(_StartType, _StartArgs) ->
emqx_connector_aggreg_sup:start_link().
stop(_State) ->
ok.
%%------------------------------------------------------------------------------
%% Internal fns
%%------------------------------------------------------------------------------

2
apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl → apps/emqx_connector_aggregator/src/emqx_connector_aggreg_buffer.erl
View File

@ -17,7 +17,7 @@
%% ...
%% ```
%% ^ ETF = Erlang External Term Format (i.e. `erlang:term_to_binary/1`).
-module(emqx_bridge_s3_aggreg_buffer).
-module(emqx_connector_aggreg_buffer).
-export([
new_writer/2,

6
apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl → apps/emqx_connector_aggregator/src/emqx_connector_aggreg_csv.erl
View File

@ -2,8 +2,8 @@
%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
%% CSV container implementation for `emqx_bridge_s3_aggregator`.
-module(emqx_bridge_s3_aggreg_csv).
%% CSV container implementation for `emqx_connector_aggregator`.
-module(emqx_connector_aggreg_csv).
%% Container API
-export([
@ -33,7 +33,7 @@
column_order => [column()]
}.
-type record() :: emqx_bridge_s3_aggregator:record().
-type record() :: emqx_connector_aggregator:record().
-type column() :: binary().
%%

195
apps/emqx_connector_aggregator/src/emqx_connector_aggreg_delivery.erl Normal file
View File

@ -0,0 +1,195 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
%% This module takes aggregated records from a buffer and delivers them to a blob storage
%% backend, wrapped in a configurable container (though currently there's only CSV).
-module(emqx_connector_aggreg_delivery).
-include_lib("snabbkaffe/include/trace.hrl").
-include("emqx_connector_aggregator.hrl").
-export([start_link/3]).
%% Internal exports
-export([
init/4,
loop/3
]).
%% Sys
-export([
system_continue/3,
system_terminate/4,
format_status/2
]).
-record(delivery, {
name :: _Name,
callback_module :: module(),
container :: emqx_connector_aggreg_csv:container(),
reader :: emqx_connector_aggreg_buffer:reader(),
transfer :: transfer_state(),
empty :: boolean()
}).
-type state() :: #delivery{}.
-type init_opts() :: #{
callback_module := module(),
any() => term()
}.
-type transfer_state() :: term().
%% @doc Initialize the transfer state, such as blob storage path, transfer options, client
%% credentials, etc. .
-callback init_transfer_state(buffer_map(), map()) -> transfer_state().
%% @doc Append data to the transfer before sending. Usually should not fail.
-callback process_append(iodata(), transfer_state()) -> transfer_state().
%% @doc Push appended transfer data to its destination (e.g.: upload a part of a
%% multi-part upload). May fail.
-callback process_write(transfer_state()) -> {ok, transfer_state()} | {error, term()}.
%% @doc Finalize the transfer and clean up any resources. May return a term summarizing
%% the transfer.
-callback process_complete(transfer_state()) -> {ok, term()}.
%%
start_link(Name, Buffer, Opts) ->
proc_lib:start_link(?MODULE, init, [self(), Name, Buffer, Opts]).
%%
-spec init(pid(), _Name, buffer(), init_opts()) -> no_return().
init(Parent, Name, Buffer, Opts) ->
?tp(connector_aggreg_delivery_started, #{action => Name, buffer => Buffer}),
Reader = open_buffer(Buffer),
Delivery = init_delivery(Name, Reader, Buffer, Opts#{action => Name}),
_ = erlang:process_flag(trap_exit, true),
ok = proc_lib:init_ack({ok, self()}),
loop(Delivery, Parent, []).
init_delivery(
Name,
Reader,
Buffer,
Opts = #{
container := ContainerOpts,
callback_module := Mod
}
) ->
BufferMap = emqx_connector_aggregator:buffer_to_map(Buffer),
#delivery{
name = Name,
callback_module = Mod,
container = mk_container(ContainerOpts),
reader = Reader,
transfer = Mod:init_transfer_state(BufferMap, Opts),
empty = true
}.
open_buffer(#buffer{filename = Filename}) ->
case file:open(Filename, [read, binary, raw]) of
{ok, FD} ->
{_Meta, Reader} = emqx_connector_aggreg_buffer:new_reader(FD),
Reader;
{error, Reason} ->
error(#{reason => buffer_open_failed, file => Filename, posix => Reason})
end.
mk_container(#{type := csv, column_order := OrderOpt}) ->
%% TODO: Deduplicate?
ColumnOrder = lists:map(fun emqx_utils_conv:bin/1, OrderOpt),
emqx_connector_aggreg_csv:new(#{column_order => ColumnOrder}).
%%
-spec loop(state(), pid(), [sys:debug_option()]) -> no_return().
loop(Delivery, Parent, Debug) ->
%% NOTE: This function is mocked in tests.
receive
Msg -> handle_msg(Msg, Delivery, Parent, Debug)
after 0 ->
process_delivery(Delivery, Parent, Debug)
end.
process_delivery(Delivery0 = #delivery{reader = Reader0}, Parent, Debug) ->
case emqx_connector_aggreg_buffer:read(Reader0) of
{Records = [#{} | _], Reader} ->
Delivery1 = Delivery0#delivery{reader = Reader},
Delivery2 = process_append_records(Records, Delivery1),
Delivery = process_write(Delivery2),
?MODULE:loop(Delivery, Parent, Debug);
{[], Reader} ->
Delivery = Delivery0#delivery{reader = Reader},
?MODULE:loop(Delivery, Parent, Debug);
eof ->
process_complete(Delivery0);
{Unexpected, _Reader} ->
exit({buffer_unexpected_record, Unexpected})
end.
process_append_records(
Records,
Delivery = #delivery{
callback_module = Mod,
container = Container0,
transfer = Transfer0
}
) ->
{Writes, Container} = emqx_connector_aggreg_csv:fill(Records, Container0),
Transfer = Mod:process_append(Writes, Transfer0),
Delivery#delivery{
container = Container,
transfer = Transfer,
empty = false
}.
process_write(Delivery = #delivery{callback_module = Mod, transfer = Transfer0}) ->
case Mod:process_write(Transfer0) of
{ok, Transfer} ->
Delivery#delivery{transfer = Transfer};
{error, Reason} ->
%% Todo: handle more gracefully? Retry?
error({transfer_failed, Reason})
end.
process_complete(#delivery{name = Name, empty = true}) ->
?tp(connector_aggreg_delivery_completed, #{action => Name, transfer => empty}),
exit({shutdown, {skipped, empty}});
process_complete(#delivery{
name = Name, callback_module = Mod, container = Container, transfer = Transfer0
}) ->
Trailer = emqx_connector_aggreg_csv:close(Container),
Transfer = Mod:process_append(Trailer, Transfer0),
{ok, Completed} = Mod:process_complete(Transfer),
?tp(connector_aggreg_delivery_completed, #{action => Name, transfer => Completed}),
ok.
%%
handle_msg({system, From, Msg}, Delivery, Parent, Debug) ->
sys:handle_system_msg(Msg, From, Parent, ?MODULE, Debug, Delivery);
handle_msg({'EXIT', Parent, Reason}, Delivery, Parent, Debug) ->
system_terminate(Reason, Parent, Debug, Delivery);
handle_msg(_Msg, Delivery, Parent, Debug) ->
?MODULE:loop(Parent, Debug, Delivery).
-spec system_continue(pid(), [sys:debug_option()], state()) -> no_return().
system_continue(Parent, Debug, Delivery) ->
?MODULE:loop(Delivery, Parent, Debug).
-spec system_terminate(_Reason, pid(), [sys:debug_option()], state()) -> _.
system_terminate(_Reason, _Parent, _Debug, #delivery{callback_module = Mod, transfer = Transfer}) ->
Mod:process_terminate(Transfer).
-spec format_status(normal, Args :: [term()]) -> _StateFormatted.
format_status(_Normal, [_PDict, _SysState, _Parent, _Debug, Delivery]) ->
#delivery{callback_module = Mod} = Delivery,
Delivery#delivery{
transfer = Mod:process_format_status(Delivery#delivery.transfer)
}.

2
apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl → apps/emqx_connector_aggregator/src/emqx_connector_aggreg_sup.erl
View File

@ -2,7 +2,7 @@
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_bridge_s3_sup).
-module(emqx_connector_aggreg_sup).
-export([
start_link/0,

6
apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl → apps/emqx_connector_aggregator/src/emqx_connector_aggreg_upload_sup.erl
View File

@ -2,7 +2,7 @@
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_bridge_s3_aggreg_upload_sup).
-module(emqx_connector_aggreg_upload_sup).
-export([
start_link/3,
@ -33,7 +33,7 @@ start_delivery(Name, Buffer) ->
supervisor:start_child(?SUPREF(Name), [Buffer]).
start_delivery_proc(Name, DeliveryOpts, Buffer) ->
emqx_bridge_s3_aggreg_delivery:start_link(Name, Buffer, DeliveryOpts).
emqx_connector_aggreg_delivery:start_link(Name, Buffer, DeliveryOpts).
%%
@ -45,7 +45,7 @@ init({root, Name, AggregOpts, DeliveryOpts}) ->
},
AggregatorChildSpec = #{
id => aggregator,
start => {emqx_bridge_s3_aggregator, start_link, [Name, AggregOpts]},
start => {emqx_connector_aggregator, start_link, [Name, AggregOpts]},
type => worker,
restart => permanent
},

13
apps/emqx_connector_aggregator/src/emqx_connector_aggregator.app.src Normal file
View File

@ -0,0 +1,13 @@
{application, emqx_connector_aggregator, [
{description, "EMQX Enterprise Connector Data Aggregator"},
{vsn, "0.1.0"},
{registered, []},
{applications, [
kernel,
stdlib
]},
{env, []},
{mod, {emqx_connector_aggreg_app, []}},
{modules, []},
{links, []}
]}.

32
apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl → apps/emqx_connector_aggregator/src/emqx_connector_aggregator.erl
View File

@ -5,18 +5,19 @@
%% This module manages buffers for aggregating records and offloads them
%% to separate "delivery" processes when they are full or time interval
%% is over.
-module(emqx_bridge_s3_aggregator).
-module(emqx_connector_aggregator).
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/trace.hrl").
-include("emqx_bridge_s3_aggregator.hrl").
-include("emqx_connector_aggregator.hrl").
-export([
start_link/2,
push_records/3,
tick/2,
take_error/1
take_error/1,
buffer_to_map/1
]).
-behaviour(gen_server).
@ -72,6 +73,15 @@ tick(Name, Timestamp) ->
take_error(Name) ->
gen_server:call(?SRVREF(Name), take_error).
buffer_to_map(#buffer{} = Buffer) ->
#{
since => Buffer#buffer.since,
until => Buffer#buffer.until,
seq => Buffer#buffer.seq,
filename => Buffer#buffer.filename,
max_records => Buffer#buffer.max_records
}.
%%
write_records_limited(Name, Buffer = #buffer{max_records = undefined}, Records) ->
@ -90,9 +100,9 @@ write_records_limited(Name, Buffer = #buffer{max_records = MaxRecords}, Records)
end.
write_records(Name, Buffer = #buffer{fd = Writer}, Records) ->
case emqx_bridge_s3_aggreg_buffer:write(Records, Writer) of
case emqx_connector_aggreg_buffer:write(Records, Writer) of
ok ->
?tp(s3_aggreg_records_written, #{action => Name, records => Records}),
?tp(connector_aggreg_records_written, #{action => Name, records => Records}),
ok;
{error, terminated} ->
BufferNext = rotate_buffer(Name, Buffer),
@ -250,9 +260,9 @@ compute_since(Timestamp, PrevSince, Interval) ->
allocate_buffer(Since, Seq, St = #st{name = Name}) ->
Buffer = #buffer{filename = Filename, cnt_records = Counter} = mk_buffer(Since, Seq, St),
{ok, FD} = file:open(Filename, [write, binary]),
Writer = emqx_bridge_s3_aggreg_buffer:new_writer(FD, _Meta = []),
Writer = emqx_connector_aggreg_buffer:new_writer(FD, _Meta = []),
_ = add_counter(Counter),
?tp(s3_aggreg_buffer_allocated, #{action => Name, filename => Filename}),
?tp(connector_aggreg_buffer_allocated, #{action => Name, filename => Filename}),
Buffer#buffer{fd = Writer}.
recover_buffer(Buffer = #buffer{filename = Filename, cnt_records = Counter}) ->
@ -274,7 +284,7 @@ recover_buffer(Buffer = #buffer{filename = Filename, cnt_records = Counter}) ->
end.
recover_buffer_writer(FD, Filename) ->
try emqx_bridge_s3_aggreg_buffer:new_reader(FD) of
try emqx_connector_aggreg_buffer:new_reader(FD) of
{_Meta, Reader} -> recover_buffer_writer(FD, Filename, Reader, 0)
catch
error:Reason ->
@ -282,7 +292,7 @@ recover_buffer_writer(FD, Filename) ->
end.
recover_buffer_writer(FD, Filename, Reader0, NWritten) ->
try emqx_bridge_s3_aggreg_buffer:read(Reader0) of
try emqx_connector_aggreg_buffer:read(Reader0) of
{Records, Reader} when is_list(Records) ->
recover_buffer_writer(FD, Filename, Reader, NWritten + length(Records));
{Unexpected, _Reader} ->
@ -303,7 +313,7 @@ recover_buffer_writer(FD, Filename, Reader0, NWritten) ->
"Buffer is truncated or corrupted somewhere in the middle. "
"Corrupted records will be discarded."
}),
Writer = emqx_bridge_s3_aggreg_buffer:takeover(Reader0),
Writer = emqx_connector_aggreg_buffer:takeover(Reader0),
{ok, Writer, NWritten}
end.
@ -362,7 +372,7 @@ lookup_current_buffer(Name) ->
%%
enqueue_delivery(Buffer, St = #st{name = Name, deliveries = Ds}) ->
{ok, Pid} = emqx_bridge_s3_aggreg_upload_sup:start_delivery(Name, Buffer),
{ok, Pid} = emqx_connector_aggreg_upload_sup:start_delivery(Name, Buffer),
MRef = erlang:monitor(process, Pid),
St#st{deliveries = Ds#{MRef => Buffer}}.

50
apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl → apps/emqx_connector_aggregator/test/emqx_connector_aggreg_buffer_SUITE.erl
View File

@ -2,7 +2,7 @@
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_bridge_s3_aggreg_buffer_SUITE).
-module(emqx_connector_aggreg_buffer_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
@ -29,7 +29,7 @@ t_write_read_cycle(Config) ->
Filename = mk_filename(?FUNCTION_NAME, Config),
Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}},
{ok, WFD} = file:open(Filename, [write, binary]),
Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata),
Writer = emqx_connector_aggreg_buffer:new_writer(WFD, Metadata),
Terms = [
[],
[[[[[[[[]]]]]]]],
@ -43,12 +43,12 @@ t_write_read_cycle(Config) ->
{<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})}
],
ok = lists:foreach(
fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end,
fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer)) end,
Terms
),
ok = file:close(WFD),
{ok, RFD} = file:open(Filename, [read, binary, raw]),
{MetadataRead, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD),
{MetadataRead, Reader} = emqx_connector_aggreg_buffer:new_reader(RFD),
?assertEqual(Metadata, MetadataRead),
TermsRead = read_until_eof(Reader),
?assertEqual(Terms, TermsRead).
@ -60,7 +60,7 @@ t_read_empty(Config) ->
{ok, RFD} = file:open(Filename, [read, binary]),
?assertError(
{buffer_incomplete, header},
emqx_bridge_s3_aggreg_buffer:new_reader(RFD)
emqx_connector_aggreg_buffer:new_reader(RFD)
).
t_read_garbage(Config) ->
@ -71,14 +71,14 @@ t_read_garbage(Config) ->
{ok, RFD} = file:open(Filename, [read, binary]),
?assertError(
badarg,
emqx_bridge_s3_aggreg_buffer:new_reader(RFD)
emqx_connector_aggreg_buffer:new_reader(RFD)
).
t_read_truncated(Config) ->
Filename = mk_filename(?FUNCTION_NAME, Config),
{ok, WFD} = file:open(Filename, [write, binary]),
Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}},
Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata),
Writer = emqx_connector_aggreg_buffer:new_writer(WFD, Metadata),
Terms = [
[[[[[[[[[[[]]]]]]]]]]],
lists:seq(1, 100000),
@ -88,36 +88,36 @@ t_read_truncated(Config) ->
LastTerm =
{<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})},
ok = lists:foreach(
fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end,
fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer)) end,
Terms
),
{ok, WPos} = file:position(WFD, cur),
?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(LastTerm, Writer)),
?assertEqual(ok, emqx_connector_aggreg_buffer:write(LastTerm, Writer)),
ok = file:close(WFD),
ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos + 1),
ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, WPos + 1),
{ok, RFD1} = file:open(Filename, [read, binary]),
{Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD1),
{Metadata, Reader0} = emqx_connector_aggreg_buffer:new_reader(RFD1),
{ReadTerms1, Reader1} = read_terms(length(Terms), Reader0),
?assertEqual(Terms, ReadTerms1),
?assertError(
badarg,
emqx_bridge_s3_aggreg_buffer:read(Reader1)
emqx_connector_aggreg_buffer:read(Reader1)
),
ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2),
ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, WPos div 2),
{ok, RFD2} = file:open(Filename, [read, binary]),
{Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD2),
{Metadata, Reader2} = emqx_connector_aggreg_buffer:new_reader(RFD2),
{ReadTerms2, Reader3} = read_terms(_FitsInto = 3, Reader2),
?assertEqual(lists:sublist(Terms, 3), ReadTerms2),
?assertError(
badarg,
emqx_bridge_s3_aggreg_buffer:read(Reader3)
emqx_connector_aggreg_buffer:read(Reader3)
).
t_read_truncated_takeover_write(Config) ->
Filename = mk_filename(?FUNCTION_NAME, Config),
{ok, WFD} = file:open(Filename, [write, binary]),
Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}},
Writer1 = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata),
Writer1 = emqx_connector_aggreg_buffer:new_writer(WFD, Metadata),
Terms1 = [
[[[[[[[[[[[]]]]]]]]]]],
lists:seq(1, 10000),
@ -129,14 +129,14 @@ t_read_truncated_takeover_write(Config) ->
{<<"application/x-octet-stream">>, rand:bytes(102400)}
],
ok = lists:foreach(
fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer1)) end,
fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer1)) end,
Terms1
),
{ok, WPos} = file:position(WFD, cur),
ok = file:close(WFD),
ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2),
ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, WPos div 2),
{ok, RWFD} = file:open(Filename, [read, write, binary]),
{Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RWFD),
{Metadata, Reader0} = emqx_connector_aggreg_buffer:new_reader(RWFD),
{ReadTerms1, Reader1} = read_terms(_Survived = 3, Reader0),
?assertEqual(
lists:sublist(Terms1, 3),
@ -144,16 +144,16 @@ t_read_truncated_takeover_write(Config) ->
),
?assertError(
badarg,
emqx_bridge_s3_aggreg_buffer:read(Reader1)
emqx_connector_aggreg_buffer:read(Reader1)
),
Writer2 = emqx_bridge_s3_aggreg_buffer:takeover(Reader1),
Writer2 = emqx_connector_aggreg_buffer:takeover(Reader1),
ok = lists:foreach(
fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer2)) end,
fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer2)) end,
Terms2
),
ok = file:close(RWFD),
{ok, RFD} = file:open(Filename, [read, binary]),
{Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD),
{Metadata, Reader2} = emqx_connector_aggreg_buffer:new_reader(RFD),
ReadTerms2 = read_until_eof(Reader2),
?assertEqual(
lists:sublist(Terms1, 3) ++ Terms2,
@ -168,12 +168,12 @@ mk_filename(Name, Config) ->
read_terms(0, Reader) ->
{[], Reader};
read_terms(N, Reader0) ->
{Term, Reader1} = emqx_bridge_s3_aggreg_buffer:read(Reader0),
{Term, Reader1} = emqx_connector_aggreg_buffer:read(Reader0),
{Terms, Reader} = read_terms(N - 1, Reader1),
{[Term | Terms], Reader}.
read_until_eof(Reader0) ->
case emqx_bridge_s3_aggreg_buffer:read(Reader0) of
case emqx_connector_aggreg_buffer:read(Reader0) of
{Term, Reader} ->
[Term | read_until_eof(Reader)];
eof ->

10
apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl → apps/emqx_connector_aggregator/test/emqx_connector_aggreg_csv_tests.erl
View File

@ -2,12 +2,12 @@
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_bridge_s3_aggreg_csv_tests).
-module(emqx_connector_aggreg_csv_tests).
-include_lib("eunit/include/eunit.hrl").
encoding_test() ->
CSV = emqx_bridge_s3_aggreg_csv:new(#{}),
CSV = emqx_connector_aggreg_csv:new(#{}),
?assertEqual(
"A,B,Ç\n"
"1.2345,string,0.0\n"
@ -28,7 +28,7 @@ encoding_test() ->
column_order_test() ->
Order = [<<"ID">>, <<"TS">>],
CSV = emqx_bridge_s3_aggreg_csv:new(#{column_order => Order}),
CSV = emqx_connector_aggreg_csv:new(#{column_order => Order}),
?assertEqual(
"ID,TS,A,B,D\n"
"1,2024-01-01,12.34,str,\"[]\"\n"
@ -63,10 +63,10 @@ fill_close(CSV, LRecords) ->
string(fill_close_(CSV, LRecords)).
fill_close_(CSV0, [Records | LRest]) ->
{Writes, CSV} = emqx_bridge_s3_aggreg_csv:fill(Records, CSV0),
{Writes, CSV} = emqx_connector_aggreg_csv:fill(Records, CSV0),
[Writes | fill_close_(CSV, LRest)];
fill_close_(CSV, []) ->
[emqx_bridge_s3_aggreg_csv:close(CSV)].
[emqx_connector_aggreg_csv:close(CSV)].
string(Writes) ->
unicode:characters_to_list(Writes).

25
apps/emqx_connector_aggregator/test/emqx_connector_aggregator_test_helpers.erl Normal file
View File

@ -0,0 +1,25 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_connector_aggregator_test_helpers).
-compile(nowarn_export_all).
-compile(export_all).
%% API
-export([]).
%%------------------------------------------------------------------------------
%% File utilities
%%------------------------------------------------------------------------------
truncate_at(Filename, Pos) ->
{ok, FD} = file:open(Filename, [read, write, binary]),
{ok, Pos} = file:position(FD, Pos),
ok = file:truncate(FD),
ok = file:close(FD).
%%------------------------------------------------------------------------------
%% Internal fns
%%------------------------------------------------------------------------------

2
apps/emqx_dashboard/src/emqx_dashboard_api.erl
View File

@ -311,7 +311,7 @@ user(delete, #{bindings := #{username := Username0}, headers := Headers} = Req)
end.
is_self_auth(?SSO_USERNAME(_, _), _) ->
fasle;
false;
is_self_auth(Username, #{<<"authorization">> := Token}) ->
is_self_auth(Username, Token);
is_self_auth(Username, #{<<"Authorization">> := Token}) ->

112
apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
View File

@ -20,11 +20,13 @@
-compile(export_all).
-import(emqx_dashboard_SUITE, [auth_header_/0]).
-import(emqx_common_test_helpers, [on_exit/1]).
-include("emqx_dashboard.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-define(SERVER, "http://127.0.0.1:18083").
-define(BASE_PATH, "/api/v5").
@ -52,10 +54,47 @@
%%--------------------------------------------------------------------
all() ->
emqx_common_test_helpers:all(?MODULE).
[
{group, common},
{group, persistent_sessions}
].
groups() ->
AllTCs = emqx_common_test_helpers:all(?MODULE),
PSTCs = persistent_session_testcases(),
[
{common, [], AllTCs -- PSTCs},
{persistent_sessions, [], PSTCs}
].
persistent_session_testcases() ->
[
t_persistent_session_stats
].
init_per_suite(Config) ->
emqx_common_test_helpers:clear_screen(),
Config.
end_per_suite(_Config) ->
ok.
init_per_group(persistent_sessions = Group, Config) ->
Apps = emqx_cth_suite:start(
[
emqx_conf,
{emqx, "session_persistence {enable = true}"},
{emqx_retainer, ?BASE_RETAINER_CONF},
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard(
"dashboard.listeners.http { enable = true, bind = 18083 }\n"
"dashboard.sample_interval = 1s"
)
],
#{work_dir => emqx_cth_suite:work_dir(Group, Config)}
),
{ok, _} = emqx_common_test_http:create_default_app(),
[{apps, Apps} | Config];
init_per_group(common = Group, Config) ->
Apps = emqx_cth_suite:start(
[
emqx,
@ -67,12 +106,12 @@ init_per_suite(Config) ->
"dashboard.sample_interval = 1s"
)
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
#{work_dir => emqx_cth_suite:work_dir(Group, Config)}
),
{ok, _} = emqx_common_test_http:create_default_app(),
[{apps, Apps} | Config].
end_per_suite(Config) ->
end_per_group(_Group, Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
@ -84,6 +123,7 @@ init_per_testcase(_TestCase, Config) ->
end_per_testcase(_TestCase, _Config) ->
ok = snabbkaffe:stop(),
emqx_common_test_helpers:call_janitor(),
ok.
%%--------------------------------------------------------------------
@ -272,6 +312,51 @@ t_monitor_api_error(_) ->
request(["monitor"], "latest=-1"),
ok.
%% Verifies that subscriptions from persistent sessions are correctly accounted for.
t_persistent_session_stats(_Config) ->
%% pre-condition
true = emqx_persistent_message:is_persistence_enabled(),
NonPSClient = start_and_connect(#{
clientid => <<"non-ps">>,
expiry_interval => 0
}),
PSClient = start_and_connect(#{
clientid => <<"ps">>,
expiry_interval => 30
}),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"non/ps/topic/+">>, 2),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"non/ps/topic">>, 2),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"common/topic/+">>, 2),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"common/topic">>, 2),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"ps/topic/+">>, 2),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"ps/topic">>, 2),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"common/topic/+">>, 2),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"common/topic">>, 2),
{ok, _} =
snabbkaffe:block_until(
?match_n_events(2, #{?snk_kind := dashboard_monitor_flushed}),
infinity
),
?retry(1_000, 10, begin
?assertMatch(
{ok, #{
%% N.B.: we currently don't perform any deduplication between persistent
%% and non-persistent routes, so we count `commont/topic' twice and get 8
%% instead of 6 here.
<<"topics">> := 8,
<<"subscriptions">> := 8
}},
request(["monitor_current"])
)
end),
%% Sanity checks
PSRouteCount = emqx_persistent_session_ds_router:stats(n_routes),
?assert(PSRouteCount > 0, #{ps_route_count => PSRouteCount}),
PSSubCount = emqx_persistent_session_bookkeeper:get_subscription_count(),
?assert(PSSubCount > 0, #{ps_sub_count => PSSubCount}),
ok.
request(Path) ->
request(Path, "").
@ -340,3 +425,22 @@ waiting_emqx_stats_and_monitor_update(WaitKey) ->
%% manually call monitor update
_ = emqx_dashboard_monitor:current_rate_cluster(),
ok.
start_and_connect(Opts) ->
Defaults = #{clean_start => false, expiry_interval => 30},
#{
clientid := ClientId,
clean_start := CleanStart,
expiry_interval := EI
} = maps:merge(Defaults, Opts),
{ok, Client} = emqtt:start_link([
{clientid, ClientId},
{clean_start, CleanStart},
{proto_ver, v5},
{properties, #{'Session-Expiry-Interval' => EI}}
]),
on_exit(fun() ->
catch emqtt:disconnect(Client, ?RC_NORMAL_DISCONNECTION, #{'Session-Expiry-Interval' => 0})
end),
{ok, _} = emqtt:connect(Client),
Client.

21
apps/emqx_durable_storage/src/emqx_ds_replication_layer_shard.erl
View File

@ -82,6 +82,8 @@ server_name(DB, Shard, Site) ->
%%
-spec servers(emqx_ds:db(), emqx_ds_replication_layer:shard_id(), Order) -> [server(), ...] when
Order :: leader_preferred | undefined.
servers(DB, Shard, _Order = leader_preferred) ->
get_servers_leader_preferred(DB, Shard);
servers(DB, Shard, _Order = undefined) ->
@ -98,7 +100,7 @@ get_servers_leader_preferred(DB, Shard) ->
Servers = ra_leaderboard:lookup_members(ClusterName),
[Leader | lists:delete(Leader, Servers)];
undefined ->
get_shard_servers(DB, Shard)
get_online_servers(DB, Shard)
end.
get_server_local_preferred(DB, Shard) ->
@ -111,7 +113,7 @@ get_server_local_preferred(DB, Shard) ->
%% TODO
%% Leader is unkonwn if there are no servers of this group on the
%% local node. We want to pick a replica in that case as well.
pick_random(get_shard_servers(DB, Shard))
pick_random(get_online_servers(DB, Shard))
end.
lookup_leader(DB, Shard) ->
@ -121,6 +123,21 @@ lookup_leader(DB, Shard) ->
ClusterName = get_cluster_name(DB, Shard),
ra_leaderboard:lookup_leader(ClusterName).
get_online_servers(DB, Shard) ->
filter_online(get_shard_servers(DB, Shard)).
filter_online(Servers) ->
case lists:filter(fun is_server_online/1, Servers) of
[] ->
%% NOTE: Must return non-empty list.
Servers;
Online ->
Online
end.
is_server_online({_Name, Node}) ->
Node == node() orelse lists:member(Node, nodes()).
pick_local(Servers) ->
case lists:keyfind(node(), 2, Servers) of
Local when is_tuple(Local) ->

2
apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
View File

@ -518,7 +518,7 @@ handle_msg({inet_reply, _Sock, ok}, State = #state{active_n = ActiveN}) ->
handle_msg({inet_reply, _Sock, {error, Reason}}, State) ->
handle_info({sock_error, Reason}, State);
handle_msg({close, Reason}, State) ->
?SLOG(debug, #{msg => "force_socket_close", reason => Reason}),
?tp(debug, force_socket_close, #{reason => Reason}),
handle_info({sock_closed, Reason}, close_socket(State));
handle_msg(
{event, connected},

22
apps/emqx_gateway/src/emqx_gateway_ctx.erl
View File

@ -39,6 +39,7 @@
%% Authentication circle
-export([
authenticate/2,
connection_expire_interval/2,
open_session/5,
open_session/6,
insert_channel_info/4,
@ -78,6 +79,13 @@ authenticate(_Ctx, ClientInfo0) ->
{error, Reason}
end.
-spec connection_expire_interval(context(), emqx_types:clientinfo()) ->
undefined | non_neg_integer().
connection_expire_interval(_Ctx, #{auth_expire_at := undefined}) ->
undefined;
connection_expire_interval(_Ctx, #{auth_expire_at := ExpireAt}) ->
max(0, ExpireAt - erlang:system_time(millisecond)).
%% @doc Register the session to the cluster.
%%
%% This function should be called after the client has authenticated
@ -157,6 +165,9 @@ set_chan_stats(_Ctx = #{gwname := GwName}, ClientId, Stats) ->
connection_closed(_Ctx = #{gwname := GwName}, ClientId) ->
emqx_gateway_cm:connection_closed(GwName, ClientId).
%%--------------------------------------------------------------------
%% Message circle
-spec authorize(
context(),
emqx_types:clientinfo(),
@ -167,6 +178,9 @@ connection_closed(_Ctx = #{gwname := GwName}, ClientId) ->
authorize(_Ctx, ClientInfo, Action, Topic) ->
emqx_access_control:authorize(ClientInfo, Action, Topic).
%%--------------------------------------------------------------------
%% Metrics & Stats
metrics_inc(_Ctx = #{gwname := GwName}, Name) ->
emqx_gateway_metrics:inc(GwName, Name).
@ -183,6 +197,8 @@ eval_mountpoint(ClientInfo = #{mountpoint := MountPoint}) ->
MountPoint1 = emqx_mountpoint:replvar(MountPoint, ClientInfo),
ClientInfo#{mountpoint := MountPoint1}.
merge_auth_result(ClientInfo, AuthResult) when is_map(ClientInfo) andalso is_map(AuthResult) ->
IsSuperuser = maps:get(is_superuser, AuthResult, false),
maps:merge(ClientInfo, AuthResult#{is_superuser => IsSuperuser}).
merge_auth_result(ClientInfo, AuthResult0) when is_map(ClientInfo) andalso is_map(AuthResult0) ->
IsSuperuser = maps:get(is_superuser, AuthResult0, false),
ExpireAt = maps:get(expire_at, AuthResult0, undefined),
AuthResult1 = maps:without([expire_at], AuthResult0),
maps:merge(ClientInfo#{auth_expire_at => ExpireAt}, AuthResult1#{is_superuser => IsSuperuser}).

2
apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl
View File

@ -82,4 +82,4 @@ t_authenticate(_) ->
?assertMatch({ok, #{is_superuser := true}}, emqx_gateway_ctx:authenticate(Ctx, Info4)),
ok.
default_result(Info) -> Info#{zone => default, is_superuser => false}.
default_result(Info) -> Info#{zone => default, is_superuser => false, auth_expire_at => undefined}.

12
apps/emqx_gateway_coap/src/emqx_coap_channel.erl
View File

@ -214,6 +214,8 @@ handle_timeout(_, {transport, Msg}, Channel) ->
call_session(timeout, Msg, Channel);
handle_timeout(_, disconnect, Channel) ->
{shutdown, normal, Channel};
handle_timeout(_, connection_expire, Channel) ->
{shutdown, expired, Channel};
handle_timeout(_, _, Channel) ->
{ok, Channel}.
@ -595,6 +597,14 @@ process_connect(
iter(Iter, reply({error, bad_request}, Msg, Result), Channel)
end.
schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
undefined ->
Channel;
Interval ->
ensure_timer(connection_expire_timer, Interval, connection_expire, Channel)
end.
run_hooks(Ctx, Name, Args) ->
emqx_gateway_ctx:metrics_inc(Ctx, Name),
emqx_hooks:run(Name, Args).
@ -619,7 +629,7 @@ ensure_connected(
NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
_ = run_hooks(Ctx, 'client.connack', [NConnInfo, connection_accepted, #{}]),
ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
Channel#channel{conninfo = NConnInfo, conn_state = connected}.
schedule_connection_expire(Channel#channel{conninfo = NConnInfo, conn_state = connected}).
%%--------------------------------------------------------------------
%% Ensure disconnected

2
apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway_coap, [
{description, "CoAP Gateway"},
{vsn, "0.1.7"},
{vsn, "0.1.8"},
{registered, []},
{applications, [kernel, stdlib, emqx, emqx_gateway]},
{env, []},

36
apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl
View File

@ -29,6 +29,7 @@
-include_lib("er_coap_client/include/coap.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/asserts.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@ -83,6 +84,17 @@ init_per_testcase(t_connection_with_authn_failed, Config) ->
fun(_) -> {error, bad_username_or_password} end
),
Config;
init_per_testcase(t_connection_with_expire, Config) ->
ok = meck:new(emqx_access_control, [passthrough, no_history]),
ok = meck:expect(
emqx_access_control,
authenticate,
fun(_) ->
{ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 100}}
end
),
snabbkaffe:start_trace(),
Config;
init_per_testcase(t_heartbeat, Config) ->
NewHeartbeat = 800,
OldConf = emqx:get_raw_config([gateway, coap]),
@ -103,6 +115,10 @@ end_per_testcase(t_heartbeat, Config) ->
OldConf = ?config(old_conf, Config),
{ok, _} = emqx_gateway_conf:update_gateway(coap, OldConf),
ok;
end_per_testcase(t_connection_with_expire, Config) ->
snabbkaffe:stop(),
meck:unload(emqx_access_control),
Config;
end_per_testcase(_, Config) ->
ok = meck:unload(emqx_access_control),
Config.
@ -270,6 +286,26 @@ t_connection_with_authn_failed(_) ->
),
ok.
t_connection_with_expire(_) ->
ChId = {{127, 0, 0, 1}, 5683},
{ok, Sock} = er_coap_udp_socket:start_link(),
{ok, Channel} = er_coap_udp_socket:get_channel(Sock, ChId),
URI = ?MQTT_PREFIX ++ "/connection?clientid=client1",
?assertWaitEvent(
begin
Req = make_req(post),
{ok, created, _Data} = do_request(Channel, URI, Req)
end,
#{
?snk_kind := conn_process_terminated,
clientid := <<"client1">>,
reason := {shutdown, expired}
},
5000
).
t_publish(_) ->
%% can publish to a normal topic
Topics = [

15
apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl
View File

@ -302,6 +302,9 @@ handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) -
{shutdown, Reason, Channel};
handle_timeout(_TRef, force_close_idle, Channel) ->
{shutdown, idle_timeout, Channel};
handle_timeout(_TRef, connection_expire, Channel) ->
NChannel = remove_timer_ref(connection_expire, Channel),
{ok, [{event, disconnected}, {close, expired}], NChannel};
handle_timeout(_TRef, Msg, Channel) ->
?SLOG(warning, #{
msg => "unexpected_timeout_signal",
@ -666,10 +669,18 @@ ensure_connected(
) ->
NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
Channel#channel{
schedule_connection_expire(Channel#channel{
conninfo = NConnInfo,
conn_state = connected
}.
}).
schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
undefined ->
Channel;
Interval ->
ensure_timer(connection_expire, Interval, Channel)
end.
ensure_disconnected(
Reason,

2
apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway_exproto, [
{description, "ExProto Gateway"},
{vsn, "0.1.9"},
{vsn, "0.1.10"},
{registered, []},
{applications, [kernel, stdlib, grpc, emqx, emqx_gateway]},
{env, []},

49
apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl
View File

@ -21,6 +21,7 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/asserts.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
@ -81,6 +82,7 @@ groups() ->
t_raw_publish,
t_auth_deny,
t_acl_deny,
t_auth_expire,
t_hook_connected_disconnected,
t_hook_session_subscribed_unsubscribed,
t_hook_message_delivered
@ -157,14 +159,17 @@ end_per_group(_, Cfg) ->
init_per_testcase(TestCase, Cfg) when
TestCase == t_enter_passive_mode
->
snabbkaffe:start_trace(),
case proplists:get_value(listener_type, Cfg) of
udp -> {skip, ignore};
_ -> Cfg
end;
init_per_testcase(_TestCase, Cfg) ->
snabbkaffe:start_trace(),
Cfg.
end_per_testcase(_TestCase, _Cfg) ->
snabbkaffe:stop(),
ok.
listener_confs(Type) ->
@ -290,6 +295,42 @@ t_auth_deny(Cfg) ->
end,
meck:unload([emqx_gateway_ctx]).
t_auth_expire(Cfg) ->
SockType = proplists:get_value(listener_type, Cfg),
Sock = open(SockType),
Client = #{
proto_name => <<"demo">>,
proto_ver => <<"v0.1">>,
clientid => <<"test_client_1">>
},
Password = <<"123456">>,
ok = meck:new(emqx_access_control, [passthrough, no_history]),
ok = meck:expect(
emqx_access_control,
authenticate,
fun(_) ->
{ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
end
),
ConnBin = frame_connect(Client, Password),
ConnAckBin = frame_connack(0),
?assertWaitEvent(
begin
send(Sock, ConnBin),
{ok, ConnAckBin} = recv(Sock, 5000)
end,
#{
?snk_kind := conn_process_terminated,
clientid := <<"test_client_1">>,
reason := {shutdown, expired}
},
5000
).
t_acl_deny(Cfg) ->
SockType = proplists:get_value(listener_type, Cfg),
Sock = open(SockType),
@ -332,7 +373,6 @@ t_acl_deny(Cfg) ->
close(Sock).
t_keepalive_timeout(Cfg) ->
ok = snabbkaffe:start_trace(),
SockType = proplists:get_value(listener_type, Cfg),
Sock = open(SockType),
@ -383,8 +423,7 @@ t_keepalive_timeout(Cfg) ->
?assertEqual(1, length(?of_kind(conn_process_terminated, Trace))),
%% socket port should be closed
?assertEqual({error, closed}, recv(Sock, 5000))
end,
snabbkaffe:stop().
end.
t_hook_connected_disconnected(Cfg) ->
SockType = proplists:get_value(listener_type, Cfg),
@ -513,7 +552,6 @@ t_hook_message_delivered(Cfg) ->
emqx_hooks:del('message.delivered', {?MODULE, hook_fun5}).
t_idle_timeout(Cfg) ->
ok = snabbkaffe:start_trace(),
SockType = proplists:get_value(listener_type, Cfg),
Sock = open(SockType),
@ -551,8 +589,7 @@ t_idle_timeout(Cfg) ->
{ok, #{reason := {shutdown, idle_timeout}}},
?block_until(#{?snk_kind := conn_process_terminated}, 10000)
)
end,
snabbkaffe:stop().
end.
%%--------------------------------------------------------------------
%% Utils

2
apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway_gbt32960, [
{description, "GBT32960 Gateway"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{applications, [kernel, stdlib, emqx, emqx_gateway]},
{env, []},

22
apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl
View File

@ -72,7 +72,8 @@
-define(TIMER_TABLE, #{
alive_timer => keepalive,
retry_timer => retry_delivery
retry_timer => retry_delivery,
connection_expire_timer => connection_expire
}).
-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]).
@ -468,6 +469,13 @@ handle_timeout(
{Outgoings2, NChannel} = dispatch_frame(Channel#channel{inflight = NInflight}),
{ok, [{outgoing, Outgoings ++ Outgoings2}], reset_timer(retry_timer, NChannel)}
end;
handle_timeout(
_TRef,
connection_expire,
Channel
) ->
NChannel = clean_timer(connection_expire_timer, Channel),
{ok, [{event, disconnected}, {close, expired}], NChannel};
handle_timeout(_TRef, Msg, Channel) ->
log(error, #{msg => "unexpected_timeout", content => Msg}, Channel),
{ok, Channel}.
@ -591,10 +599,18 @@ ensure_connected(
) ->
NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
Channel#channel{
schedule_connection_expire(Channel#channel{
conninfo = NConnInfo,
conn_state = connected
}.
}).
schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
undefined ->
Channel;
Interval ->
ensure_timer(connection_expire_timer, Interval, Channel)
end.
process_connect(
Frame,

31
apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl
View File

@ -11,6 +11,7 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/asserts.hrl").
-define(BYTE, 8 / big - integer).
-define(WORD, 16 / big - integer).
@ -52,6 +53,14 @@ end_per_suite(Config) ->
emqx_cth_suite:stop(?config(suite_apps, Config)),
ok.
init_per_testcase(_, Config) ->
snabbkaffe:start_trace(),
Config.
end_per_testcase(_, _Config) ->
snabbkaffe:stop(),
ok.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% helper functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
encode(Cmd, Vin, Data) ->
@ -171,6 +180,28 @@ t_case01_login_channel_info(_Config) ->
ok = gen_tcp:close(Socket).
t_case01_auth_expire(_Config) ->
ok = meck:new(emqx_access_control, [passthrough, no_history]),
ok = meck:expect(
emqx_access_control,
authenticate,
fun(_) ->
{ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
end
),
?assertWaitEvent(
begin
{ok, _Socket} = login_first()
end,
#{
?snk_kind := conn_process_terminated,
clientid := <<"1G1BL52P7TR115520">>,
reason := {shutdown, expired}
},
5000
).
t_case02_reportinfo_0x01(_Config) ->
% send VEHICLE LOGIN
{ok, Socket} = login_first(),

2
apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway_lwm2m, [
{description, "LwM2M Gateway"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap, xmerl]},
{env, []},

14
apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl
View File

@ -202,6 +202,8 @@ handle_timeout(_, {transport, _} = Msg, Channel) ->
call_session(timeout, Msg, Channel);
handle_timeout(_, disconnect, Channel) ->
{shutdown, normal, Channel};
handle_timeout(_, connection_expire, Channel) ->
{shutdown, expired, Channel};
handle_timeout(_, _, Channel) ->
{ok, Channel}.
@ -353,10 +355,18 @@ ensure_connected(
NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
Channel#channel{
schedule_connection_expire(Channel#channel{
conninfo = NConnInfo,
conn_state = connected
}.
}).
schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
undefined ->
Channel;
Interval ->
make_timer(connection_expire, Interval, connection_expire, Channel)
end.
%%--------------------------------------------------------------------
%% Ensure disconnected

41
apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl
View File

@ -36,6 +36,7 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/asserts.hrl").
-record(coap_content, {content_format, payload = <<>>}).
@ -66,6 +67,7 @@ groups() ->
[
{test_grp_0_register, [RepeatOpt], [
case01_register,
case01_auth_expire,
case01_register_additional_opts,
%% TODO now we can't handle partial decode packet
%% case01_register_incorrect_opts,
@ -145,6 +147,7 @@ end_per_suite(Config) ->
Config.
init_per_testcase(TestCase, Config) ->
snabbkaffe:start_trace(),
GatewayConfig =
case TestCase of
case09_auto_observe ->
@ -171,6 +174,7 @@ end_per_testcase(_AllTestCase, Config) ->
timer:sleep(300),
gen_udp:close(?config(sock, Config)),
emqtt:disconnect(?config(emqx_c, Config)),
snabbkaffe:stop(),
ok = application:stop(emqx_gateway).
default_config() ->
@ -280,6 +284,43 @@ case01_register(Config) ->
timer:sleep(50),
false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
case01_auth_expire(Config) ->
ok = meck:new(emqx_access_control, [passthrough, no_history]),
ok = meck:expect(
emqx_access_control,
authenticate,
fun(_) ->
{ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
end
),
%%----------------------------------------
%% REGISTER command
%%----------------------------------------
UdpSock = ?config(sock, Config),
Epn = "urn:oma:lwm2m:oma:3",
MsgId = 12,
?assertWaitEvent(
test_send_coap_request(
UdpSock,
post,
sprintf("coap://127.0.0.1:~b/rd?ep=~ts&lt=345&lwm2m=1", [?PORT, Epn]),
#coap_content{
content_format = <<"text/plain">>,
payload = <<"</1>, </2>, </3>, </4>, </5>">>
},
[],
MsgId
),
#{
?snk_kind := conn_process_terminated,
clientid := <<"urn:oma:lwm2m:oma:3">>,
reason := {shutdown, expired}
},
5000
).
case01_register_additional_opts(Config) ->
%%----------------------------------------
%% REGISTER command

15
apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl
View File

@ -364,10 +364,18 @@ ensure_connected(
) ->
NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
Channel#channel{
schedule_connection_expire(Channel#channel{
conninfo = NConnInfo,
conn_state = connected
}.
}).
schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
undefined ->
Channel;
Interval ->
ensure_timer(connection_expire, Interval, Channel)
end.
process_connect(
Channel = #channel{
@ -2122,6 +2130,9 @@ handle_timeout(_TRef, expire_session, Channel) ->
shutdown(expired, Channel);
handle_timeout(_TRef, expire_asleep, Channel) ->
shutdown(asleep_timeout, Channel);
handle_timeout(_TRef, connection_expire, Channel) ->
NChannel = clean_timer(connection_expire, Channel),
handle_out(disconnect, expired, NChannel);
handle_timeout(_TRef, Msg, Channel) ->
%% NOTE
%% We do not expect `emqx_mqttsn_session` to set up any custom timers (i.e with

39
apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl
View File

@ -33,6 +33,7 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/asserts.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
@ -141,6 +142,14 @@ end_per_suite(Config) ->
emqx_common_test_http:delete_default_app(),
emqx_cth_suite:stop(?config(suite_apps, Config)).
init_per_testcase(_TestCase, Config) ->
snabbkaffe:start_trace(),
Config.
end_per_testcase(_TestCase, _Config) ->
snabbkaffe:stop(),
ok.
restart_mqttsn_with_subs_resume_on() ->
Conf = emqx:get_raw_config([gateway, mqttsn]),
emqx_gateway_conf:update_gateway(
@ -206,6 +215,36 @@ t_connect(_) ->
?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
gen_udp:close(Socket).
t_auth_expire(_) ->
SockName = {'mqttsn:udp:default', 1884},
?assertEqual(true, lists:keymember(SockName, 1, esockd:listeners())),
ok = meck:new(emqx_access_control, [passthrough, no_history]),
ok = meck:expect(
emqx_access_control,
authenticate,
fun(_) ->
{ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
end
),
?assertWaitEvent(
begin
{ok, Socket} = gen_udp:open(0, [binary]),
send_connect_msg(Socket, <<"client_id_test1">>),
?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
gen_udp:close(Socket)
end,
#{
?snk_kind := conn_process_terminated,
clientid := <<"client_id_test1">>,
reason := {shutdown, expired}
},
5000
).
t_first_disconnect(_) ->
SockName = {'mqttsn:udp:default', 1884},
?assertEqual(true, lists:keymember(SockName, 1, esockd:listeners())),

36
apps/emqx_gateway_ocpp/src/emqx_ocpp_channel.erl
View File

@ -89,7 +89,8 @@
-type replies() :: reply() | [reply()].
-define(TIMER_TABLE, #{
alive_timer => keepalive
alive_timer => keepalive,
connection_expire_timer => connection_expire
}).
-define(INFO_KEYS, [
@ -315,20 +316,13 @@ enrich_client(
expiry_interval => 0,
receive_maximum => 1
},
NClientInfo = fix_mountpoint(
NClientInfo =
ClientInfo#{
clientid => ClientId,
username => Username
}
),
},
{ok, Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo}}.
fix_mountpoint(ClientInfo = #{mountpoint := undefined}) ->
ClientInfo;
fix_mountpoint(ClientInfo = #{mountpoint := Mountpoint}) ->
Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo),
ClientInfo#{mountpoint := Mountpoint1}.
set_log_meta(#channel{
clientinfo = #{clientid := ClientId},
conninfo = #{peername := Peername}
@ -350,15 +344,14 @@ check_banned(_UserInfo, #channel{clientinfo = ClientInfo}) ->
auth_connect(
#{password := Password},
#channel{clientinfo = ClientInfo} = Channel
#channel{ctx = Ctx, clientinfo = ClientInfo} = Channel
) ->
#{
clientid := ClientId,
username := Username
} = ClientInfo,
case emqx_access_control:authenticate(ClientInfo#{password => Password}) of
{ok, AuthResult} ->
NClientInfo = maps:merge(ClientInfo, AuthResult),
case emqx_gateway_ctx:authenticate(Ctx, ClientInfo#{password => Password}) of
{ok, NClientInfo} ->
{ok, Channel#channel{clientinfo = NClientInfo}};
{error, Reason} ->
?SLOG(warning, #{
@ -659,6 +652,9 @@ handle_timeout(
{error, timeout} ->
handle_out(disconnect, keepalive_timeout, Channel)
end;
handle_timeout(_TRef, connection_expire, Channel) ->
%% No take over implemented, so just shutdown
shutdown(expired, Channel);
handle_timeout(_TRef, Msg, Channel) ->
?SLOG(error, #{msg => "unexpected_timeout", timeout_msg => Msg}),
{ok, Channel}.
@ -796,10 +792,18 @@ ensure_connected(
) ->
NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
ok = run_hooks('client.connected', [ClientInfo, NConnInfo]),
Channel#channel{
schedule_connection_expire(Channel#channel{
conninfo = NConnInfo,
conn_state = connected
}.
}).
schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
undefined ->
Channel;
Interval ->
ensure_timer(connection_expire_timer, Interval, Channel)
end.
ensure_disconnected(
Reason,

4
apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl
View File

@ -20,6 +20,7 @@
-include("emqx_ocpp.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/types.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-logger_header("[OCPP/WS]").
@ -513,7 +514,8 @@ websocket_close(Reason, State) ->
handle_info({sock_closed, Reason}, State).
terminate(Reason, _Req, #state{channel = Channel}) ->
?SLOG(debug, #{msg => "terminated", reason => Reason}),
ClientId = emqx_ocpp_channel:info(clientid, Channel),
?tp(debug, conn_process_terminated, #{reason => Reason, clientid => ClientId}),
emqx_ocpp_channel:terminate(Reason, Channel);
terminate(_Reason, _Req, _UnExpectedState) ->
ok.

31
apps/emqx_gateway_ocpp/test/emqx_ocpp_SUITE.erl
View File

@ -19,6 +19,7 @@
-include("emqx_ocpp.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/asserts.hrl").
-compile(export_all).
-compile(nowarn_export_all).
@ -32,8 +33,6 @@
]
).
-define(HEARTBEAT, <<$\n>>).
%% erlfmt-ignore
-define(CONF_DEFAULT, <<"
gateway.ocpp {
@ -82,6 +81,14 @@ end_per_suite(Config) ->
emqx_cth_suite:stop(?config(suite_apps, Config)),
ok.
init_per_testcase(_TestCase, Config) ->
snabbkaffe:start_trace(),
Config.
end_per_testcase(_TestCase, _Config) ->
snabbkaffe:stop(),
ok.
default_config() ->
?CONF_DEFAULT.
@ -188,6 +195,26 @@ t_adjust_keepalive_timer(_Config) ->
?assertEqual(undefined, emqx_gateway_cm:get_chan_info(ocpp, <<"client1">>)),
ok.
t_auth_expire(_Config) ->
ok = meck:new(emqx_access_control, [passthrough, no_history]),
ok = meck:expect(
emqx_access_control,
authenticate,
fun(_) ->
{ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
end
),
?assertWaitEvent(
{ok, _ClientPid} = connect("127.0.0.1", 33033, <<"client1">>),
#{
?snk_kind := conn_process_terminated,
clientid := <<"client1">>,
reason := {shutdown, expired}
},
5000
).
t_listeners_status(_Config) ->
{200, [Listener]} = request(get, "/gateways/ocpp/listeners"),
?assertMatch(

20
apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl
View File

@ -93,7 +93,8 @@
-define(TIMER_TABLE, #{
incoming_timer => keepalive,
outgoing_timer => keepalive_send,
clean_trans_timer => clean_trans
clean_trans_timer => clean_trans,
connection_expire_timer => connection_expire
}).
-define(TRANS_TIMEOUT, 60000).
@ -356,10 +357,18 @@ ensure_connected(
) ->
NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
Channel#channel{
schedule_connection_expire(Channel#channel{
conninfo = NConnInfo,
conn_state = connected
}.
}).
schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
undefined ->
Channel;
Interval ->
ensure_timer(connection_expire_timer, Interval, Channel)
end.
process_connect(
Channel = #channel{
@ -1137,7 +1146,10 @@ handle_timeout(_TRef, clean_trans, Channel = #channel{transaction = Trans}) ->
end,
Trans
),
{ok, ensure_clean_trans_timer(Channel#channel{transaction = NTrans})}.
{ok, ensure_clean_trans_timer(Channel#channel{transaction = NTrans})};
handle_timeout(_TRef, connection_expire, Channel) ->
%% No session take over implemented, just shut down
shutdown(expired, Channel).
%%--------------------------------------------------------------------
%% Terminate

37
apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl
View File

@ -18,6 +18,7 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/asserts.hrl").
-include("emqx_stomp.hrl").
-compile(export_all).
@ -78,6 +79,14 @@ end_per_suite(Config) ->
emqx_cth_suite:stop(?config(suite_apps, Config)),
ok.
init_per_testcase(_TestCase, Config) ->
snabbkaffe:start_trace(),
Config.
end_per_testcase(_TestCase, _Config) ->
snabbkaffe:stop(),
ok.
default_config() ->
?CONF_DEFAULT.
@ -141,6 +150,34 @@ t_connect(_) ->
end,
with_connection(ProtocolError).
t_auth_expire(_) ->
ok = meck:new(emqx_access_control, [passthrough, no_history]),
ok = meck:expect(
emqx_access_control,
authenticate,
fun(_) ->
{ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
end
),
ConnectWithExpire = fun(Sock) ->
?assertWaitEvent(
begin
ok = send_connection_frame(Sock, <<"guest">>, <<"guest">>, <<"1000,2000">>),
{ok, Frame} = recv_a_frame(Sock),
?assertMatch(<<"CONNECTED">>, Frame#stomp_frame.command)
end,
#{
?snk_kind := conn_process_terminated,
clientid := _,
reason := {shutdown, expired}
},
5000
)
end,
with_connection(ConnectWithExpire),
meck:unload(emqx_access_control).
t_heartbeat(_) ->
%% Test heart beat
with_connection(fun(Sock) ->

4
apps/emqx_license/src/emqx_license.erl
View File

@ -117,7 +117,9 @@ import_config(#{<<"license">> := Config}) ->
{ok, #{root_key => license, changed => Changed1}};
Error ->
{error, #{root_key => license, reason => Error}}
end.
end;
import_config(_RawConf) ->
{ok, #{root_key => license, changed => []}}.
%%------------------------------------------------------------------------------
%% emqx_config_handler callbacks

1
apps/emqx_machine/priv/reboot_lists.eterm
View File

@ -89,6 +89,7 @@
emqx_license,
emqx_enterprise,
emqx_message_validation,
emqx_connector_aggregator,
emqx_bridge_kafka,
emqx_bridge_pulsar,
emqx_bridge_gcp_pubsub,

13
apps/emqx_management/src/emqx_mgmt_api_clients.erl
View File

@ -1745,6 +1745,18 @@ format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}, Opts) ->
format_channel_info(undefined, {ClientId, PSInfo0 = #{}}, _Opts) ->
format_persistent_session_info(ClientId, PSInfo0).
format_persistent_session_info(
_ClientId, #{metadata := #{offline_info := #{chan_info := ChanInfo, stats := Stats}}} = PSInfo
) ->
Info0 = format_channel_info(_Node = undefined, {_Key = undefined, ChanInfo, Stats}, #{
fields => all
}),
Info0#{
connected => false,
durable => true,
is_persistent => true,
subscriptions_cnt => maps:size(maps:get(subscriptions, PSInfo, #{}))
};
format_persistent_session_info(ClientId, PSInfo0) ->
Metadata = maps:get(metadata, PSInfo0, #{}),
{ProtoName, ProtoVer} = maps:get(protocol, Metadata),
@ -1762,6 +1774,7 @@ format_persistent_session_info(ClientId, PSInfo0) ->
clientid => ClientId,
connected => false,
connected_at => CreatedAt,
durable => true,
ip_address => IpAddress,
is_persistent => true,
port => Port,

19
apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
View File

@ -171,21 +171,32 @@ subscriptions(get, #{query_string := QString}) ->
{200, Result}
end.
format(WhichNode, {{Topic, _Subscriber}, SubOpts}) ->
format(WhichNode, {{Topic, Subscriber}, SubOpts}) ->
FallbackClientId =
case is_binary(Subscriber) of
true ->
Subscriber;
false ->
%% e.g.: could be a pid...
null
end,
maps:merge(
#{
topic => emqx_topic:maybe_format_share(Topic),
clientid => maps:get(subid, SubOpts, null),
node => WhichNode,
clientid => maps:get(subid, SubOpts, FallbackClientId),
node => convert_null(WhichNode),
durable => false
},
maps:with([qos, nl, rap, rh], SubOpts)
maps:with([qos, nl, rap, rh, durable], SubOpts)
).
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
convert_null(undefined) -> null;
convert_null(Val) -> Val.
check_match_topic(#{<<"match_topic">> := MatchTopic}) ->
try emqx_topic:parse(MatchTopic) of
{#share{}, _} -> {error, invalid_match_topic};

90
apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
View File

@ -49,6 +49,7 @@ persistent_session_testcases() ->
t_persistent_sessions3,
t_persistent_sessions4,
t_persistent_sessions5,
t_persistent_sessions_subscriptions1,
t_list_clients_v2
].
client_msgs_testcases() ->
@ -333,7 +334,7 @@ t_persistent_sessions2(Config) ->
%% 2) Client connects to the same node and takes over, listed only once.
C2 = connect_client(#{port => Port1, clientid => ClientId}),
assert_single_client(O#{node => N1, clientid => ClientId, status => connected}),
ok = emqtt:disconnect(C2, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}),
disconnect_and_destroy_session(C2),
?retry(
100,
20,
@ -377,7 +378,7 @@ t_persistent_sessions3(Config) ->
list_request(APIPort, "node=" ++ atom_to_list(N1))
)
),
ok = emqtt:disconnect(C2, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0})
disconnect_and_destroy_session(C2)
end,
[]
),
@ -417,7 +418,7 @@ t_persistent_sessions4(Config) ->
list_request(APIPort, "node=" ++ atom_to_list(N1))
)
),
ok = emqtt:disconnect(C2, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0})
disconnect_and_destroy_session(C2)
end,
[]
),
@ -552,6 +553,63 @@ t_persistent_sessions5(Config) ->
),
ok.
%% Check that the output of `/clients/:clientid/subscriptions' has the expected keys.
t_persistent_sessions_subscriptions1(Config) ->
[N1, _N2] = ?config(nodes, Config),
APIPort = 18084,
Port1 = get_mqtt_port(N1, tcp),
?assertMatch({ok, {{_, 200, _}, _, #{<<"data">> := []}}}, list_request(APIPort)),
?check_trace(
begin
ClientId = <<"c1">>,
C1 = connect_client(#{port => Port1, clientid => ClientId}),
{ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(C1, <<"topic/1">>, 1),
?assertMatch(
{ok,
{{_, 200, _}, _, [
#{
<<"durable">> := true,
<<"node">> := <<_/binary>>,
<<"clientid">> := ClientId,
<<"qos">> := 1,
<<"rap">> := 0,
<<"rh">> := 0,
<<"nl">> := 0,
<<"topic">> := <<"topic/1">>
}
]}},
get_subscriptions_request(APIPort, ClientId)
),
%% Just disconnect
ok = emqtt:disconnect(C1),
?assertMatch(
{ok,
{{_, 200, _}, _, [
#{
<<"durable">> := true,
<<"node">> := null,
<<"clientid">> := ClientId,
<<"qos">> := 1,
<<"rap">> := 0,
<<"rh">> := 0,
<<"nl">> := 0,
<<"topic">> := <<"topic/1">>
}
]}},
get_subscriptions_request(APIPort, ClientId)
),
C2 = connect_client(#{port => Port1, clientid => ClientId}),
disconnect_and_destroy_session(C2),
ok
end,
[]
),
ok.
t_clients_bad_value_type(_) ->
%% get /clients
AuthHeader = [emqx_common_test_http:default_auth_header()],
@ -1800,6 +1858,16 @@ maybe_json_decode(X) ->
{error, _} -> X
end.
get_subscriptions_request(APIPort, ClientId) ->
Host = "http://127.0.0.1:" ++ integer_to_list(APIPort),
Path = emqx_mgmt_api_test_util:api_path(Host, ["clients", ClientId, "subscriptions"]),
request(get, Path, []).
get_client_request(Port, ClientId) ->
Host = "http://127.0.0.1:" ++ integer_to_list(Port),
Path = emqx_mgmt_api_test_util:api_path(Host, ["clients", ClientId]),
request(get, Path, []).
list_request(Port) ->
list_request(Port, _QueryParams = "").
@ -1874,6 +1942,19 @@ assert_single_client(Opts) ->
{ok, {{_, 200, _}, _, #{<<"connected">> := IsConnected}}},
lookup_request(ClientId, APIPort)
),
?assertMatch(
{ok,
{{_, 200, _}, _, #{
<<"connected">> := IsConnected,
<<"is_persistent">> := true,
%% contains statistics from disconnect time
<<"recv_pkt">> := _,
%% contains channel info from disconnect time
<<"listener">> := _,
<<"clean_start">> := _
}}},
get_client_request(APIPort, ClientId)
),
ok.
connect_client(Opts) ->
@ -1937,3 +2018,6 @@ do_traverse_in_reverse_v2(APIPort, QueryParams0, [Cursor | Rest], DirectOrderCli
{ok, {{_, 200, _}, _, #{<<"data">> := Rows}}} = Res0,
ClientIds = [ClientId || #{<<"clientid">> := ClientId} <- Rows],
do_traverse_in_reverse_v2(APIPort, QueryParams0, Rest, DirectOrderClientIds, ClientIds ++ Acc).
disconnect_and_destroy_session(Client) ->
ok = emqtt:disconnect(Client, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}).

163
apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
View File

@ -290,54 +290,125 @@ t_http_test_json_formatter(_Config) ->
end
|| JSONEntry <- LogEntries
],
ListIterFun =
fun
ListIterFunRec([]) ->
ok;
ListIterFunRec([Item | Rest]) ->
receive
From ->
From ! {list_iter_item, Item}
end,
ListIterFunRec(Rest)
end,
ListIter = spawn_link(fun() -> ListIterFun(DecodedLogEntries) end),
NextFun =
fun() ->
ListIter ! self(),
receive
{list_iter_item, Item} ->
Item
end
end,
?assertMatch(
[
#{<<"meta">> := #{<<"payload">> := <<"log_this_message">>}},
#{<<"meta">> := #{<<"payload">> := <<"\nlog\nthis\nmessage">>}},
#{
<<"meta">> := #{<<"payload">> := <<"\\\nlog\n_\\n_this\nmessage\\">>}
},
#{<<"meta">> := #{<<"payload">> := <<"\"log_this_message\"">>}},
#{<<"meta">> := #{<<"str">> := <<"str">>}},
#{<<"meta">> := #{<<"term">> := <<"{notjson}">>}},
#{<<"meta">> := <<_/binary>>},
#{<<"meta">> := #{<<"integer">> := 42}},
#{<<"meta">> := #{<<"float">> := 1.2}},
#{<<"meta">> := <<_/binary>>},
#{<<"meta">> := <<_/binary>>},
#{<<"meta">> := <<_/binary>>},
#{<<"meta">> := #{<<"sub">> := #{}}},
#{<<"meta">> := #{<<"sub">> := #{<<"key">> := <<"value">>}}},
#{<<"meta">> := #{<<"true">> := <<"true">>, <<"false">> := <<"false">>}},
#{
<<"meta">> := #{
<<"list">> := #{
<<"key">> := <<"value">>,
<<"key2">> := <<"value2">>
}
}
},
#{
<<"meta">> := #{
<<"client_ids">> := [<<"a">>, <<"b">>, <<"c">>]
}
},
#{
<<"meta">> := #{
<<"rule_ids">> := [<<"a">>, <<"b">>, <<"c">>]
}
},
#{
<<"meta">> := #{
<<"action_info">> := #{
<<"type">> := <<"http">>,
<<"name">> := <<"emqx_bridge_http_test_lib">>
}
#{<<"meta">> := #{<<"payload">> := <<"log_this_message">>}},
NextFun()
),
?assertMatch(
#{<<"meta">> := #{<<"payload">> := <<"\nlog\nthis\nmessage">>}},
NextFun()
),
?assertMatch(
#{
<<"meta">> := #{<<"payload">> := <<"\\\nlog\n_\\n_this\nmessage\\">>}
},
NextFun()
),
?assertMatch(
#{<<"meta">> := #{<<"payload">> := <<"\"log_this_message\"">>}},
NextFun()
),
?assertMatch(
#{<<"meta">> := #{<<"str">> := <<"str">>}},
NextFun()
),
?assertMatch(
#{<<"meta">> := #{<<"term">> := <<"{notjson}">>}},
NextFun()
),
?assertMatch(
#{<<"meta">> := <<_/binary>>},
NextFun()
),
?assertMatch(
#{<<"meta">> := #{<<"integer">> := 42}},
NextFun()
),
?assertMatch(
#{<<"meta">> := #{<<"float">> := 1.2}},
NextFun()
),
?assertMatch(
#{<<"meta">> := <<_/binary>>},
NextFun()
),
?assertMatch(
#{<<"meta">> := <<_/binary>>},
NextFun()
),
?assertMatch(
#{<<"meta">> := <<_/binary>>},
NextFun()
),
?assertMatch(
#{<<"meta">> := #{<<"sub">> := #{}}},
NextFun()
),
?assertMatch(
#{<<"meta">> := #{<<"sub">> := #{<<"key">> := <<"value">>}}},
NextFun()
),
?assertMatch(
#{<<"meta">> := #{<<"true">> := true, <<"false">> := false}},
NextFun()
),
?assertMatch(
#{
<<"meta">> := #{
<<"list">> := #{
<<"key">> := <<"value">>,
<<"key2">> := <<"value2">>
}
}
| _
],
DecodedLogEntries
},
NextFun()
),
?assertMatch(
#{
<<"meta">> := #{
<<"client_ids">> := [<<"a">>, <<"b">>, <<"c">>]
}
},
NextFun()
),
?assertMatch(
#{
<<"meta">> := #{
<<"rule_ids">> := [<<"a">>, <<"b">>, <<"c">>]
}
},
NextFun()
),
?assertMatch(
#{
<<"meta">> := #{
<<"action_info">> := #{
<<"type">> := <<"http">>,
<<"name">> := <<"emqx_bridge_http_test_lib">>
}
}
},
NextFun()
),
{ok, Delete} = request_api(delete, api_path("trace/" ++ binary_to_list(Name))),
?assertEqual(<<>>, Delete),
@ -495,7 +566,7 @@ create_trace(Name, Type, TypeValue, Start) ->
?block_until(#{?snk_kind := update_trace_done})
end,
fun(Trace) ->
?assertMatch([#{}], ?of_kind(update_trace_done, Trace))
?assertMatch([#{} | _], ?of_kind(update_trace_done, Trace))
end
).

28
apps/emqx_message_validation/src/emqx_message_validation.erl
View File

@ -69,10 +69,22 @@ remove_handler() ->
ok.
load() ->
lists:foreach(fun insert/1, emqx:get_config(?VALIDATIONS_CONF_PATH, [])).
Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
lists:foreach(
fun({Pos, Validation}) ->
ok = emqx_message_validation_registry:insert(Pos, Validation)
end,
lists:enumerate(Validations)
).
unload() ->
lists:foreach(fun delete/1, emqx:get_config(?VALIDATIONS_CONF_PATH, [])).
Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
lists:foreach(
fun(Validation) ->
ok = emqx_message_validation_registry:delete(Validation)
end,
Validations
).
-spec list() -> [validation()].
list() ->
@ -81,7 +93,7 @@ list() ->
-spec reorder([validation_name()]) ->
{ok, _} | {error, _}.
reorder(Order) ->
emqx:update_config(
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{reorder, Order},
#{override_to => cluster}
@ -95,7 +107,7 @@ lookup(Name) ->
-spec insert(validation()) ->
{ok, _} | {error, _}.
insert(Validation) ->
emqx:update_config(
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{append, Validation},
#{override_to => cluster}
@ -104,7 +116,7 @@ insert(Validation) ->
-spec update(validation()) ->
{ok, _} | {error, _}.
update(Validation) ->
emqx:update_config(
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{update, Validation},
#{override_to => cluster}
@ -113,7 +125,7 @@ update(Validation) ->
-spec delete(validation_name()) ->
{ok, _} | {error, _}.
delete(Name) ->
emqx:update_config(
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{delete, Name},
#{override_to => cluster}
@ -247,8 +259,8 @@ evaluate_schema_check(Check, Validation, #message{payload = Data}) ->
#{name := Name} = Validation,
ExtraArgs =
case Check of
#{type := protobuf, message_name := MessageName} ->
[MessageName];
#{type := protobuf, message_type := MessageType} ->
[MessageType];
_ ->
[]
end,

18
apps/emqx_message_validation/src/emqx_message_validation_schema.erl
View File

@ -18,8 +18,6 @@
api_schema/1
]).
-export([validate_name/1]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
@ -55,7 +53,7 @@ fields(validation) ->
binary(),
#{
required => true,
validator => fun validate_name/1,
validator => fun emqx_resource:validate_name/1,
desc => ?DESC("name")
}
)},
@ -123,8 +121,8 @@ fields(check_protobuf) ->
[
{type, mk(protobuf, #{default => protobuf, desc => ?DESC("check_protobuf_type")})},
{schema, mk(binary(), #{required => true, desc => ?DESC("check_protobuf_schema")})},
{message_name,
mk(binary(), #{required => true, desc => ?DESC("check_protobuf_message_name")})}
{message_type,
mk(binary(), #{required => true, desc => ?DESC("check_protobuf_message_type")})}
];
fields(check_avro) ->
[
@ -200,16 +198,6 @@ ensure_array(undefined, _) -> undefined;
ensure_array(L, _) when is_list(L) -> L;
ensure_array(B, _) -> [B].
validate_name(Name) ->
%% see `MAP_KEY_RE' in hocon_tconf
RE = <<"^[A-Za-z0-9]+[A-Za-z0-9-_]*$">>,
case re:run(Name, RE, [{capture, none}]) of
match ->
ok;
nomatch ->
{error, <<"must conform to regex: ", RE/binary>>}
end.
validate_sql(SQL) ->
case emqx_message_validation:parse_sql_check(SQL) of
{ok, _Parsed} ->

14
apps/emqx_message_validation/test/emqx_message_validation_http_api_SUITE.erl
View File

@ -317,9 +317,9 @@ avro_create_serde(SerdeName) ->
on_exit(fun() -> ok = emqx_schema_registry:delete_schema(SerdeName) end),
ok.
protobuf_valid_payloads(SerdeName, MessageName) ->
protobuf_valid_payloads(SerdeName, MessageType) ->
lists:map(
fun(Payload) -> emqx_schema_registry_serde:encode(SerdeName, Payload, [MessageName]) end,
fun(Payload) -> emqx_schema_registry_serde:encode(SerdeName, Payload, [MessageType]) end,
[
#{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>},
#{<<"name">> => <<"some name">>, <<"id">> => 10}
@ -1176,11 +1176,11 @@ t_schema_check_avro(_Config) ->
t_schema_check_protobuf(_Config) ->
SerdeName = <<"myserde">>,
MessageName = <<"Person">>,
MessageType = <<"Person">>,
protobuf_create_serde(SerdeName),
Name1 = <<"foo">>,
Check1 = schema_check(protobuf, SerdeName, #{<<"message_name">> => MessageName}),
Check1 = schema_check(protobuf, SerdeName, #{<<"message_type">> => MessageType}),
Validation1 = validation(Name1, [Check1]),
{201, _} = insert(Validation1),
@ -1192,7 +1192,7 @@ t_schema_check_protobuf(_Config) ->
ok = publish(C, <<"t/1">>, {raw, Payload}),
?assertReceive({publish, _})
end,
protobuf_valid_payloads(SerdeName, MessageName)
protobuf_valid_payloads(SerdeName, MessageType)
),
lists:foreach(
fun(Payload) ->
@ -1203,7 +1203,7 @@ t_schema_check_protobuf(_Config) ->
),
%% Bad config: unknown message name
Check2 = schema_check(protobuf, SerdeName, #{<<"message_name">> => <<"idontexist">>}),
Check2 = schema_check(protobuf, SerdeName, #{<<"message_type">> => <<"idontexist">>}),
Validation2 = validation(Name1, [Check2]),
{200, _} = update(Validation2),
@ -1212,7 +1212,7 @@ t_schema_check_protobuf(_Config) ->
ok = publish(C, <<"t/1">>, {raw, Payload}),
?assertNotReceive({publish, _})
end,
protobuf_valid_payloads(SerdeName, MessageName)
protobuf_valid_payloads(SerdeName, MessageType)
),
ok.

Some files were not shown because too many files have changed in this diff Show More