diff --git a/apps/emqx/include/emqx_trace.hrl b/apps/emqx/include/emqx_trace.hrl index 27dd8b6c8..7630fbc56 100644 --- a/apps/emqx/include/emqx_trace.hrl +++ b/apps/emqx/include/emqx_trace.hrl @@ -35,6 +35,11 @@ end_at :: integer() | undefined | '_' }). +-record(emqx_trace_format_func_data, { + function :: fun((any()) -> any()), + data :: any() +}). + -define(SHARD, ?COMMON_SHARD). -define(MAX_SIZE, 30). diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index 368398b92..854e56fc5 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -110,7 +110,7 @@ reclaim_seq(Topic) -> stats_fun() -> safe_update_stats(subscriber_val(), 'subscribers.count', 'subscribers.max'), - safe_update_stats(table_size(?SUBSCRIPTION), 'subscriptions.count', 'subscriptions.max'), + safe_update_stats(subscription_count(), 'subscriptions.count', 'subscriptions.max'), safe_update_stats(table_size(?SUBOPTION), 'suboptions.count', 'suboptions.max'). safe_update_stats(undefined, _Stat, _MaxStat) -> @@ -118,6 +118,16 @@ safe_update_stats(undefined, _Stat, _MaxStat) -> safe_update_stats(Val, Stat, MaxStat) when is_integer(Val) -> emqx_stats:setstat(Stat, MaxStat, Val). +subscription_count() -> + NonPSCount = table_size(?SUBSCRIPTION), + PSCount = emqx_persistent_session_bookkeeper:get_subscription_count(), + case is_integer(NonPSCount) of + true -> + NonPSCount + PSCount; + false -> + PSCount + end. + subscriber_val() -> sum_subscriber(table_size(?SUBSCRIBER), table_size(?SHARED_SUBSCRIBER)). diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 05358f889..eb54f6ba1 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1075,7 +1075,7 @@ handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel = ?IS_MQTT_V5) - Packet = ?DISCONNECT_PACKET(ReasonCode, Props), {ok, [?REPLY_OUTGOING(Packet), ?REPLY_CLOSE(ReasonName)], Channel}; handle_out(disconnect, {_ReasonCode, ReasonName, _Props}, Channel) -> - {ok, {close, ReasonName}, Channel}; + {ok, ?REPLY_CLOSE(ReasonName), Channel}; handle_out(auth, {ReasonCode, Properties}, Channel) -> {ok, ?AUTH_PACKET(ReasonCode, Properties), Channel}; handle_out(Type, Data, Channel) -> @@ -1406,6 +1406,16 @@ handle_timeout( {_, Quota2} -> {ok, clean_timer(TimerName, Channel#channel{quota = Quota2})} end; +handle_timeout( + _TRef, + connection_expire, + #channel{conn_state = ConnState} = Channel0 +) -> + Channel1 = clean_timer(connection_expire, Channel0), + case ConnState of + disconnected -> {ok, Channel1}; + _ -> handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel1) + end; handle_timeout(TRef, Msg, Channel) -> case emqx_hooks:run_fold('client.timeout', [TRef, Msg], []) of [] -> @@ -1810,18 +1820,23 @@ log_auth_failure(Reason) -> %% Merge authentication result into ClientInfo %% Authentication result may include: %% 1. `is_superuser': The superuser flag from various backends -%% 2. `acl': ACL rules from JWT, HTTP auth backend -%% 3. `client_attrs': Extra client attributes from JWT, HTTP auth backend -%% 4. Maybe more non-standard fields used by hook callbacks +%% 2. `expire_at`: Authentication validity deadline, the client will be disconnected after this time +%% 3. `acl': ACL rules from JWT, HTTP auth backend +%% 4. `client_attrs': Extra client attributes from JWT, HTTP auth backend +%% 5. Maybe more non-standard fields used by hook callbacks merge_auth_result(ClientInfo, AuthResult0) when is_map(ClientInfo) andalso is_map(AuthResult0) -> IsSuperuser = maps:get(is_superuser, AuthResult0, false), - AuthResult = maps:without([client_attrs], AuthResult0), + ExpireAt = maps:get(expire_at, AuthResult0, undefined), + AuthResult = maps:without([client_attrs, expire_at], AuthResult0), Attrs0 = maps:get(client_attrs, ClientInfo, #{}), Attrs1 = maps:get(client_attrs, AuthResult0, #{}), Attrs = maps:merge(Attrs0, Attrs1), NewClientInfo = maps:merge( ClientInfo#{client_attrs => Attrs}, - AuthResult#{is_superuser => IsSuperuser} + AuthResult#{ + is_superuser => IsSuperuser, + auth_expire_at => ExpireAt + } ), fix_mountpoint(NewClientInfo). @@ -2228,10 +2243,16 @@ ensure_connected( ) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), - Channel#channel{ + schedule_connection_expire(Channel#channel{ conninfo = trim_conninfo(NConnInfo), conn_state = connected - }. + }). + +schedule_connection_expire(Channel = #channel{clientinfo = #{auth_expire_at := undefined}}) -> + Channel; +schedule_connection_expire(Channel = #channel{clientinfo = #{auth_expire_at := ExpireAt}}) -> + Interval = max(0, ExpireAt - erlang:system_time(millisecond)), + ensure_timer(connection_expire, Interval, Channel). trim_conninfo(ConnInfo) -> maps:without( @@ -2615,10 +2636,15 @@ disconnect_and_shutdown( -> NChannel = ensure_disconnected(Reason, Channel), shutdown(Reason, Reply, ?DISCONNECT_PACKET(reason_code(Reason)), NChannel); -%% mqtt v3/v4 sessions, mqtt v5 other conn_state sessions -disconnect_and_shutdown(Reason, Reply, Channel) -> +%% mqtt v3/v4 connected sessions +disconnect_and_shutdown(Reason, Reply, Channel = #channel{conn_state = ConnState}) when + ConnState =:= connected orelse ConnState =:= reauthenticating +-> NChannel = ensure_disconnected(Reason, Channel), - shutdown(Reason, Reply, NChannel). + shutdown(Reason, Reply, NChannel); +%% other conn_state sessions +disconnect_and_shutdown(Reason, Reply, Channel) -> + shutdown(Reason, Reply, Channel). -compile({inline, [sp/1, flag/1]}). sp(true) -> 1; diff --git a/apps/emqx/src/emqx_cm_sup.erl b/apps/emqx/src/emqx_cm_sup.erl index 3b8e53961..d8b9aeb57 100644 --- a/apps/emqx/src/emqx_cm_sup.erl +++ b/apps/emqx/src/emqx_cm_sup.erl @@ -53,6 +53,7 @@ init([]) -> RegistryKeeper = child_spec(emqx_cm_registry_keeper, 5000, worker), Manager = child_spec(emqx_cm, 5000, worker), DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor), + DSSessionBookkeeper = child_spec(emqx_persistent_session_bookkeeper, 5_000, worker), Children = [ Banned, @@ -62,7 +63,8 @@ init([]) -> Registry, RegistryKeeper, Manager, - DSSessionGCSup + DSSessionGCSup, + DSSessionBookkeeper ], {ok, {SupFlags, Children}}. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 6b5a946c5..5ea39d5e8 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -1036,8 +1036,8 @@ to_quicer_listener_opts(Opts) -> SSLOpts = maps:from_list(ssl_opts(Opts)), Opts1 = maps:filter( fun - (cacertfile, undefined) -> fasle; - (password, undefined) -> fasle; + (cacertfile, undefined) -> false; + (password, undefined) -> false; (_, _) -> true end, Opts diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 92c0bb561..856b6111c 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -229,7 +229,7 @@ best_effort_json_obj(Map, Config) -> do_format_msg("~p", [Map], Config) end. -json(A, _) when is_atom(A) -> atom_to_binary(A, utf8); +json(A, _) when is_atom(A) -> A; json(I, _) when is_integer(I) -> I; json(F, _) when is_float(F) -> F; json(P, C) when is_pid(P) -> json(pid_to_list(P), C); diff --git a/apps/emqx/src/emqx_persistent_session_bookkeeper.erl b/apps/emqx/src/emqx_persistent_session_bookkeeper.erl new file mode 100644 index 000000000..42751161f --- /dev/null +++ b/apps/emqx/src/emqx_persistent_session_bookkeeper.erl @@ -0,0 +1,107 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_persistent_session_bookkeeper). + +-behaviour(gen_server). + +%% API +-export([ + start_link/0, + get_subscription_count/0 +]). + +%% `gen_server' API +-export([ + init/1, + handle_continue/2, + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +%%------------------------------------------------------------------------------ +%% Type declarations +%%------------------------------------------------------------------------------ + +%% call/cast/info events +-record(tally_subs, {}). +-record(get_subscription_count, {}). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +-spec start_link() -> gen_server:start_ret(). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, _InitOpts = #{}, _Opts = []). + +%% @doc Gets a cached view of the cluster-global count of persistent subscriptions. +-spec get_subscription_count() -> non_neg_integer(). +get_subscription_count() -> + case emqx_persistent_message:is_persistence_enabled() of + true -> + gen_server:call(?MODULE, #get_subscription_count{}, infinity); + false -> + 0 + end. + +%%------------------------------------------------------------------------------ +%% `gen_server' API +%%------------------------------------------------------------------------------ + +init(_Opts) -> + case emqx_persistent_message:is_persistence_enabled() of + true -> + State = #{subs_count => 0}, + {ok, State, {continue, #tally_subs{}}}; + false -> + ignore + end. + +handle_continue(#tally_subs{}, State0) -> + State = tally_persistent_subscriptions(State0), + ensure_subs_tally_timer(), + {noreply, State}. + +handle_call(#get_subscription_count{}, _From, State) -> + #{subs_count := N} = State, + {reply, N, State}; +handle_call(_Call, _From, State) -> + {reply, {error, bad_call}, State}. + +handle_cast(_Cast, State) -> + {noreply, State}. + +handle_info(#tally_subs{}, State0) -> + State = tally_persistent_subscriptions(State0), + ensure_subs_tally_timer(), + {noreply, State}; +handle_info(_Info, State) -> + {noreply, State}. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ + +tally_persistent_subscriptions(State0) -> + N = emqx_persistent_session_ds_state:total_subscription_count(), + State0#{subs_count := N}. + +ensure_subs_tally_timer() -> + Timeout = emqx_config:get([session_persistence, subscription_count_refresh_interval]), + _ = erlang:send_after(Timeout, self(), #tally_subs{}), + ok. diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 4bfefe5b6..28c370ba9 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -658,16 +658,17 @@ replay_batch(Srs0, Session0, ClientInfo) -> %%-------------------------------------------------------------------- -spec disconnect(session(), emqx_types:conninfo()) -> {shutdown, session()}. -disconnect(Session = #{s := S0}, ConnInfo) -> - S1 = emqx_persistent_session_ds_state:set_last_alive_at(now_ms(), S0), - S2 = +disconnect(Session = #{id := Id, s := S0}, ConnInfo) -> + S1 = maybe_set_offline_info(S0, Id), + S2 = emqx_persistent_session_ds_state:set_last_alive_at(now_ms(), S1), + S3 = case ConnInfo of #{expiry_interval := EI} when is_number(EI) -> - emqx_persistent_session_ds_state:set_expiry_interval(EI, S1); + emqx_persistent_session_ds_state:set_expiry_interval(EI, S2); _ -> - S1 + S2 end, - S = emqx_persistent_session_ds_state:commit(S2), + S = emqx_persistent_session_ds_state:commit(S3), {shutdown, Session#{s => S}}. -spec terminate(Reason :: term(), session()) -> ok. @@ -702,7 +703,7 @@ list_client_subscriptions(ClientId) -> maps:fold( fun(Topic, #{current_state := CS}, Acc) -> #{subopts := SubOpts} = maps:get(CS, SStates), - Elem = {Topic, SubOpts}, + Elem = {Topic, SubOpts#{durable => true}}, [Elem | Acc] end, [], @@ -1175,6 +1176,19 @@ try_get_live_session(ClientId) -> not_found end. +-spec maybe_set_offline_info(emqx_persistent_session_ds_state:t(), emqx_types:clientid()) -> + emqx_persistent_session_ds_state:t(). +maybe_set_offline_info(S, Id) -> + case emqx_cm:lookup_client({clientid, Id}) of + [{_Key, ChannelInfo, Stats}] -> + emqx_persistent_session_ds_state:set_offline_info( + #{chan_info => ChannelInfo, stats => Stats}, + S + ); + _ -> + S + end. + %%-------------------------------------------------------------------- %% SeqNo tracking %% -------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_persistent_session_ds.hrl b/apps/emqx/src/emqx_persistent_session_ds.hrl index 79920629a..12372e5be 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.hrl +++ b/apps/emqx/src/emqx_persistent_session_ds.hrl @@ -81,5 +81,6 @@ -define(will_message, will_message). -define(clientinfo, clientinfo). -define(protocol, protocol). +-define(offline_info, offline_info). -endif. diff --git a/apps/emqx/src/emqx_persistent_session_ds_state.erl b/apps/emqx/src/emqx_persistent_session_ds_state.erl index 9efffc7ff..8f5f01e5c 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_state.erl +++ b/apps/emqx/src/emqx_persistent_session_ds_state.erl @@ -35,6 +35,7 @@ -export([get_expiry_interval/1, set_expiry_interval/2]). -export([get_clientinfo/1, set_clientinfo/2]). -export([get_will_message/1, set_will_message/2, clear_will_message/1, clear_will_message_now/1]). +-export([set_offline_info/2]). -export([get_peername/1, set_peername/2]). -export([get_protocol/1, set_protocol/2]). -export([new_id/1]). @@ -53,6 +54,7 @@ cold_get_subscription/2, fold_subscriptions/3, n_subscriptions/1, + total_subscription_count/0, put_subscription/3, del_subscription/2 ]). @@ -372,6 +374,10 @@ clear_will_message_now(SessionId) when is_binary(SessionId) -> clear_will_message(Rec) -> set_will_message(undefined, Rec). +-spec set_offline_info(_Info :: map(), t()) -> t(). +set_offline_info(Info, Rec) -> + set_meta(?offline_info, Info, Rec). + -spec new_id(t()) -> {emqx_persistent_session_ds:subscription_id(), t()}. new_id(Rec) -> LastId = @@ -401,6 +407,12 @@ fold_subscriptions(Fun, Acc, Rec) -> n_subscriptions(Rec) -> gen_size(?subscriptions, Rec). +-spec total_subscription_count() -> non_neg_integer(). +total_subscription_count() -> + mria:async_dirty(?DS_MRIA_SHARD, fun() -> + mnesia:foldl(fun(#kv{}, Acc) -> Acc + 1 end, 0, ?subscription_tab) + end). + -spec put_subscription( emqx_persistent_session_ds:topic_filter(), emqx_persistent_session_ds_subs:subscription(), diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index 17222446e..710665ba4 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -189,7 +189,17 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- stats_fun() -> - emqx_stats:setstat('topics.count', 'topics.max', emqx_router:stats(n_routes)). + PSRouteCount = persistent_route_count(), + NonPSRouteCount = emqx_router:stats(n_routes), + emqx_stats:setstat('topics.count', 'topics.max', PSRouteCount + NonPSRouteCount). + +persistent_route_count() -> + case emqx_persistent_message:is_persistence_enabled() of + true -> + emqx_persistent_session_ds_router:stats(n_routes); + false -> + 0 + end. cleanup_routes(Node) -> emqx_router:cleanup_routes(Node). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 4a9672e34..6ab133757 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1713,6 +1713,14 @@ fields("session_persistence") -> desc => ?DESC(session_ds_session_gc_batch_size) } )}, + {"subscription_count_refresh_interval", + sc( + timeout_duration(), + #{ + default => <<"5s">>, + importance => ?IMPORTANCE_HIDDEN + } + )}, {"message_retention_period", sc( timeout_duration(), diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 6a623f6a8..09a846832 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -545,13 +545,19 @@ to_client_opts(Type, Opts) -> {depth, Get(depth)}, {password, ensure_str(Get(password))}, {secure_renegotiate, Get(secure_renegotiate)} - ], + ] ++ hostname_check(Verify), Versions ); false -> [] end. +hostname_check(verify_none) -> + []; +hostname_check(verify_peer) -> + %% allow wildcard certificates + [{customize_hostname_check, [{match_fun, public_key:pkix_verify_hostname_match_fun(https)}]}]. + resolve_cert_path_for_read_strict(Path) -> case resolve_cert_path_for_read(Path) of undefined -> diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 91de65b39..3e8f36890 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -31,7 +31,8 @@ log/4, rendered_action_template/2, make_rendered_action_template_trace_context/1, - rendered_action_template_with_ctx/2 + rendered_action_template_with_ctx/2, + is_rule_trace_active/0 ]). -export([ @@ -96,6 +97,16 @@ unsubscribe(Topic, SubOpts) -> ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}). rendered_action_template(<<"action:", _/binary>> = ActionID, RenderResult) -> + do_rendered_action_template(ActionID, RenderResult); +rendered_action_template(#{mod := _, func := _} = ActionID, RenderResult) -> + do_rendered_action_template(ActionID, RenderResult); +rendered_action_template(_ActionID, _RenderResult) -> + %% We do nothing if we don't get a valid Action ID. This can happen when + %% called from connectors that are used for actions as well as authz and + %% authn. + ok. + +do_rendered_action_template(ActionID, RenderResult) -> TraceResult = ?TRACE( "QUERY_RENDER", "action_template_rendered", @@ -108,23 +119,25 @@ rendered_action_template(<<"action:", _/binary>> = ActionID, RenderResult) -> #{stop_action_after_render := true} -> %% We throw an unrecoverable error to stop action before the %% resource is called/modified - StopMsg = lists:flatten( + ActionIDStr = + case ActionID of + Bin when is_binary(Bin) -> + Bin; + Term -> + ActionIDFormatted = io_lib:format("~tw", [Term]), + unicode:characters_to_binary(ActionIDFormatted) + end, + StopMsg = io_lib:format( "Action ~ts stopped after template rendering due to test setting.", - [ActionID] - ) - ), + [ActionIDStr] + ), MsgBin = unicode:characters_to_binary(StopMsg), error(?EMQX_TRACE_STOP_ACTION(MsgBin)); _ -> ok end, - TraceResult; -rendered_action_template(_ActionID, _RenderResult) -> - %% We do nothing if we don't get a valid Action ID. This can happen when - %% called from connectors that are used for actions as well as authz and - %% authn. - ok. + TraceResult. %% The following two functions are used for connectors that don't do the %% rendering in the main process (the one that called on_*query). In this case @@ -165,6 +178,16 @@ rendered_action_template_with_ctx( logger:set_process_metadata(OldMetaData) end. +is_rule_trace_active() -> + case logger:get_process_metadata() of + #{rule_id := RID} when is_binary(RID) -> + true; + #{rule_ids := RIDs} when map_size(RIDs) > 0 -> + true; + _ -> + false + end. + log(List, Msg, Meta) -> log(debug, List, Msg, Meta). @@ -382,7 +405,14 @@ code_change(_, State, _Extra) -> {ok, State}. insert_new_trace(Trace) -> - transaction(fun emqx_trace_dl:insert_new_trace/1, [Trace]). + case transaction(fun emqx_trace_dl:insert_new_trace/1, [Trace]) of + {error, _} = Error -> + Error; + Res -> + %% We call this to ensure the trace is active when we return + check(), + Res + end. update_trace(Traces) -> Now = now_second(), diff --git a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl index 6cbcc0510..1e40c6849 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_formatter.erl @@ -15,9 +15,11 @@ %%-------------------------------------------------------------------- -module(emqx_trace_formatter). -include("emqx_mqtt.hrl"). +-include("emqx_trace.hrl"). -export([format/2]). -export([format_meta_map/1]). +-export([evaluate_lazy_values/1]). %% logger_formatter:config/0 is not exported. -type config() :: map(). @@ -28,18 +30,35 @@ LogEvent :: logger:log_event(), Config :: config(). format( - #{level := debug, meta := Meta = #{trace_tag := Tag}, msg := Msg}, + #{level := debug, meta := Meta0 = #{trace_tag := Tag}, msg := Msg}, #{payload_encode := PEncode} ) -> + Meta1 = evaluate_lazy_values(Meta0), Time = emqx_utils_calendar:now_to_rfc3339(microsecond), - ClientId = to_iolist(maps:get(clientid, Meta, "")), - Peername = maps:get(peername, Meta, ""), - MetaBin = format_meta(Meta, PEncode), + ClientId = to_iolist(maps:get(clientid, Meta1, "")), + Peername = maps:get(peername, Meta1, ""), + MetaBin = format_meta(Meta1, PEncode), Msg1 = to_iolist(Msg), Tag1 = to_iolist(Tag), [Time, " [", Tag1, "] ", ClientId, "@", Peername, " msg: ", Msg1, ", ", MetaBin, "\n"]; format(Event, Config) -> - emqx_logger_textfmt:format(Event, Config). + emqx_logger_textfmt:format(evaluate_lazy_values(Event), Config). + +evaluate_lazy_values(Map) when is_map(Map) -> + maps:map(fun evaluate_lazy_values_kv/2, Map); +evaluate_lazy_values(V) -> + V. + +evaluate_lazy_values_kv(_K, #emqx_trace_format_func_data{function = Formatter, data = V}) -> + try + NewV = Formatter(V), + evaluate_lazy_values(NewV) + catch + _:_ -> + V + end; +evaluate_lazy_values_kv(_K, V) -> + evaluate_lazy_values(V). format_meta_map(Meta) -> Encode = emqx_trace_handler:payload_encode(), diff --git a/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl index 8f748ed9f..82c5a31ee 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl @@ -16,6 +16,7 @@ -module(emqx_trace_json_formatter). -include("emqx_mqtt.hrl"). +-include("emqx_trace.hrl"). -export([format/2]). @@ -30,15 +31,16 @@ LogEvent :: logger:log_event(), Config :: config(). format( - LogMap, + LogMap0, #{payload_encode := PEncode} ) -> + LogMap1 = emqx_trace_formatter:evaluate_lazy_values(LogMap0), %% We just make some basic transformations on the input LogMap and then do %% an external call to create the JSON text Time = emqx_utils_calendar:now_to_rfc3339(microsecond), - LogMap1 = LogMap#{time => Time}, - LogMap2 = prepare_log_map(LogMap1, PEncode), - [emqx_logger_jsonfmt:best_effort_json(LogMap2, [force_utf8]), "\n"]. + LogMap2 = LogMap1#{time => Time}, + LogMap3 = prepare_log_map(LogMap2, PEncode), + [emqx_logger_jsonfmt:best_effort_json(LogMap3, [force_utf8]), "\n"]. %%%----------------------------------------------------------------- %%% Helper Functions @@ -48,21 +50,26 @@ prepare_log_map(LogMap, PEncode) -> NewKeyValuePairs = [prepare_key_value(K, V, PEncode) || {K, V} <- maps:to_list(LogMap)], maps:from_list(NewKeyValuePairs). -prepare_key_value(K, {Formatter, V}, PEncode) when is_function(Formatter, 1) -> - %% A cusom formatter is provided with the value - try - NewV = Formatter(V), - prepare_key_value(K, NewV, PEncode) - catch - _:_ -> - {K, V} - end; -prepare_key_value(K, {ok, Status, Headers, Body}, PEncode) when - is_integer(Status), is_list(Headers), is_binary(Body) +prepare_key_value(host, {I1, I2, I3, I4} = IP, _PEncode) when + is_integer(I1), + is_integer(I2), + is_integer(I3), + is_integer(I4) -> - %% This is unlikely anything else then info about a HTTP request so we make - %% it more structured - prepare_key_value(K, #{status => Status, headers => Headers, body => Body}, PEncode); + %% We assume this is an IP address + {host, unicode:characters_to_binary(inet:ntoa(IP))}; +prepare_key_value(host, {I1, I2, I3, I4, I5, I6, I7, I8} = IP, _PEncode) when + is_integer(I1), + is_integer(I2), + is_integer(I3), + is_integer(I4), + is_integer(I5), + is_integer(I6), + is_integer(I7), + is_integer(I8) +-> + %% We assume this is an IP address + {host, unicode:characters_to_binary(inet:ntoa(IP))}; prepare_key_value(payload = K, V, PEncode) -> NewV = try @@ -81,6 +88,21 @@ prepare_key_value(packet = K, V, PEncode) -> V end, {K, NewV}; +prepare_key_value(K, {recoverable_error, Msg} = OrgV, PEncode) -> + try + prepare_key_value( + K, + #{ + error_type => recoverable_error, + msg => Msg, + additional_info => <<"The operation may be retried.">> + }, + PEncode + ) + catch + _:_ -> + {K, OrgV} + end; prepare_key_value(rule_ids = K, V, _PEncode) -> NewV = try @@ -137,6 +159,8 @@ format_map_set_to_list(Map) -> ], lists:sort(Items). +format_action_info(#{mod := _Mod, func := _Func} = FuncCall) -> + FuncCall; format_action_info(V) -> [<<"action">>, Type, Name | _] = binary:split(V, <<":">>, [global]), #{ diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 4b3fa1209..a30cb33f6 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1061,6 +1061,7 @@ clientinfo(InitProps) -> clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, + auth_expire_at => undefined, is_bridge => false, mountpoint => undefined }, diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index a9b4a8328..54cc2ee51 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -475,6 +475,7 @@ zone_global_defaults() -> message_retention_period => 86400000, renew_streams_interval => 5000, session_gc_batch_size => 100, - session_gc_interval => 600000 + session_gc_interval => 600000, + subscription_count_refresh_interval => 5000 } }. diff --git a/apps/emqx/test/emqx_connection_expire_SUITE.erl b/apps/emqx/test/emqx_connection_expire_SUITE.erl new file mode 100644 index 000000000..c7a76dc2a --- /dev/null +++ b/apps/emqx/test/emqx_connection_expire_SUITE.erl @@ -0,0 +1,54 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connection_expire_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +%%-------------------------------------------------------------------- +%% CT callbacks +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). + +t_disonnect_by_auth_info(_) -> + _ = process_flag(trap_exit, true), + + _ = meck:new(emqx_access_control, [passthrough, no_history]), + _ = meck:expect(emqx_access_control, authenticate, fun(_) -> + {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}} + end), + + {ok, C} = emqtt:start_link([{proto_ver, v5}]), + {ok, _} = emqtt:connect(C), + + receive + {disconnected, ?RC_NOT_AUTHORIZED, #{}} -> ok + after 5000 -> + ct:fail("Client should be disconnected by timeout") + end. diff --git a/apps/emqx/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl index 53eccfb25..a3ebb09c9 100644 --- a/apps/emqx/test/emqx_tls_lib_tests.erl +++ b/apps/emqx/test/emqx_tls_lib_tests.erl @@ -240,7 +240,7 @@ to_client_opts_test() -> Versions13Only = ['tlsv1.3'], Options = #{ enable => true, - verify => "Verify", + verify => verify_none, server_name_indication => "SNI", ciphers => "Ciphers", depth => "depth", @@ -249,9 +249,16 @@ to_client_opts_test() -> secure_renegotiate => "secure_renegotiate", reuse_sessions => "reuse_sessions" }, - Expected1 = lists:usort(maps:keys(Options) -- [enable]), + Expected0 = lists:usort(maps:keys(Options) -- [enable]), + Expected1 = lists:sort(Expected0 ++ [customize_hostname_check]), ?assertEqual( - Expected1, lists:usort(proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options))) + Expected0, lists:usort(proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options))) + ), + ?assertEqual( + Expected1, + lists:usort( + proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options#{verify => verify_peer})) + ) ), Expected2 = lists:usort( diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_chains.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_chains.erl index 0d21058e3..946ef9ff3 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_chains.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_chains.erl @@ -142,6 +142,8 @@ end). -type state() :: #{atom() => term()}. -type extra() :: #{ is_superuser := boolean(), + %% millisecond timestamp + expire_at => pos_integer(), atom() => term() }. -type user_info() :: #{ diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl index d04f8b678..c534fa3a9 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl @@ -78,6 +78,7 @@ authenticate( Credential, #{ verify_claims := VerifyClaims0, + disconnect_after_expire := DisconnectAfterExpire, jwk := JWK, acl_claim_name := AclClaimName, from := From @@ -86,11 +87,12 @@ authenticate( JWT = maps:get(From, Credential), JWKs = [JWK], VerifyClaims = render_expected(VerifyClaims0, Credential), - verify(JWT, JWKs, VerifyClaims, AclClaimName); + verify(JWT, JWKs, VerifyClaims, AclClaimName, DisconnectAfterExpire); authenticate( Credential, #{ verify_claims := VerifyClaims0, + disconnect_after_expire := DisconnectAfterExpire, jwk_resource := ResourceId, acl_claim_name := AclClaimName, from := From @@ -106,7 +108,7 @@ authenticate( {ok, JWKs} -> JWT = maps:get(From, Credential), VerifyClaims = render_expected(VerifyClaims0, Credential), - verify(JWT, JWKs, VerifyClaims, AclClaimName) + verify(JWT, JWKs, VerifyClaims, AclClaimName, DisconnectAfterExpire) end. destroy(#{jwk_resource := ResourceId}) -> @@ -125,6 +127,7 @@ create2(#{ secret := Secret0, secret_base64_encoded := Base64Encoded, verify_claims := VerifyClaims, + disconnect_after_expire := DisconnectAfterExpire, acl_claim_name := AclClaimName, from := From }) -> @@ -136,6 +139,7 @@ create2(#{ {ok, #{ jwk => JWK, verify_claims => VerifyClaims, + disconnect_after_expire => DisconnectAfterExpire, acl_claim_name => AclClaimName, from => From }} @@ -145,6 +149,7 @@ create2(#{ algorithm := 'public-key', public_key := PublicKey, verify_claims := VerifyClaims, + disconnect_after_expire := DisconnectAfterExpire, acl_claim_name := AclClaimName, from := From }) -> @@ -152,6 +157,7 @@ create2(#{ {ok, #{ jwk => JWK, verify_claims => VerifyClaims, + disconnect_after_expire => DisconnectAfterExpire, acl_claim_name => AclClaimName, from => From }}; @@ -159,6 +165,7 @@ create2( #{ use_jwks := true, verify_claims := VerifyClaims, + disconnect_after_expire := DisconnectAfterExpire, acl_claim_name := AclClaimName, from := From } = Config @@ -173,6 +180,7 @@ create2( {ok, #{ jwk_resource => ResourceId, verify_claims => VerifyClaims, + disconnect_after_expire => DisconnectAfterExpire, acl_claim_name => AclClaimName, from => From }}. @@ -211,23 +219,12 @@ render_expected([{Name, ExpectedTemplate} | More], Variables) -> Expected = emqx_authn_utils:render_str(ExpectedTemplate, Variables), [{Name, Expected} | render_expected(More, Variables)]. -verify(undefined, _, _, _) -> +verify(undefined, _, _, _, _) -> ignore; -verify(JWT, JWKs, VerifyClaims, AclClaimName) -> +verify(JWT, JWKs, VerifyClaims, AclClaimName, DisconnectAfterExpire) -> case do_verify(JWT, JWKs, VerifyClaims) of {ok, Extra} -> - IsSuperuser = emqx_authn_utils:is_superuser(Extra), - Attrs = emqx_authn_utils:client_attrs(Extra), - try - ACL = acl(Extra, AclClaimName), - Result = maps:merge(IsSuperuser, maps:merge(ACL, Attrs)), - {ok, Result} - catch - throw:{bad_acl_rule, Reason} -> - %% it's a invalid token, so ok to log - ?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{jwt => JWT}), - {error, bad_username_or_password} - end; + extra_to_auth_data(Extra, JWT, AclClaimName, DisconnectAfterExpire); {error, {missing_claim, Claim}} -> %% it's a invalid token, so it's ok to log ?TRACE_AUTHN_PROVIDER("missing_jwt_claim", #{jwt => JWT, claim => Claim}), @@ -242,6 +239,28 @@ verify(JWT, JWKs, VerifyClaims, AclClaimName) -> {error, bad_username_or_password} end. +extra_to_auth_data(Extra, JWT, AclClaimName, DisconnectAfterExpire) -> + IsSuperuser = emqx_authn_utils:is_superuser(Extra), + Attrs = emqx_authn_utils:client_attrs(Extra), + ExpireAt = expire_at(DisconnectAfterExpire, Extra), + try + ACL = acl(Extra, AclClaimName), + Result = merge_maps([ExpireAt, IsSuperuser, ACL, Attrs]), + {ok, Result} + catch + throw:{bad_acl_rule, Reason} -> + %% it's a invalid token, so ok to log + ?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{jwt => JWT}), + {error, bad_username_or_password} + end. + +expire_at(false, _Extra) -> + #{}; +expire_at(true, #{<<"exp">> := ExpireTime}) -> + #{expire_at => erlang:convert_time_unit(ExpireTime, second, millisecond)}; +expire_at(true, #{}) -> + #{}. + acl(Claims, AclClaimName) -> case Claims of #{AclClaimName := Rules} -> @@ -379,3 +398,6 @@ parse_rule(Rule) -> {error, Reason} -> throw({bad_acl_rule, Reason}) end. + +merge_maps([]) -> #{}; +merge_maps([Map | Maps]) -> maps:merge(Map, merge_maps(Maps)). diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl index f52992564..274f2b72b 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl @@ -127,6 +127,7 @@ common_fields() -> desc => ?DESC(acl_claim_name) }}, {verify_claims, fun verify_claims/1}, + {disconnect_after_expire, fun disconnect_after_expire/1}, {from, fun from/1} ] ++ emqx_authn_schema:common_fields(). @@ -188,6 +189,11 @@ verify_claims(required) -> verify_claims(_) -> undefined. +disconnect_after_expire(type) -> boolean(); +disconnect_after_expire(desc) -> ?DESC(?FUNCTION_NAME); +disconnect_after_expire(default) -> true; +disconnect_after_expire(_) -> undefined. + do_check_verify_claims([]) -> true; %% _Expected can't be invalid since tuples may come only from converter diff --git a/apps/emqx_auth_jwt/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_authn_jwt_SUITE.erl index a7eca75aa..89336e396 100644 --- a/apps/emqx_auth_jwt/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_authn_jwt_SUITE.erl @@ -55,7 +55,8 @@ t_hmac_based(_) -> algorithm => 'hmac-based', secret => Secret, secret_base64_encoded => false, - verify_claims => [{<<"username">>, <<"${username}">>}] + verify_claims => [{<<"username">>, <<"${username}">>}], + disconnect_after_expire => false }, {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config), @@ -179,7 +180,8 @@ t_public_key(_) -> use_jwks => false, algorithm => 'public-key', public_key => PublicKey, - verify_claims => [] + verify_claims => [], + disconnect_after_expire => false }, {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config), @@ -207,7 +209,8 @@ t_jwt_in_username(_) -> algorithm => 'hmac-based', secret => Secret, secret_base64_encoded => false, - verify_claims => [] + verify_claims => [], + disconnect_after_expire => false }, {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config), @@ -229,7 +232,8 @@ t_complex_template(_) -> algorithm => 'hmac-based', secret => Secret, secret_base64_encoded => false, - verify_claims => [{<<"id">>, <<"${username}-${clientid}">>}] + verify_claims => [{<<"id">>, <<"${username}-${clientid}">>}], + disconnect_after_expire => false }, {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config), @@ -269,7 +273,7 @@ t_jwks_renewal(_Config) -> algorithm => 'public-key', ssl => #{enable => false}, verify_claims => [], - + disconnect_after_expire => false, use_jwks => true, endpoint => "https://127.0.0.1:" ++ integer_to_list(?JWKS_PORT + 1) ++ ?JWKS_PATH, refresh_interval => 1000, @@ -366,7 +370,8 @@ t_verify_claims(_) -> algorithm => 'hmac-based', secret => Secret, secret_base64_encoded => false, - verify_claims => [{<<"foo">>, <<"bar">>}] + verify_claims => [{<<"foo">>, <<"bar">>}], + disconnect_after_expire => false }, {ok, State0} = emqx_authn_jwt:create(?AUTHN_ID, Config0), @@ -456,7 +461,8 @@ t_verify_claim_clientid(_) -> algorithm => 'hmac-based', secret => Secret, secret_base64_encoded => false, - verify_claims => [{<<"cl">>, <<"${clientid}">>}] + verify_claims => [{<<"cl">>, <<"${clientid}">>}], + disconnect_after_expire => false }, {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config), diff --git a/apps/emqx_auth_jwt/test/emqx_authn_jwt_expire_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_authn_jwt_expire_SUITE.erl new file mode 100644 index 000000000..91bd7189a --- /dev/null +++ b/apps/emqx_auth_jwt/test/emqx_authn_jwt_expire_SUITE.erl @@ -0,0 +1,96 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_jwt_expire_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_auth/include/emqx_authn.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(PATH, [authentication]). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_testcase(_, Config) -> + _ = emqx_authn_test_lib:delete_authenticators(?PATH, ?GLOBAL), + Config. + +end_per_testcase(_, _Config) -> + _ = emqx_authn_test_lib:delete_authenticators(?PATH, ?GLOBAL), + ok. + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_auth, emqx_auth_jwt], #{ + work_dir => ?config(priv_dir, Config) + }), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + emqx_authn_test_lib:delete_authenticators(?PATH, ?GLOBAL), + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. + +%%-------------------------------------------------------------------- +%% CT cases +%%-------------------------------------------------------------------- + +t_jwt_expire(_Config) -> + _ = process_flag(trap_exit, true), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, auth_config()} + ), + + {ok, [#{provider := emqx_authn_jwt}]} = emqx_authn_chains:list_authenticators(?GLOBAL), + + Expire = erlang:system_time(second) + 3, + + Payload = #{ + <<"username">> => <<"myuser">>, + <<"exp">> => Expire + }, + JWS = emqx_authn_jwt_SUITE:generate_jws('hmac-based', Payload, <<"secret">>), + + {ok, C} = emqtt:start_link([{username, <<"myuser">>}, {password, JWS}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C), + + receive + {disconnected, ?RC_NOT_AUTHORIZED, #{}} -> + ?assert(erlang:system_time(second) >= Expire) + after 5000 -> + ct:fail("Client should be disconnected by timeout") + end. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +auth_config() -> + #{ + <<"use_jwks">> => false, + <<"algorithm">> => <<"hmac-based">>, + <<"acl_claim_name">> => <<"acl">>, + <<"secret">> => <<"secret">>, + <<"mechanism">> => <<"jwt">>, + <<"verify_claims">> => #{<<"username">> => <<"${username}">>} + %% Should be enabled by default + %% <<"disconnect_after_expire">> => true + }. diff --git a/apps/emqx_auth_jwt/test/emqx_authz_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_authz_jwt_SUITE.erl index e7f78230a..813cb20e4 100644 --- a/apps/emqx_auth_jwt/test/emqx_authz_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_authz_jwt_SUITE.erl @@ -455,11 +455,12 @@ t_invalid_rule(_Config) -> authn_config() -> #{ <<"mechanism">> => <<"jwt">>, - <<"use_jwks">> => <<"false">>, + <<"use_jwks">> => false, <<"algorithm">> => <<"hmac-based">>, <<"secret">> => ?SECRET, - <<"secret_base64_encoded">> => <<"false">>, + <<"secret_base64_encoded">> => false, <<"acl_claim_name">> => <<"acl">>, + <<"disconnect_after_expire">> => false, <<"verify_claims">> => #{ <<"username">> => ?PH_USERNAME } diff --git a/apps/emqx_bridge/test/emqx_bridge_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_testlib.erl index 66fbb5d79..2803c9e53 100644 --- a/apps/emqx_bridge/test/emqx_bridge_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_testlib.erl @@ -252,11 +252,14 @@ create_rule_and_action_http(BridgeType, RuleTopic, Config) -> create_rule_and_action_http(BridgeType, RuleTopic, Config, Opts) -> BridgeName = ?config(bridge_name, Config), BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), + create_rule_and_action(BridgeId, RuleTopic, Opts). + +create_rule_and_action(Action, RuleTopic, Opts) -> SQL = maps:get(sql, Opts, <<"SELECT * FROM \"", RuleTopic/binary, "\"">>), Params = #{ enable => true, sql => SQL, - actions => [BridgeId] + actions => [Action] }, Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index ef79f78fe..87da71449 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -29,7 +29,8 @@ on_query_async/4, on_batch_query/3, on_batch_query_async/4, - on_get_status/2 + on_get_status/2, + on_format_query_result/1 ]). %% callbacks of ecpool @@ -459,6 +460,11 @@ handle_result({error, Error}) -> handle_result(Res) -> Res. +on_format_query_result({ok, Result}) -> + #{result => ok, info => Result}; +on_format_query_result(Result) -> + Result. + %%-------------------------------------------------------------------- %% utils diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index 2c824aa95..f6888cad5 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -38,7 +38,8 @@ on_get_channels/1, on_query/3, on_batch_query/3, - on_get_status/2 + on_get_status/2, + on_format_query_result/1 ]). %% callbacks for ecpool @@ -519,6 +520,13 @@ transform_and_log_clickhouse_result(ClickhouseErrorResult, ResourceID, SQL) -> to_error_tuple(ClickhouseErrorResult) end. +on_format_query_result(ok) -> + #{result => ok, message => <<"">>}; +on_format_query_result({ok, Message}) -> + #{result => ok, message => Message}; +on_format_query_result(Result) -> + Result. + to_recoverable_error({error, Reason}) -> {error, {recoverable_error, Reason}}; to_recoverable_error(Error) -> diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index eb2df7755..82f5fb18d 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -26,7 +26,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). -export([ @@ -184,6 +185,11 @@ on_batch_query(InstanceId, [{_ChannelId, _} | _] = Query, State) -> on_batch_query(_InstanceId, Query, _State) -> {error, {unrecoverable_error, {invalid_request, Query}}}. +on_format_query_result({ok, Result}) -> + #{result => ok, info => Result}; +on_format_query_result(Result) -> + Result. + health_check_timeout() -> 2500. diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl index 08de50963..0613ca3bb 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl @@ -27,6 +27,8 @@ -export([execute/2]). -endif. +-include_lib("emqx/include/emqx_trace.hrl"). + %%%=================================================================== %%% API %%%=================================================================== @@ -107,7 +109,10 @@ do_query(Table, Query0, Templates, TraceRenderedCTX) -> Query = apply_template(Query0, Templates), emqx_trace:rendered_action_template_with_ctx(TraceRenderedCTX, #{ table => Table, - query => {fun trace_format_query/1, Query} + query => #emqx_trace_format_func_data{ + function = fun trace_format_query/1, + data = Query + } }), execute(Query, Table) catch diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index c47176859..20de92e6e 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -23,7 +23,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). -export([ @@ -286,6 +287,9 @@ on_query_async( InstanceId, {ChannelId, Msg}, ReplyFunAndArgs, State ). +on_format_query_result(Result) -> + emqx_bridge_http_connector:on_format_query_result(Result). + on_add_channel( InstanceId, #{channels := Channels} = State0, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl index 12d5d1f2f..48e50c416 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl @@ -53,7 +53,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). -export([reply_delegator/2]). @@ -489,6 +490,11 @@ handle_result({error, Reason} = Result, _Request, QueryMode, ResourceId) -> handle_result({ok, _} = Result, _Request, _QueryMode, _ResourceId) -> Result. +on_format_query_result({ok, Info}) -> + #{result => ok, info => Info}; +on_format_query_result(Result) -> + Result. + reply_delegator(ReplyFunAndArgs, Response) -> case Response of {error, Reason} when diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 97eedf3f6..963f0efd0 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -27,7 +27,8 @@ on_batch_query/3, on_query_async/4, on_batch_query_async/4, - on_get_status/2 + on_get_status/2, + on_format_query_result/1 ]). -export([reply_callback/2]). @@ -453,6 +454,11 @@ do_query(InstId, Channel, Client, Points) -> end end. +on_format_query_result({ok, {affected_rows, Rows}}) -> + #{result => ok, affected_rows => Rows}; +on_format_query_result(Result) -> + Result. + do_async_query(InstId, Channel, Client, Points, ReplyFunAndArgs) -> ?SLOG(info, #{ msg => "greptimedb_write_point_async", diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 6de4dffe6..36dfc0468 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -20,6 +20,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -behaviour(emqx_resource). @@ -35,7 +36,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). -export([reply_delegator/3]). @@ -231,6 +233,7 @@ on_start( host => Host, port => Port, connect_timeout => ConnectTimeout, + scheme => Scheme, request => preprocess_request(maps:get(request, Config, undefined)) }, case start_pool(InstId, PoolOpts) of @@ -358,7 +361,7 @@ on_query(InstId, {Method, Request, Timeout}, State) -> on_query( InstId, {ActionId, KeyOrNum, Method, Request, Timeout, Retry}, - #{host := Host} = State + #{host := Host, port := Port, scheme := Scheme} = State ) -> ?TRACE( "QUERY", @@ -372,7 +375,7 @@ on_query( } ), NRequest = formalize_request(Method, Request), - trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout), + trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout), Worker = resolve_pool_worker(State, KeyOrNum), Result0 = ehttpc:request( Worker, @@ -468,7 +471,7 @@ on_query_async( InstId, {ActionId, KeyOrNum, Method, Request, Timeout}, ReplyFunAndArgs, - #{host := Host} = State + #{host := Host, port := Port, scheme := Scheme} = State ) -> Worker = resolve_pool_worker(State, KeyOrNum), ?TRACE( @@ -482,7 +485,7 @@ on_query_async( } ), NRequest = formalize_request(Method, Request), - trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout), + trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout), MaxAttempts = maps:get(max_attempts, State, 3), Context = #{ attempt => 1, @@ -491,7 +494,8 @@ on_query_async( key_or_num => KeyOrNum, method => Method, request => NRequest, - timeout => Timeout + timeout => Timeout, + trace_metadata => logger:get_process_metadata() }, ok = ehttpc:request_async( Worker, @@ -502,17 +506,25 @@ on_query_async( ), {ok, Worker}. -trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) -> +trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout) -> case NRequest of {Path, Headers} -> emqx_trace:rendered_action_template( ActionId, #{ host => Host, + port => Port, path => Path, method => Method, - headers => {fun emqx_utils_redact:redact_headers/1, Headers}, - timeout => Timeout + headers => #emqx_trace_format_func_data{ + function = fun emqx_utils_redact:redact_headers/1, + data = Headers + }, + timeout => Timeout, + url => #emqx_trace_format_func_data{ + function = fun render_url/1, + data = {Scheme, Host, Port, Path} + } } ); {Path, Headers, Body} -> @@ -520,15 +532,42 @@ trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) -> ActionId, #{ host => Host, + port => Port, path => Path, method => Method, - headers => {fun emqx_utils_redact:redact_headers/1, Headers}, + headers => #emqx_trace_format_func_data{ + function = fun emqx_utils_redact:redact_headers/1, + data = Headers + }, timeout => Timeout, - body => {fun log_format_body/1, Body} + body => #emqx_trace_format_func_data{ + function = fun log_format_body/1, + data = Body + }, + url => #emqx_trace_format_func_data{ + function = fun render_url/1, + data = {Scheme, Host, Port, Path} + } } ) end. +render_url({Scheme, Host, Port, Path}) -> + SchemeStr = + case Scheme of + http -> + <<"http://">>; + https -> + <<"https://">> + end, + unicode:characters_to_binary([ + SchemeStr, + Host, + <<":">>, + erlang:integer_to_binary(Port), + Path + ]). + log_format_body(Body) -> unicode:characters_to_binary(Body). @@ -604,6 +643,26 @@ on_get_channel_status( %% XXX: Reuse the connector status on_get_status(InstId, State). +on_format_query_result({ok, Status, Headers, Body}) -> + #{ + result => ok, + response => #{ + status => Status, + headers => Headers, + body => Body + } + }; +on_format_query_result({ok, Status, Headers}) -> + #{ + result => ok, + response => #{ + status => Status, + headers => Headers + } + }; +on_format_query_result(Result) -> + Result. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- @@ -809,9 +868,15 @@ to_bin(Str) when is_list(Str) -> to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). -reply_delegator(Context, ReplyFunAndArgs, Result0) -> +reply_delegator( + #{trace_metadata := TraceMetadata} = Context, + ReplyFunAndArgs, + Result0 +) -> spawn(fun() -> + logger:set_process_metadata(TraceMetadata), Result = transform_result(Result0), + logger:unset_process_metadata(), maybe_retry(Result, Context, ReplyFunAndArgs) end). diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index f239d3735..88065b7b3 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -27,7 +27,8 @@ on_batch_query/3, on_query_async/4, on_batch_query_async/4, - on_get_status/2 + on_get_status/2, + on_format_query_result/1 ]). -export([reply_callback/2]). @@ -209,6 +210,9 @@ on_batch_query_async( {error, {unrecoverable_error, Reason}} end. +on_format_query_result(Result) -> + emqx_bridge_http_connector:on_format_query_result(Result). + on_get_status(_InstId, #{client := Client}) -> case influxdb:is_alive(Client) andalso ok =:= influxdb:check_auth(Client) of true -> diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index a3e9b5687..78866ef79 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -26,7 +26,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). -export([ @@ -388,6 +389,9 @@ on_batch_query( Error end. +on_format_query_result(Result) -> + emqx_bridge_http_connector:on_format_query_result(Result). + on_add_channel( InstanceId, #{iotdb_version := Version, channels := Channels} = OldState0, diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl index 8744dfd71..95d193d92 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl @@ -39,7 +39,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). -export([ @@ -318,6 +319,11 @@ handle_result({error, Reason} = Error, Requests, InstanceId) -> }), Error. +on_format_query_result({ok, Result}) -> + #{result => ok, info => Result}; +on_format_query_result(Result) -> + Result. + parse_template(Config) -> #{payload_template := PayloadTemplate, partition_key := PartitionKeyTemplate} = Config, Templates = #{send_message => PayloadTemplate, partition_key => PartitionKeyTemplate}, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl index 69c2242e4..6b6db358a 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl @@ -18,7 +18,8 @@ on_get_status/2, on_query/3, on_start/2, - on_stop/2 + on_stop/2, + on_format_query_result/1 ]). %%======================================================================================== @@ -85,6 +86,11 @@ on_query(InstanceId, {Channel, Message0}, #{channels := Channels, connector_stat on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) -> emqx_mongodb:on_query(InstanceId, Request, ConnectorState). +on_format_query_result({{Result, Info}, Documents}) -> + #{result => Result, info => Info, documents => Documents}; +on_format_query_result(Result) -> + Result. + on_remove_channel(_InstanceId, #{channels := Channels} = State, ChannelId) -> NewState = State#{channels => maps:remove(ChannelId, Channels)}, {ok, NewState}. diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index 509d53284..19e117a0d 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -27,7 +27,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). -export([connector_examples/1]). @@ -175,6 +176,11 @@ on_batch_query( Error end. +on_format_query_result({ok, StatusCode, BodyMap}) -> + #{result => ok, status_code => StatusCode, body => BodyMap}; +on_format_query_result(Result) -> + Result. + on_get_status(_InstanceId, #{server := Server}) -> Result = case opentsdb_connectivity(Server) of diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index 0cddfab66..9d269493d 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -20,7 +20,8 @@ on_get_status/2, on_get_channel_status/3, on_query/3, - on_query_async/4 + on_query_async/4, + on_format_query_result/1 ]). -type pulsar_client_id() :: atom(). @@ -234,6 +235,11 @@ on_query_async2(ChannelId, Producers, Message, MessageTmpl, AsyncReplyFn) -> }), pulsar:send(Producers, [PulsarMessage], #{callback_fn => AsyncReplyFn}). +on_format_query_result({ok, Info}) -> + #{result => ok, info => Info}; +on_format_query_result(Result) -> + Result. + %%------------------------------------------------------------------------------------- %% Internal fns %%------------------------------------------------------------------------------------- diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl index 3f7c4897c..28f7c6f8e 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl @@ -20,7 +20,8 @@ on_query/3, on_batch_query/3, on_get_status/2, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). %% ------------------------------------------------------------------------------------------------- @@ -161,6 +162,11 @@ on_batch_query( Error end. +on_format_query_result({ok, Msg}) -> + #{result => ok, message => Msg}; +on_format_query_result(Res) -> + Res. + %% ------------------------------------------------------------------------------------------------- %% private helpers %% ------------------------------------------------------------------------------------------------- diff --git a/apps/emqx_bridge_s3/rebar.config b/apps/emqx_bridge_s3/rebar.config index 51bf0e0b6..34f7bb51e 100644 --- a/apps/emqx_bridge_s3/rebar.config +++ b/apps/emqx_bridge_s3/rebar.config @@ -2,5 +2,6 @@ {erl_opts, [debug_info]}. {deps, [ - {emqx_resource, {path, "../../apps/emqx_resource"}} + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_connector_aggregator, {path, "../../apps/emqx_connector_aggregator"}} ]}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src b/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src index 5cdf3fb82..46f8db64b 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src @@ -7,6 +7,7 @@ stdlib, erlcloud, emqx_resource, + emqx_connector_aggregator, emqx_s3 ]}, {env, [ @@ -18,7 +19,6 @@ emqx_bridge_s3_connector_info ]} ]}, - {mod, {emqx_bridge_s3_app, []}}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl deleted file mode 100644 index 02099dbec..000000000 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl +++ /dev/null @@ -1,212 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - -%% This module takes aggregated records from a buffer and delivers them to S3, -%% wrapped in a configurable container (though currently there's only CSV). --module(emqx_bridge_s3_aggreg_delivery). - --include_lib("snabbkaffe/include/trace.hrl"). --include("emqx_bridge_s3_aggregator.hrl"). - --export([start_link/3]). - -%% Internal exports --export([ - init/4, - loop/3 -]). - --behaviour(emqx_template). --export([lookup/2]). - -%% Sys --export([ - system_continue/3, - system_terminate/4, - format_status/2 -]). - --record(delivery, { - name :: _Name, - container :: emqx_bridge_s3_aggreg_csv:container(), - reader :: emqx_bridge_s3_aggreg_buffer:reader(), - upload :: emqx_s3_upload:t(), - empty :: boolean() -}). - --type state() :: #delivery{}. - -%% - -start_link(Name, Buffer, Opts) -> - proc_lib:start_link(?MODULE, init, [self(), Name, Buffer, Opts]). - -%% - --spec init(pid(), _Name, buffer(), _Opts :: map()) -> no_return(). -init(Parent, Name, Buffer, Opts) -> - ?tp(s3_aggreg_delivery_started, #{action => Name, buffer => Buffer}), - Reader = open_buffer(Buffer), - Delivery = init_delivery(Name, Reader, Buffer, Opts#{action => Name}), - _ = erlang:process_flag(trap_exit, true), - ok = proc_lib:init_ack({ok, self()}), - loop(Delivery, Parent, []). - -init_delivery(Name, Reader, Buffer, Opts = #{container := ContainerOpts}) -> - #delivery{ - name = Name, - container = mk_container(ContainerOpts), - reader = Reader, - upload = mk_upload(Buffer, Opts), - empty = true - }. - -open_buffer(#buffer{filename = Filename}) -> - case file:open(Filename, [read, binary, raw]) of - {ok, FD} -> - {_Meta, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(FD), - Reader; - {error, Reason} -> - error({buffer_open_failed, Reason}) - end. - -mk_container(#{type := csv, column_order := OrderOpt}) -> - %% TODO: Deduplicate? - ColumnOrder = lists:map(fun emqx_utils_conv:bin/1, OrderOpt), - emqx_bridge_s3_aggreg_csv:new(#{column_order => ColumnOrder}). - -mk_upload( - Buffer, - Opts = #{ - bucket := Bucket, - upload_options := UploadOpts, - client_config := Config, - uploader_config := UploaderConfig - } -) -> - Client = emqx_s3_client:create(Bucket, Config), - Key = mk_object_key(Buffer, Opts), - emqx_s3_upload:new(Client, Key, UploadOpts, UploaderConfig). - -mk_object_key(Buffer, #{action := Name, key := Template}) -> - emqx_template:render_strict(Template, {?MODULE, {Name, Buffer}}). - -%% - --spec loop(state(), pid(), [sys:debug_option()]) -> no_return(). -loop(Delivery, Parent, Debug) -> - %% NOTE: This function is mocked in tests. - receive - Msg -> handle_msg(Msg, Delivery, Parent, Debug) - after 0 -> - process_delivery(Delivery, Parent, Debug) - end. - -process_delivery(Delivery0 = #delivery{reader = Reader0}, Parent, Debug) -> - case emqx_bridge_s3_aggreg_buffer:read(Reader0) of - {Records = [#{} | _], Reader} -> - Delivery1 = Delivery0#delivery{reader = Reader}, - Delivery2 = process_append_records(Records, Delivery1), - Delivery = process_write(Delivery2), - loop(Delivery, Parent, Debug); - {[], Reader} -> - Delivery = Delivery0#delivery{reader = Reader}, - loop(Delivery, Parent, Debug); - eof -> - process_complete(Delivery0); - {Unexpected, _Reader} -> - exit({buffer_unexpected_record, Unexpected}) - end. - -process_append_records(Records, Delivery = #delivery{container = Container0, upload = Upload0}) -> - {Writes, Container} = emqx_bridge_s3_aggreg_csv:fill(Records, Container0), - {ok, Upload} = emqx_s3_upload:append(Writes, Upload0), - Delivery#delivery{ - container = Container, - upload = Upload, - empty = false - }. - -process_write(Delivery = #delivery{upload = Upload0}) -> - case emqx_s3_upload:write(Upload0) of - {ok, Upload} -> - Delivery#delivery{upload = Upload}; - {cont, Upload} -> - process_write(Delivery#delivery{upload = Upload}); - {error, Reason} -> - _ = emqx_s3_upload:abort(Upload0), - exit({upload_failed, Reason}) - end. - -process_complete(#delivery{name = Name, empty = true}) -> - ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => empty}), - exit({shutdown, {skipped, empty}}); -process_complete(#delivery{name = Name, container = Container, upload = Upload0}) -> - Trailer = emqx_bridge_s3_aggreg_csv:close(Container), - {ok, Upload} = emqx_s3_upload:append(Trailer, Upload0), - case emqx_s3_upload:complete(Upload) of - {ok, Completed} -> - ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => Completed}), - ok; - {error, Reason} -> - _ = emqx_s3_upload:abort(Upload), - exit({upload_failed, Reason}) - end. - -%% - -handle_msg({system, From, Msg}, Delivery, Parent, Debug) -> - sys:handle_system_msg(Msg, From, Parent, ?MODULE, Debug, Delivery); -handle_msg({'EXIT', Parent, Reason}, Delivery, Parent, Debug) -> - system_terminate(Reason, Parent, Debug, Delivery); -handle_msg(_Msg, Delivery, Parent, Debug) -> - loop(Parent, Debug, Delivery). - --spec system_continue(pid(), [sys:debug_option()], state()) -> no_return(). -system_continue(Parent, Debug, Delivery) -> - loop(Delivery, Parent, Debug). - --spec system_terminate(_Reason, pid(), [sys:debug_option()], state()) -> _. -system_terminate(_Reason, _Parent, _Debug, #delivery{upload = Upload}) -> - emqx_s3_upload:abort(Upload). - --spec format_status(normal, Args :: [term()]) -> _StateFormatted. -format_status(_Normal, [_PDict, _SysState, _Parent, _Debug, Delivery]) -> - Delivery#delivery{ - upload = emqx_s3_upload:format(Delivery#delivery.upload) - }. - -%% - --spec lookup(emqx_template:accessor(), {_Name, buffer()}) -> - {ok, integer() | string()} | {error, undefined}. -lookup([<<"action">>], {Name, _Buffer}) -> - {ok, mk_fs_safe_string(Name)}; -lookup(Accessor, {_Name, Buffer = #buffer{}}) -> - lookup_buffer_var(Accessor, Buffer); -lookup(_Accessor, _Context) -> - {error, undefined}. - -lookup_buffer_var([<<"datetime">>, Format], #buffer{since = Since}) -> - {ok, format_timestamp(Since, Format)}; -lookup_buffer_var([<<"datetime_until">>, Format], #buffer{until = Until}) -> - {ok, format_timestamp(Until, Format)}; -lookup_buffer_var([<<"sequence">>], #buffer{seq = Seq}) -> - {ok, Seq}; -lookup_buffer_var([<<"node">>], #buffer{}) -> - {ok, mk_fs_safe_string(atom_to_binary(erlang:node()))}; -lookup_buffer_var(_Binding, _Context) -> - {error, undefined}. - -format_timestamp(Timestamp, <<"rfc3339utc">>) -> - String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}, {offset, "Z"}]), - mk_fs_safe_string(String); -format_timestamp(Timestamp, <<"rfc3339">>) -> - String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}]), - mk_fs_safe_string(String); -format_timestamp(Timestamp, <<"unix">>) -> - Timestamp. - -mk_fs_safe_string(String) -> - unicode:characters_to_binary(string:replace(String, ":", "_", all)). diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl deleted file mode 100644 index e5b77f9d6..000000000 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl +++ /dev/null @@ -1,16 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_bridge_s3_app). - --behaviour(application). --export([start/2, stop/1]). - -%% - -start(_StartType, _StartArgs) -> - emqx_bridge_s3_sup:start_link(). - -stop(_State) -> - ok. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl index d135a087a..f9d3af478 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl @@ -7,6 +7,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). +-include_lib("emqx_connector_aggregator/include/emqx_connector_aggregator.hrl"). -include("emqx_bridge_s3.hrl"). -behaviour(emqx_resource). @@ -23,6 +25,19 @@ on_get_channel_status/3 ]). +-behaviour(emqx_connector_aggreg_delivery). +-export([ + init_transfer_state/2, + process_append/2, + process_write/1, + process_complete/1, + process_format_status/1, + process_terminate/1 +]). + +-behaviour(emqx_template). +-export([lookup/2]). + -type config() :: #{ access_key_id => string(), secret_access_key => emqx_secret:t(string()), @@ -204,13 +219,14 @@ start_channel(State, #{ key => emqx_bridge_s3_aggreg_upload:mk_key_template(Parameters), container => Container, upload_options => emqx_bridge_s3_aggreg_upload:mk_upload_options(Parameters), + callback_module => ?MODULE, client_config => maps:get(client_config, State), uploader_config => maps:with([min_part_size, max_part_size], Parameters) }, - _ = emqx_bridge_s3_sup:delete_child({Type, Name}), - {ok, SupPid} = emqx_bridge_s3_sup:start_child(#{ + _ = emqx_connector_aggreg_sup:delete_child({Type, Name}), + {ok, SupPid} = emqx_connector_aggreg_sup:start_child(#{ id => {Type, Name}, - start => {emqx_bridge_s3_aggreg_upload_sup, start_link, [Name, AggregOpts, DeliveryOpts]}, + start => {emqx_connector_aggreg_upload_sup, start_link, [Name, AggregOpts, DeliveryOpts]}, type => supervisor, restart => permanent }), @@ -219,7 +235,7 @@ start_channel(State, #{ name => Name, bucket => Bucket, supervisor => SupPid, - on_stop => fun() -> emqx_bridge_s3_sup:delete_child({Type, Name}) end + on_stop => fun() -> emqx_connector_aggreg_sup:delete_child({Type, Name}) end }. upload_options(Parameters) -> @@ -241,7 +257,7 @@ channel_status(#{type := ?ACTION_UPLOAD}, _State) -> channel_status(#{type := ?ACTION_AGGREGATED_UPLOAD, name := Name, bucket := Bucket}, State) -> %% NOTE: This will effectively trigger uploads of buffers yet to be uploaded. Timestamp = erlang:system_time(second), - ok = emqx_bridge_s3_aggregator:tick(Name, Timestamp), + ok = emqx_connector_aggregator:tick(Name, Timestamp), ok = check_bucket_accessible(Bucket, State), ok = check_aggreg_upload_errors(Name), ?status_connected. @@ -263,7 +279,7 @@ check_bucket_accessible(Bucket, #{client_config := Config}) -> end. check_aggreg_upload_errors(Name) -> - case emqx_bridge_s3_aggregator:take_error(Name) of + case emqx_connector_aggregator:take_error(Name) of [Error] -> %% TODO %% This approach means that, for example, 3 upload failures will cause @@ -320,7 +336,10 @@ run_simple_upload( emqx_trace:rendered_action_template(ChannelID, #{ bucket => Bucket, key => Key, - content => Content + content => #emqx_trace_format_func_data{ + function = fun unicode:characters_to_binary/1, + data = Content + } }), case emqx_s3_client:put_object(Client, Key, UploadOpts, Content) of ok -> @@ -336,7 +355,7 @@ run_simple_upload( run_aggregated_upload(InstId, Records, #{name := Name}) -> Timestamp = erlang:system_time(second), - case emqx_bridge_s3_aggregator:push_records(Name, Timestamp, Records) of + case emqx_connector_aggregator:push_records(Name, Timestamp, Records) of ok -> ?tp(s3_bridge_aggreg_push_ok, #{instance_id => InstId, name => Name}), ok; @@ -372,3 +391,84 @@ render_content(Template, Data) -> iolist_to_string(IOList) -> unicode:characters_to_list(IOList). + +%% `emqx_connector_aggreg_delivery` APIs + +-spec init_transfer_state(buffer_map(), map()) -> emqx_s3_upload:t(). +init_transfer_state(BufferMap, Opts) -> + #{ + bucket := Bucket, + upload_options := UploadOpts, + client_config := Config, + uploader_config := UploaderConfig + } = Opts, + Client = emqx_s3_client:create(Bucket, Config), + Key = mk_object_key(BufferMap, Opts), + emqx_s3_upload:new(Client, Key, UploadOpts, UploaderConfig). + +mk_object_key(BufferMap, #{action := Name, key := Template}) -> + emqx_template:render_strict(Template, {?MODULE, {Name, BufferMap}}). + +process_append(Writes, Upload0) -> + {ok, Upload} = emqx_s3_upload:append(Writes, Upload0), + Upload. + +process_write(Upload0) -> + case emqx_s3_upload:write(Upload0) of + {ok, Upload} -> + {ok, Upload}; + {cont, Upload} -> + process_write(Upload); + {error, Reason} -> + _ = emqx_s3_upload:abort(Upload0), + {error, Reason} + end. + +process_complete(Upload) -> + case emqx_s3_upload:complete(Upload) of + {ok, Completed} -> + {ok, Completed}; + {error, Reason} -> + _ = emqx_s3_upload:abort(Upload), + exit({upload_failed, Reason}) + end. + +process_format_status(Upload) -> + emqx_s3_upload:format(Upload). + +process_terminate(Upload) -> + emqx_s3_upload:abort(Upload). + +%% `emqx_template` APIs + +-spec lookup(emqx_template:accessor(), {_Name, buffer_map()}) -> + {ok, integer() | string()} | {error, undefined}. +lookup([<<"action">>], {Name, _Buffer}) -> + {ok, mk_fs_safe_string(Name)}; +lookup(Accessor, {_Name, Buffer = #{}}) -> + lookup_buffer_var(Accessor, Buffer); +lookup(_Accessor, _Context) -> + {error, undefined}. + +lookup_buffer_var([<<"datetime">>, Format], #{since := Since}) -> + {ok, format_timestamp(Since, Format)}; +lookup_buffer_var([<<"datetime_until">>, Format], #{until := Until}) -> + {ok, format_timestamp(Until, Format)}; +lookup_buffer_var([<<"sequence">>], #{seq := Seq}) -> + {ok, Seq}; +lookup_buffer_var([<<"node">>], #{}) -> + {ok, mk_fs_safe_string(atom_to_binary(erlang:node()))}; +lookup_buffer_var(_Binding, _Context) -> + {error, undefined}. + +format_timestamp(Timestamp, <<"rfc3339utc">>) -> + String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}, {offset, "Z"}]), + mk_fs_safe_string(String); +format_timestamp(Timestamp, <<"rfc3339">>) -> + String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}]), + mk_fs_safe_string(String); +format_timestamp(Timestamp, <<"unix">>) -> + Timestamp. + +mk_fs_safe_string(String) -> + unicode:characters_to_binary(string:replace(String, ":", "_", all)). diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl index 6577b45ed..0ae34486f 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl @@ -170,7 +170,7 @@ t_aggreg_upload(Config) -> ]), ok = send_messages(BridgeName, MessageEvents), %% Wait until the delivery is completed. - ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}), %% Check the uploaded objects. _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), ?assertMatch( @@ -217,7 +217,7 @@ t_aggreg_upload_rule(Config) -> emqx_message:make(?FUNCTION_NAME, T3 = <<"s3/empty">>, P3 = <<>>), emqx_message:make(?FUNCTION_NAME, <<"not/s3">>, <<"should not be here">>) ]), - ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}), %% Check the uploaded objects. _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), _CSV = [Header | Rows] = fetch_parse_csv(Bucket, Key), @@ -258,15 +258,15 @@ t_aggreg_upload_restart(Config) -> {<<"C3">>, T3 = <<"t/42">>, P3 = <<"">>} ]), ok = send_messages(BridgeName, MessageEvents), - {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}), + {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_records_written, action := BridgeName}), %% Restart the bridge. {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName), {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName), %% Send some more messages. ok = send_messages(BridgeName, MessageEvents), - {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}), + {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_records_written, action := BridgeName}), %% Wait until the delivery is completed. - {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}), %% Check there's still only one upload. _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), _Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), @@ -300,18 +300,18 @@ t_aggreg_upload_restart_corrupted(Config) -> %% Ensure that they span multiple batch queries. ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages1), 1), {ok, _} = ?block_until( - #{?snk_kind := s3_aggreg_records_written, action := BridgeName}, + #{?snk_kind := connector_aggreg_records_written, action := BridgeName}, infinity, 0 ), %% Find out the buffer file. {ok, #{filename := Filename}} = ?block_until( - #{?snk_kind := s3_aggreg_buffer_allocated, action := BridgeName} + #{?snk_kind := connector_aggreg_buffer_allocated, action := BridgeName} ), %% Stop the bridge, corrupt the buffer file, and restart the bridge. {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName), BufferFileSize = filelib:file_size(Filename), - ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, BufferFileSize div 2), + ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, BufferFileSize div 2), {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName), %% Send some more messages. Messages2 = [ @@ -320,7 +320,7 @@ t_aggreg_upload_restart_corrupted(Config) -> ], ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages2), 0), %% Wait until the delivery is completed. - {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}), %% Check that upload contains part of the first batch and all of the second batch. _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), CSV = [_Header | Rows] = fetch_parse_csv(Bucket, Key), @@ -362,7 +362,7 @@ t_aggreg_pending_upload_restart(Config) -> %% Restart the bridge. {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName), %% Wait until the delivery is completed. - {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}), %% Check that delivery contains all the messages. _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), [_Header | Rows] = fetch_parse_csv(Bucket, Key), @@ -392,7 +392,9 @@ t_aggreg_next_rotate(Config) -> NSent = receive_sender_reports(Senders), %% Wait for the last delivery to complete. ok = timer:sleep(round(?CONF_TIME_INTERVAL * 0.5)), - ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}, infinity, 0), + ?block_until( + #{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}, infinity, 0 + ), %% There should be at least 2 time windows of aggregated records. Uploads = [K || #{key := K} <- emqx_bridge_s3_test_helpers:list_objects(Bucket)], DTs = [DT || K <- Uploads, [_Action, _Node, DT | _] <- [string:split(K, "/", all)]], diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl index 21729369b..5ebeaf681 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl @@ -48,11 +48,3 @@ list_pending_uploads(Bucket, Key) -> {ok, Props} = erlcloud_s3:list_multipart_uploads(Bucket, [{prefix, Key}], [], AwsConfig), Uploads = proplists:get_value(uploads, Props), lists:map(fun maps:from_list/1, Uploads). - -%% File utilities - -truncate_at(Filename, Pos) -> - {ok, FD} = file:open(Filename, [read, write, binary]), - {ok, Pos} = file:position(FD, Pos), - ok = file:truncate(FD), - ok = file:close(FD). diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl index 683551316..726d2656a 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl @@ -39,7 +39,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). %% callbacks for ecpool @@ -320,6 +321,11 @@ on_batch_query(ResourceId, BatchRequests, State) -> ), do_query(ResourceId, BatchRequests, ?SYNC_QUERY_MODE, State). +on_format_query_result({ok, Rows}) -> + #{result => ok, rows => Rows}; +on_format_query_result(Result) -> + Result. + on_get_status(_InstanceId, #{pool_name := PoolName} = _State) -> Health = emqx_resource_pool:health_check_workers( PoolName, diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 67b0e77bc..324694edc 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -28,7 +28,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). -export([connector_examples/1]). @@ -215,6 +216,11 @@ on_batch_query(InstanceId, BatchReq, State) -> ?SLOG(error, LogMeta#{msg => "invalid_request"}), {error, {unrecoverable_error, invalid_request}}. +on_format_query_result({ok, ResultMap}) -> + #{result => ok, info => ResultMap}; +on_format_query_result(Result) -> + Result. + on_get_status(_InstanceId, #{pool_name := PoolName} = State) -> case emqx_resource_pool:health_check_workers( diff --git a/apps/emqx_connector_aggregator/BSL.txt b/apps/emqx_connector_aggregator/BSL.txt new file mode 100644 index 000000000..608706754 --- /dev/null +++ b/apps/emqx_connector_aggregator/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2024 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2028-05-06 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_connector_aggregator/README.md b/apps/emqx_connector_aggregator/README.md new file mode 100644 index 000000000..e495cc651 --- /dev/null +++ b/apps/emqx_connector_aggregator/README.md @@ -0,0 +1,11 @@ +# EMQX Connector Aggregator + +This application provides common logic for connector and action implementations of EMQX to aggregate multiple incoming messsages into a container file before sending it to a blob storage backend. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +## License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl b/apps/emqx_connector_aggregator/include/emqx_connector_aggregator.hrl similarity index 57% rename from apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl rename to apps/emqx_connector_aggregator/include/emqx_connector_aggregator.hrl index 7ac62c6b5..07d4a414f 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl +++ b/apps/emqx_connector_aggregator/include/emqx_connector_aggregator.hrl @@ -3,8 +3,8 @@ %%-------------------------------------------------------------------- -record(buffer, { - since :: emqx_bridge_s3_aggregator:timestamp(), - until :: emqx_bridge_s3_aggregator:timestamp(), + since :: emqx_connector_aggregator:timestamp(), + until :: emqx_connector_aggregator:timestamp(), seq :: non_neg_integer(), filename :: file:filename(), fd :: file:io_device() | undefined, @@ -13,3 +13,11 @@ }). -type buffer() :: #buffer{}. + +-type buffer_map() :: #{ + since := emqx_connector_aggregator:timestamp(), + until := emqx_connector_aggregator:timestamp(), + seq := non_neg_integer(), + filename := file:filename(), + max_records := pos_integer() | undefined +}. diff --git a/apps/emqx_connector_aggregator/rebar.config b/apps/emqx_connector_aggregator/rebar.config new file mode 100644 index 000000000..592bd4e41 --- /dev/null +++ b/apps/emqx_connector_aggregator/rebar.config @@ -0,0 +1,7 @@ +%% -*- mode: erlang; -*- + +{deps, [ + {emqx, {path, "../../apps/emqx"}} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_app.erl b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_app.erl new file mode 100644 index 000000000..ce6e11ab2 --- /dev/null +++ b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_app.erl @@ -0,0 +1,25 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_connector_aggreg_app). + +-behaviour(application). +-export([start/2, stop/1]). + +%%------------------------------------------------------------------------------ +%% Type declarations +%%------------------------------------------------------------------------------ + +%%------------------------------------------------------------------------------ +%% `application` API +%%------------------------------------------------------------------------------ + +start(_StartType, _StartArgs) -> + emqx_connector_aggreg_sup:start_link(). + +stop(_State) -> + ok. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_buffer.erl similarity index 99% rename from apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl rename to apps/emqx_connector_aggregator/src/emqx_connector_aggreg_buffer.erl index 503b864a7..3d0aa3157 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl +++ b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_buffer.erl @@ -17,7 +17,7 @@ %% ... %% ``` %% ^ ETF = Erlang External Term Format (i.e. `erlang:term_to_binary/1`). --module(emqx_bridge_s3_aggreg_buffer). +-module(emqx_connector_aggreg_buffer). -export([ new_writer/2, diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_csv.erl similarity index 95% rename from apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl rename to apps/emqx_connector_aggregator/src/emqx_connector_aggreg_csv.erl index 33895d8c1..bfca6f336 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl +++ b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_csv.erl @@ -2,8 +2,8 @@ %% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -%% CSV container implementation for `emqx_bridge_s3_aggregator`. --module(emqx_bridge_s3_aggreg_csv). +%% CSV container implementation for `emqx_connector_aggregator`. +-module(emqx_connector_aggreg_csv). %% Container API -export([ @@ -33,7 +33,7 @@ column_order => [column()] }. --type record() :: emqx_bridge_s3_aggregator:record(). +-type record() :: emqx_connector_aggregator:record(). -type column() :: binary(). %% diff --git a/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_delivery.erl b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_delivery.erl new file mode 100644 index 000000000..071c28ee5 --- /dev/null +++ b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_delivery.erl @@ -0,0 +1,195 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% This module takes aggregated records from a buffer and delivers them to a blob storage +%% backend, wrapped in a configurable container (though currently there's only CSV). +-module(emqx_connector_aggreg_delivery). + +-include_lib("snabbkaffe/include/trace.hrl"). +-include("emqx_connector_aggregator.hrl"). + +-export([start_link/3]). + +%% Internal exports +-export([ + init/4, + loop/3 +]). + +%% Sys +-export([ + system_continue/3, + system_terminate/4, + format_status/2 +]). + +-record(delivery, { + name :: _Name, + callback_module :: module(), + container :: emqx_connector_aggreg_csv:container(), + reader :: emqx_connector_aggreg_buffer:reader(), + transfer :: transfer_state(), + empty :: boolean() +}). + +-type state() :: #delivery{}. + +-type init_opts() :: #{ + callback_module := module(), + any() => term() +}. + +-type transfer_state() :: term(). + +%% @doc Initialize the transfer state, such as blob storage path, transfer options, client +%% credentials, etc. . +-callback init_transfer_state(buffer_map(), map()) -> transfer_state(). + +%% @doc Append data to the transfer before sending. Usually should not fail. +-callback process_append(iodata(), transfer_state()) -> transfer_state(). + +%% @doc Push appended transfer data to its destination (e.g.: upload a part of a +%% multi-part upload). May fail. +-callback process_write(transfer_state()) -> {ok, transfer_state()} | {error, term()}. + +%% @doc Finalize the transfer and clean up any resources. May return a term summarizing +%% the transfer. +-callback process_complete(transfer_state()) -> {ok, term()}. + +%% + +start_link(Name, Buffer, Opts) -> + proc_lib:start_link(?MODULE, init, [self(), Name, Buffer, Opts]). + +%% + +-spec init(pid(), _Name, buffer(), init_opts()) -> no_return(). +init(Parent, Name, Buffer, Opts) -> + ?tp(connector_aggreg_delivery_started, #{action => Name, buffer => Buffer}), + Reader = open_buffer(Buffer), + Delivery = init_delivery(Name, Reader, Buffer, Opts#{action => Name}), + _ = erlang:process_flag(trap_exit, true), + ok = proc_lib:init_ack({ok, self()}), + loop(Delivery, Parent, []). + +init_delivery( + Name, + Reader, + Buffer, + Opts = #{ + container := ContainerOpts, + callback_module := Mod + } +) -> + BufferMap = emqx_connector_aggregator:buffer_to_map(Buffer), + #delivery{ + name = Name, + callback_module = Mod, + container = mk_container(ContainerOpts), + reader = Reader, + transfer = Mod:init_transfer_state(BufferMap, Opts), + empty = true + }. + +open_buffer(#buffer{filename = Filename}) -> + case file:open(Filename, [read, binary, raw]) of + {ok, FD} -> + {_Meta, Reader} = emqx_connector_aggreg_buffer:new_reader(FD), + Reader; + {error, Reason} -> + error(#{reason => buffer_open_failed, file => Filename, posix => Reason}) + end. + +mk_container(#{type := csv, column_order := OrderOpt}) -> + %% TODO: Deduplicate? + ColumnOrder = lists:map(fun emqx_utils_conv:bin/1, OrderOpt), + emqx_connector_aggreg_csv:new(#{column_order => ColumnOrder}). + +%% + +-spec loop(state(), pid(), [sys:debug_option()]) -> no_return(). +loop(Delivery, Parent, Debug) -> + %% NOTE: This function is mocked in tests. + receive + Msg -> handle_msg(Msg, Delivery, Parent, Debug) + after 0 -> + process_delivery(Delivery, Parent, Debug) + end. + +process_delivery(Delivery0 = #delivery{reader = Reader0}, Parent, Debug) -> + case emqx_connector_aggreg_buffer:read(Reader0) of + {Records = [#{} | _], Reader} -> + Delivery1 = Delivery0#delivery{reader = Reader}, + Delivery2 = process_append_records(Records, Delivery1), + Delivery = process_write(Delivery2), + ?MODULE:loop(Delivery, Parent, Debug); + {[], Reader} -> + Delivery = Delivery0#delivery{reader = Reader}, + ?MODULE:loop(Delivery, Parent, Debug); + eof -> + process_complete(Delivery0); + {Unexpected, _Reader} -> + exit({buffer_unexpected_record, Unexpected}) + end. + +process_append_records( + Records, + Delivery = #delivery{ + callback_module = Mod, + container = Container0, + transfer = Transfer0 + } +) -> + {Writes, Container} = emqx_connector_aggreg_csv:fill(Records, Container0), + Transfer = Mod:process_append(Writes, Transfer0), + Delivery#delivery{ + container = Container, + transfer = Transfer, + empty = false + }. + +process_write(Delivery = #delivery{callback_module = Mod, transfer = Transfer0}) -> + case Mod:process_write(Transfer0) of + {ok, Transfer} -> + Delivery#delivery{transfer = Transfer}; + {error, Reason} -> + %% Todo: handle more gracefully? Retry? + error({transfer_failed, Reason}) + end. + +process_complete(#delivery{name = Name, empty = true}) -> + ?tp(connector_aggreg_delivery_completed, #{action => Name, transfer => empty}), + exit({shutdown, {skipped, empty}}); +process_complete(#delivery{ + name = Name, callback_module = Mod, container = Container, transfer = Transfer0 +}) -> + Trailer = emqx_connector_aggreg_csv:close(Container), + Transfer = Mod:process_append(Trailer, Transfer0), + {ok, Completed} = Mod:process_complete(Transfer), + ?tp(connector_aggreg_delivery_completed, #{action => Name, transfer => Completed}), + ok. + +%% + +handle_msg({system, From, Msg}, Delivery, Parent, Debug) -> + sys:handle_system_msg(Msg, From, Parent, ?MODULE, Debug, Delivery); +handle_msg({'EXIT', Parent, Reason}, Delivery, Parent, Debug) -> + system_terminate(Reason, Parent, Debug, Delivery); +handle_msg(_Msg, Delivery, Parent, Debug) -> + ?MODULE:loop(Parent, Debug, Delivery). + +-spec system_continue(pid(), [sys:debug_option()], state()) -> no_return(). +system_continue(Parent, Debug, Delivery) -> + ?MODULE:loop(Delivery, Parent, Debug). + +-spec system_terminate(_Reason, pid(), [sys:debug_option()], state()) -> _. +system_terminate(_Reason, _Parent, _Debug, #delivery{callback_module = Mod, transfer = Transfer}) -> + Mod:process_terminate(Transfer). + +-spec format_status(normal, Args :: [term()]) -> _StateFormatted. +format_status(_Normal, [_PDict, _SysState, _Parent, _Debug, Delivery]) -> + #delivery{callback_module = Mod} = Delivery, + Delivery#delivery{ + transfer = Mod:process_format_status(Delivery#delivery.transfer) + }. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_sup.erl similarity index 95% rename from apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl rename to apps/emqx_connector_aggregator/src/emqx_connector_aggreg_sup.erl index 230711a76..e80652542 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl +++ b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_sup.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_s3_sup). +-module(emqx_connector_aggreg_sup). -export([ start_link/0, diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_upload_sup.erl similarity index 91% rename from apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl rename to apps/emqx_connector_aggregator/src/emqx_connector_aggreg_upload_sup.erl index 973187b7e..d08b3b8ed 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl +++ b/apps/emqx_connector_aggregator/src/emqx_connector_aggreg_upload_sup.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_s3_aggreg_upload_sup). +-module(emqx_connector_aggreg_upload_sup). -export([ start_link/3, @@ -33,7 +33,7 @@ start_delivery(Name, Buffer) -> supervisor:start_child(?SUPREF(Name), [Buffer]). start_delivery_proc(Name, DeliveryOpts, Buffer) -> - emqx_bridge_s3_aggreg_delivery:start_link(Name, Buffer, DeliveryOpts). + emqx_connector_aggreg_delivery:start_link(Name, Buffer, DeliveryOpts). %% @@ -45,7 +45,7 @@ init({root, Name, AggregOpts, DeliveryOpts}) -> }, AggregatorChildSpec = #{ id => aggregator, - start => {emqx_bridge_s3_aggregator, start_link, [Name, AggregOpts]}, + start => {emqx_connector_aggregator, start_link, [Name, AggregOpts]}, type => worker, restart => permanent }, diff --git a/apps/emqx_connector_aggregator/src/emqx_connector_aggregator.app.src b/apps/emqx_connector_aggregator/src/emqx_connector_aggregator.app.src new file mode 100644 index 000000000..870219807 --- /dev/null +++ b/apps/emqx_connector_aggregator/src/emqx_connector_aggregator.app.src @@ -0,0 +1,13 @@ +{application, emqx_connector_aggregator, [ + {description, "EMQX Enterprise Connector Data Aggregator"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {mod, {emqx_connector_aggreg_app, []}}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl b/apps/emqx_connector_aggregator/src/emqx_connector_aggregator.erl similarity index 94% rename from apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl rename to apps/emqx_connector_aggregator/src/emqx_connector_aggregator.erl index 47ecdeb4a..f3936fd54 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl +++ b/apps/emqx_connector_aggregator/src/emqx_connector_aggregator.erl @@ -5,18 +5,19 @@ %% This module manages buffers for aggregating records and offloads them %% to separate "delivery" processes when they are full or time interval %% is over. --module(emqx_bridge_s3_aggregator). +-module(emqx_connector_aggregator). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). --include("emqx_bridge_s3_aggregator.hrl"). +-include("emqx_connector_aggregator.hrl"). -export([ start_link/2, push_records/3, tick/2, - take_error/1 + take_error/1, + buffer_to_map/1 ]). -behaviour(gen_server). @@ -72,6 +73,15 @@ tick(Name, Timestamp) -> take_error(Name) -> gen_server:call(?SRVREF(Name), take_error). +buffer_to_map(#buffer{} = Buffer) -> + #{ + since => Buffer#buffer.since, + until => Buffer#buffer.until, + seq => Buffer#buffer.seq, + filename => Buffer#buffer.filename, + max_records => Buffer#buffer.max_records + }. + %% write_records_limited(Name, Buffer = #buffer{max_records = undefined}, Records) -> @@ -90,9 +100,9 @@ write_records_limited(Name, Buffer = #buffer{max_records = MaxRecords}, Records) end. write_records(Name, Buffer = #buffer{fd = Writer}, Records) -> - case emqx_bridge_s3_aggreg_buffer:write(Records, Writer) of + case emqx_connector_aggreg_buffer:write(Records, Writer) of ok -> - ?tp(s3_aggreg_records_written, #{action => Name, records => Records}), + ?tp(connector_aggreg_records_written, #{action => Name, records => Records}), ok; {error, terminated} -> BufferNext = rotate_buffer(Name, Buffer), @@ -250,9 +260,9 @@ compute_since(Timestamp, PrevSince, Interval) -> allocate_buffer(Since, Seq, St = #st{name = Name}) -> Buffer = #buffer{filename = Filename, cnt_records = Counter} = mk_buffer(Since, Seq, St), {ok, FD} = file:open(Filename, [write, binary]), - Writer = emqx_bridge_s3_aggreg_buffer:new_writer(FD, _Meta = []), + Writer = emqx_connector_aggreg_buffer:new_writer(FD, _Meta = []), _ = add_counter(Counter), - ?tp(s3_aggreg_buffer_allocated, #{action => Name, filename => Filename}), + ?tp(connector_aggreg_buffer_allocated, #{action => Name, filename => Filename}), Buffer#buffer{fd = Writer}. recover_buffer(Buffer = #buffer{filename = Filename, cnt_records = Counter}) -> @@ -274,7 +284,7 @@ recover_buffer(Buffer = #buffer{filename = Filename, cnt_records = Counter}) -> end. recover_buffer_writer(FD, Filename) -> - try emqx_bridge_s3_aggreg_buffer:new_reader(FD) of + try emqx_connector_aggreg_buffer:new_reader(FD) of {_Meta, Reader} -> recover_buffer_writer(FD, Filename, Reader, 0) catch error:Reason -> @@ -282,7 +292,7 @@ recover_buffer_writer(FD, Filename) -> end. recover_buffer_writer(FD, Filename, Reader0, NWritten) -> - try emqx_bridge_s3_aggreg_buffer:read(Reader0) of + try emqx_connector_aggreg_buffer:read(Reader0) of {Records, Reader} when is_list(Records) -> recover_buffer_writer(FD, Filename, Reader, NWritten + length(Records)); {Unexpected, _Reader} -> @@ -303,7 +313,7 @@ recover_buffer_writer(FD, Filename, Reader0, NWritten) -> "Buffer is truncated or corrupted somewhere in the middle. " "Corrupted records will be discarded." }), - Writer = emqx_bridge_s3_aggreg_buffer:takeover(Reader0), + Writer = emqx_connector_aggreg_buffer:takeover(Reader0), {ok, Writer, NWritten} end. @@ -362,7 +372,7 @@ lookup_current_buffer(Name) -> %% enqueue_delivery(Buffer, St = #st{name = Name, deliveries = Ds}) -> - {ok, Pid} = emqx_bridge_s3_aggreg_upload_sup:start_delivery(Name, Buffer), + {ok, Pid} = emqx_connector_aggreg_upload_sup:start_delivery(Name, Buffer), MRef = erlang:monitor(process, Pid), St#st{deliveries = Ds#{MRef => Buffer}}. diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl b/apps/emqx_connector_aggregator/test/emqx_connector_aggreg_buffer_SUITE.erl similarity index 75% rename from apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl rename to apps/emqx_connector_aggregator/test/emqx_connector_aggreg_buffer_SUITE.erl index 199e070d3..0c172cc83 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl +++ b/apps/emqx_connector_aggregator/test/emqx_connector_aggreg_buffer_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_s3_aggreg_buffer_SUITE). +-module(emqx_connector_aggreg_buffer_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -29,7 +29,7 @@ t_write_read_cycle(Config) -> Filename = mk_filename(?FUNCTION_NAME, Config), Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}}, {ok, WFD} = file:open(Filename, [write, binary]), - Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata), + Writer = emqx_connector_aggreg_buffer:new_writer(WFD, Metadata), Terms = [ [], [[[[[[[[]]]]]]]], @@ -43,12 +43,12 @@ t_write_read_cycle(Config) -> {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})} ], ok = lists:foreach( - fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end, + fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer)) end, Terms ), ok = file:close(WFD), {ok, RFD} = file:open(Filename, [read, binary, raw]), - {MetadataRead, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD), + {MetadataRead, Reader} = emqx_connector_aggreg_buffer:new_reader(RFD), ?assertEqual(Metadata, MetadataRead), TermsRead = read_until_eof(Reader), ?assertEqual(Terms, TermsRead). @@ -60,7 +60,7 @@ t_read_empty(Config) -> {ok, RFD} = file:open(Filename, [read, binary]), ?assertError( {buffer_incomplete, header}, - emqx_bridge_s3_aggreg_buffer:new_reader(RFD) + emqx_connector_aggreg_buffer:new_reader(RFD) ). t_read_garbage(Config) -> @@ -71,14 +71,14 @@ t_read_garbage(Config) -> {ok, RFD} = file:open(Filename, [read, binary]), ?assertError( badarg, - emqx_bridge_s3_aggreg_buffer:new_reader(RFD) + emqx_connector_aggreg_buffer:new_reader(RFD) ). t_read_truncated(Config) -> Filename = mk_filename(?FUNCTION_NAME, Config), {ok, WFD} = file:open(Filename, [write, binary]), Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}}, - Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata), + Writer = emqx_connector_aggreg_buffer:new_writer(WFD, Metadata), Terms = [ [[[[[[[[[[[]]]]]]]]]]], lists:seq(1, 100000), @@ -88,36 +88,36 @@ t_read_truncated(Config) -> LastTerm = {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})}, ok = lists:foreach( - fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end, + fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer)) end, Terms ), {ok, WPos} = file:position(WFD, cur), - ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(LastTerm, Writer)), + ?assertEqual(ok, emqx_connector_aggreg_buffer:write(LastTerm, Writer)), ok = file:close(WFD), - ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos + 1), + ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, WPos + 1), {ok, RFD1} = file:open(Filename, [read, binary]), - {Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD1), + {Metadata, Reader0} = emqx_connector_aggreg_buffer:new_reader(RFD1), {ReadTerms1, Reader1} = read_terms(length(Terms), Reader0), ?assertEqual(Terms, ReadTerms1), ?assertError( badarg, - emqx_bridge_s3_aggreg_buffer:read(Reader1) + emqx_connector_aggreg_buffer:read(Reader1) ), - ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2), + ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, WPos div 2), {ok, RFD2} = file:open(Filename, [read, binary]), - {Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD2), + {Metadata, Reader2} = emqx_connector_aggreg_buffer:new_reader(RFD2), {ReadTerms2, Reader3} = read_terms(_FitsInto = 3, Reader2), ?assertEqual(lists:sublist(Terms, 3), ReadTerms2), ?assertError( badarg, - emqx_bridge_s3_aggreg_buffer:read(Reader3) + emqx_connector_aggreg_buffer:read(Reader3) ). t_read_truncated_takeover_write(Config) -> Filename = mk_filename(?FUNCTION_NAME, Config), {ok, WFD} = file:open(Filename, [write, binary]), Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}}, - Writer1 = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata), + Writer1 = emqx_connector_aggreg_buffer:new_writer(WFD, Metadata), Terms1 = [ [[[[[[[[[[[]]]]]]]]]]], lists:seq(1, 10000), @@ -129,14 +129,14 @@ t_read_truncated_takeover_write(Config) -> {<<"application/x-octet-stream">>, rand:bytes(102400)} ], ok = lists:foreach( - fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer1)) end, + fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer1)) end, Terms1 ), {ok, WPos} = file:position(WFD, cur), ok = file:close(WFD), - ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2), + ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, WPos div 2), {ok, RWFD} = file:open(Filename, [read, write, binary]), - {Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RWFD), + {Metadata, Reader0} = emqx_connector_aggreg_buffer:new_reader(RWFD), {ReadTerms1, Reader1} = read_terms(_Survived = 3, Reader0), ?assertEqual( lists:sublist(Terms1, 3), @@ -144,16 +144,16 @@ t_read_truncated_takeover_write(Config) -> ), ?assertError( badarg, - emqx_bridge_s3_aggreg_buffer:read(Reader1) + emqx_connector_aggreg_buffer:read(Reader1) ), - Writer2 = emqx_bridge_s3_aggreg_buffer:takeover(Reader1), + Writer2 = emqx_connector_aggreg_buffer:takeover(Reader1), ok = lists:foreach( - fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer2)) end, + fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer2)) end, Terms2 ), ok = file:close(RWFD), {ok, RFD} = file:open(Filename, [read, binary]), - {Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD), + {Metadata, Reader2} = emqx_connector_aggreg_buffer:new_reader(RFD), ReadTerms2 = read_until_eof(Reader2), ?assertEqual( lists:sublist(Terms1, 3) ++ Terms2, @@ -168,12 +168,12 @@ mk_filename(Name, Config) -> read_terms(0, Reader) -> {[], Reader}; read_terms(N, Reader0) -> - {Term, Reader1} = emqx_bridge_s3_aggreg_buffer:read(Reader0), + {Term, Reader1} = emqx_connector_aggreg_buffer:read(Reader0), {Terms, Reader} = read_terms(N - 1, Reader1), {[Term | Terms], Reader}. read_until_eof(Reader0) -> - case emqx_bridge_s3_aggreg_buffer:read(Reader0) of + case emqx_connector_aggreg_buffer:read(Reader0) of {Term, Reader} -> [Term | read_until_eof(Reader)]; eof -> diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl b/apps/emqx_connector_aggregator/test/emqx_connector_aggreg_csv_tests.erl similarity index 89% rename from apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl rename to apps/emqx_connector_aggregator/test/emqx_connector_aggreg_csv_tests.erl index 6da70c6fe..cd7ac8bcb 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl +++ b/apps/emqx_connector_aggregator/test/emqx_connector_aggreg_csv_tests.erl @@ -2,12 +2,12 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_s3_aggreg_csv_tests). +-module(emqx_connector_aggreg_csv_tests). -include_lib("eunit/include/eunit.hrl"). encoding_test() -> - CSV = emqx_bridge_s3_aggreg_csv:new(#{}), + CSV = emqx_connector_aggreg_csv:new(#{}), ?assertEqual( "A,B,Ç\n" "1.2345,string,0.0\n" @@ -28,7 +28,7 @@ encoding_test() -> column_order_test() -> Order = [<<"ID">>, <<"TS">>], - CSV = emqx_bridge_s3_aggreg_csv:new(#{column_order => Order}), + CSV = emqx_connector_aggreg_csv:new(#{column_order => Order}), ?assertEqual( "ID,TS,A,B,D\n" "1,2024-01-01,12.34,str,\"[]\"\n" @@ -63,10 +63,10 @@ fill_close(CSV, LRecords) -> string(fill_close_(CSV, LRecords)). fill_close_(CSV0, [Records | LRest]) -> - {Writes, CSV} = emqx_bridge_s3_aggreg_csv:fill(Records, CSV0), + {Writes, CSV} = emqx_connector_aggreg_csv:fill(Records, CSV0), [Writes | fill_close_(CSV, LRest)]; fill_close_(CSV, []) -> - [emqx_bridge_s3_aggreg_csv:close(CSV)]. + [emqx_connector_aggreg_csv:close(CSV)]. string(Writes) -> unicode:characters_to_list(Writes). diff --git a/apps/emqx_connector_aggregator/test/emqx_connector_aggregator_test_helpers.erl b/apps/emqx_connector_aggregator/test/emqx_connector_aggregator_test_helpers.erl new file mode 100644 index 000000000..c4e236c49 --- /dev/null +++ b/apps/emqx_connector_aggregator/test/emqx_connector_aggregator_test_helpers.erl @@ -0,0 +1,25 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_connector_aggregator_test_helpers). + +-compile(nowarn_export_all). +-compile(export_all). + +%% API +-export([]). + +%%------------------------------------------------------------------------------ +%% File utilities +%%------------------------------------------------------------------------------ + +truncate_at(Filename, Pos) -> + {ok, FD} = file:open(Filename, [read, write, binary]), + {ok, Pos} = file:position(FD, Pos), + ok = file:truncate(FD), + ok = file:close(FD). + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index 7c4dea030..0906c564b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -311,7 +311,7 @@ user(delete, #{bindings := #{username := Username0}, headers := Headers} = Req) end. is_self_auth(?SSO_USERNAME(_, _), _) -> - fasle; + false; is_self_auth(Username, #{<<"authorization">> := Token}) -> is_self_auth(Username, Token); is_self_auth(Username, #{<<"Authorization">> := Token}) -> diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index 95fe2e809..14c4f5fde 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -20,11 +20,13 @@ -compile(export_all). -import(emqx_dashboard_SUITE, [auth_header_/0]). +-import(emqx_common_test_helpers, [on_exit/1]). -include("emqx_dashboard.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -define(SERVER, "http://127.0.0.1:18083"). -define(BASE_PATH, "/api/v5"). @@ -52,10 +54,47 @@ %%-------------------------------------------------------------------- all() -> - emqx_common_test_helpers:all(?MODULE). + [ + {group, common}, + {group, persistent_sessions} + ]. + +groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), + PSTCs = persistent_session_testcases(), + [ + {common, [], AllTCs -- PSTCs}, + {persistent_sessions, [], PSTCs} + ]. + +persistent_session_testcases() -> + [ + t_persistent_session_stats + ]. init_per_suite(Config) -> - emqx_common_test_helpers:clear_screen(), + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(persistent_sessions = Group, Config) -> + Apps = emqx_cth_suite:start( + [ + emqx_conf, + {emqx, "session_persistence {enable = true}"}, + {emqx_retainer, ?BASE_RETAINER_CONF}, + emqx_management, + emqx_mgmt_api_test_util:emqx_dashboard( + "dashboard.listeners.http { enable = true, bind = 18083 }\n" + "dashboard.sample_interval = 1s" + ) + ], + #{work_dir => emqx_cth_suite:work_dir(Group, Config)} + ), + {ok, _} = emqx_common_test_http:create_default_app(), + [{apps, Apps} | Config]; +init_per_group(common = Group, Config) -> Apps = emqx_cth_suite:start( [ emqx, @@ -67,12 +106,12 @@ init_per_suite(Config) -> "dashboard.sample_interval = 1s" ) ], - #{work_dir => emqx_cth_suite:work_dir(Config)} + #{work_dir => emqx_cth_suite:work_dir(Group, Config)} ), {ok, _} = emqx_common_test_http:create_default_app(), [{apps, Apps} | Config]. -end_per_suite(Config) -> +end_per_group(_Group, Config) -> Apps = ?config(apps, Config), emqx_cth_suite:stop(Apps), ok. @@ -84,6 +123,7 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, _Config) -> ok = snabbkaffe:stop(), + emqx_common_test_helpers:call_janitor(), ok. %%-------------------------------------------------------------------- @@ -272,6 +312,51 @@ t_monitor_api_error(_) -> request(["monitor"], "latest=-1"), ok. +%% Verifies that subscriptions from persistent sessions are correctly accounted for. +t_persistent_session_stats(_Config) -> + %% pre-condition + true = emqx_persistent_message:is_persistence_enabled(), + + NonPSClient = start_and_connect(#{ + clientid => <<"non-ps">>, + expiry_interval => 0 + }), + PSClient = start_and_connect(#{ + clientid => <<"ps">>, + expiry_interval => 30 + }), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"non/ps/topic/+">>, 2), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"non/ps/topic">>, 2), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"common/topic/+">>, 2), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"common/topic">>, 2), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"ps/topic/+">>, 2), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"ps/topic">>, 2), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"common/topic/+">>, 2), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"common/topic">>, 2), + {ok, _} = + snabbkaffe:block_until( + ?match_n_events(2, #{?snk_kind := dashboard_monitor_flushed}), + infinity + ), + ?retry(1_000, 10, begin + ?assertMatch( + {ok, #{ + %% N.B.: we currently don't perform any deduplication between persistent + %% and non-persistent routes, so we count `commont/topic' twice and get 8 + %% instead of 6 here. + <<"topics">> := 8, + <<"subscriptions">> := 8 + }}, + request(["monitor_current"]) + ) + end), + %% Sanity checks + PSRouteCount = emqx_persistent_session_ds_router:stats(n_routes), + ?assert(PSRouteCount > 0, #{ps_route_count => PSRouteCount}), + PSSubCount = emqx_persistent_session_bookkeeper:get_subscription_count(), + ?assert(PSSubCount > 0, #{ps_sub_count => PSSubCount}), + ok. + request(Path) -> request(Path, ""). @@ -340,3 +425,22 @@ waiting_emqx_stats_and_monitor_update(WaitKey) -> %% manually call monitor update _ = emqx_dashboard_monitor:current_rate_cluster(), ok. + +start_and_connect(Opts) -> + Defaults = #{clean_start => false, expiry_interval => 30}, + #{ + clientid := ClientId, + clean_start := CleanStart, + expiry_interval := EI + } = maps:merge(Defaults, Opts), + {ok, Client} = emqtt:start_link([ + {clientid, ClientId}, + {clean_start, CleanStart}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => EI}} + ]), + on_exit(fun() -> + catch emqtt:disconnect(Client, ?RC_NORMAL_DISCONNECTION, #{'Session-Expiry-Interval' => 0}) + end), + {ok, _} = emqtt:connect(Client), + Client. diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_shard.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_shard.erl index e0e70596a..d4dfa9115 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_shard.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_shard.erl @@ -82,6 +82,8 @@ server_name(DB, Shard, Site) -> %% +-spec servers(emqx_ds:db(), emqx_ds_replication_layer:shard_id(), Order) -> [server(), ...] when + Order :: leader_preferred | undefined. servers(DB, Shard, _Order = leader_preferred) -> get_servers_leader_preferred(DB, Shard); servers(DB, Shard, _Order = undefined) -> @@ -98,7 +100,7 @@ get_servers_leader_preferred(DB, Shard) -> Servers = ra_leaderboard:lookup_members(ClusterName), [Leader | lists:delete(Leader, Servers)]; undefined -> - get_shard_servers(DB, Shard) + get_online_servers(DB, Shard) end. get_server_local_preferred(DB, Shard) -> @@ -111,7 +113,7 @@ get_server_local_preferred(DB, Shard) -> %% TODO %% Leader is unkonwn if there are no servers of this group on the %% local node. We want to pick a replica in that case as well. - pick_random(get_shard_servers(DB, Shard)) + pick_random(get_online_servers(DB, Shard)) end. lookup_leader(DB, Shard) -> @@ -121,6 +123,21 @@ lookup_leader(DB, Shard) -> ClusterName = get_cluster_name(DB, Shard), ra_leaderboard:lookup_leader(ClusterName). +get_online_servers(DB, Shard) -> + filter_online(get_shard_servers(DB, Shard)). + +filter_online(Servers) -> + case lists:filter(fun is_server_online/1, Servers) of + [] -> + %% NOTE: Must return non-empty list. + Servers; + Online -> + Online + end. + +is_server_online({_Name, Node}) -> + Node == node() orelse lists:member(Node, nodes()). + pick_local(Servers) -> case lists:keyfind(node(), 2, Servers) of Local when is_tuple(Local) -> diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 8df531a43..84dfe44a2 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -518,7 +518,7 @@ handle_msg({inet_reply, _Sock, ok}, State = #state{active_n = ActiveN}) -> handle_msg({inet_reply, _Sock, {error, Reason}}, State) -> handle_info({sock_error, Reason}, State); handle_msg({close, Reason}, State) -> - ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}), + ?tp(debug, force_socket_close, #{reason => Reason}), handle_info({sock_closed, Reason}, close_socket(State)); handle_msg( {event, connected}, diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl index 3609356dd..014c54ec7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -39,6 +39,7 @@ %% Authentication circle -export([ authenticate/2, + connection_expire_interval/2, open_session/5, open_session/6, insert_channel_info/4, @@ -78,6 +79,13 @@ authenticate(_Ctx, ClientInfo0) -> {error, Reason} end. +-spec connection_expire_interval(context(), emqx_types:clientinfo()) -> + undefined | non_neg_integer(). +connection_expire_interval(_Ctx, #{auth_expire_at := undefined}) -> + undefined; +connection_expire_interval(_Ctx, #{auth_expire_at := ExpireAt}) -> + max(0, ExpireAt - erlang:system_time(millisecond)). + %% @doc Register the session to the cluster. %% %% This function should be called after the client has authenticated @@ -157,6 +165,9 @@ set_chan_stats(_Ctx = #{gwname := GwName}, ClientId, Stats) -> connection_closed(_Ctx = #{gwname := GwName}, ClientId) -> emqx_gateway_cm:connection_closed(GwName, ClientId). +%%-------------------------------------------------------------------- +%% Message circle + -spec authorize( context(), emqx_types:clientinfo(), @@ -167,6 +178,9 @@ connection_closed(_Ctx = #{gwname := GwName}, ClientId) -> authorize(_Ctx, ClientInfo, Action, Topic) -> emqx_access_control:authorize(ClientInfo, Action, Topic). +%%-------------------------------------------------------------------- +%% Metrics & Stats + metrics_inc(_Ctx = #{gwname := GwName}, Name) -> emqx_gateway_metrics:inc(GwName, Name). @@ -183,6 +197,8 @@ eval_mountpoint(ClientInfo = #{mountpoint := MountPoint}) -> MountPoint1 = emqx_mountpoint:replvar(MountPoint, ClientInfo), ClientInfo#{mountpoint := MountPoint1}. -merge_auth_result(ClientInfo, AuthResult) when is_map(ClientInfo) andalso is_map(AuthResult) -> - IsSuperuser = maps:get(is_superuser, AuthResult, false), - maps:merge(ClientInfo, AuthResult#{is_superuser => IsSuperuser}). +merge_auth_result(ClientInfo, AuthResult0) when is_map(ClientInfo) andalso is_map(AuthResult0) -> + IsSuperuser = maps:get(is_superuser, AuthResult0, false), + ExpireAt = maps:get(expire_at, AuthResult0, undefined), + AuthResult1 = maps:without([expire_at], AuthResult0), + maps:merge(ClientInfo#{auth_expire_at => ExpireAt}, AuthResult1#{is_superuser => IsSuperuser}). diff --git a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl index aead0e554..3d30aa585 100644 --- a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl @@ -82,4 +82,4 @@ t_authenticate(_) -> ?assertMatch({ok, #{is_superuser := true}}, emqx_gateway_ctx:authenticate(Ctx, Info4)), ok. -default_result(Info) -> Info#{zone => default, is_superuser => false}. +default_result(Info) -> Info#{zone => default, is_superuser => false, auth_expire_at => undefined}. diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index c753fd361..fbab1ff14 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -214,6 +214,8 @@ handle_timeout(_, {transport, Msg}, Channel) -> call_session(timeout, Msg, Channel); handle_timeout(_, disconnect, Channel) -> {shutdown, normal, Channel}; +handle_timeout(_, connection_expire, Channel) -> + {shutdown, expired, Channel}; handle_timeout(_, _, Channel) -> {ok, Channel}. @@ -595,6 +597,14 @@ process_connect( iter(Iter, reply({error, bad_request}, Msg, Result), Channel) end. +schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of + undefined -> + Channel; + Interval -> + ensure_timer(connection_expire_timer, Interval, connection_expire, Channel) + end. + run_hooks(Ctx, Name, Args) -> emqx_gateway_ctx:metrics_inc(Ctx, Name), emqx_hooks:run(Name, Args). @@ -619,7 +629,7 @@ ensure_connected( NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, _ = run_hooks(Ctx, 'client.connack', [NConnInfo, connection_accepted, #{}]), ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), - Channel#channel{conninfo = NConnInfo, conn_state = connected}. + schedule_connection_expire(Channel#channel{conninfo = NConnInfo, conn_state = connected}). %%-------------------------------------------------------------------- %% Ensure disconnected diff --git a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src index 5f17360a7..3a715eac4 100644 --- a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway_coap, [ {description, "CoAP Gateway"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl index c3a35774c..3201d5dbf 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl @@ -29,6 +29,7 @@ -include_lib("er_coap_client/include/coap.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/asserts.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -83,6 +84,17 @@ init_per_testcase(t_connection_with_authn_failed, Config) -> fun(_) -> {error, bad_username_or_password} end ), Config; +init_per_testcase(t_connection_with_expire, Config) -> + ok = meck:new(emqx_access_control, [passthrough, no_history]), + ok = meck:expect( + emqx_access_control, + authenticate, + fun(_) -> + {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 100}} + end + ), + snabbkaffe:start_trace(), + Config; init_per_testcase(t_heartbeat, Config) -> NewHeartbeat = 800, OldConf = emqx:get_raw_config([gateway, coap]), @@ -103,6 +115,10 @@ end_per_testcase(t_heartbeat, Config) -> OldConf = ?config(old_conf, Config), {ok, _} = emqx_gateway_conf:update_gateway(coap, OldConf), ok; +end_per_testcase(t_connection_with_expire, Config) -> + snabbkaffe:stop(), + meck:unload(emqx_access_control), + Config; end_per_testcase(_, Config) -> ok = meck:unload(emqx_access_control), Config. @@ -270,6 +286,26 @@ t_connection_with_authn_failed(_) -> ), ok. +t_connection_with_expire(_) -> + ChId = {{127, 0, 0, 1}, 5683}, + {ok, Sock} = er_coap_udp_socket:start_link(), + {ok, Channel} = er_coap_udp_socket:get_channel(Sock, ChId), + + URI = ?MQTT_PREFIX ++ "/connection?clientid=client1", + + ?assertWaitEvent( + begin + Req = make_req(post), + {ok, created, _Data} = do_request(Channel, URI, Req) + end, + #{ + ?snk_kind := conn_process_terminated, + clientid := <<"client1">>, + reason := {shutdown, expired} + }, + 5000 + ). + t_publish(_) -> %% can publish to a normal topic Topics = [ diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index 7268fa77a..c145506c9 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -302,6 +302,9 @@ handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) - {shutdown, Reason, Channel}; handle_timeout(_TRef, force_close_idle, Channel) -> {shutdown, idle_timeout, Channel}; +handle_timeout(_TRef, connection_expire, Channel) -> + NChannel = remove_timer_ref(connection_expire, Channel), + {ok, [{event, disconnected}, {close, expired}], NChannel}; handle_timeout(_TRef, Msg, Channel) -> ?SLOG(warning, #{ msg => "unexpected_timeout_signal", @@ -666,10 +669,18 @@ ensure_connected( ) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), - Channel#channel{ + schedule_connection_expire(Channel#channel{ conninfo = NConnInfo, conn_state = connected - }. + }). + +schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of + undefined -> + Channel; + Interval -> + ensure_timer(connection_expire, Interval, Channel) + end. ensure_disconnected( Reason, diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src index 3d11acf12..34fcca216 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway_exproto, [ {description, "ExProto Gateway"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {registered, []}, {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl index 2517e9fa3..2e73ce8b8 100644 --- a/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/asserts.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -81,6 +82,7 @@ groups() -> t_raw_publish, t_auth_deny, t_acl_deny, + t_auth_expire, t_hook_connected_disconnected, t_hook_session_subscribed_unsubscribed, t_hook_message_delivered @@ -157,14 +159,17 @@ end_per_group(_, Cfg) -> init_per_testcase(TestCase, Cfg) when TestCase == t_enter_passive_mode -> + snabbkaffe:start_trace(), case proplists:get_value(listener_type, Cfg) of udp -> {skip, ignore}; _ -> Cfg end; init_per_testcase(_TestCase, Cfg) -> + snabbkaffe:start_trace(), Cfg. end_per_testcase(_TestCase, _Cfg) -> + snabbkaffe:stop(), ok. listener_confs(Type) -> @@ -290,6 +295,42 @@ t_auth_deny(Cfg) -> end, meck:unload([emqx_gateway_ctx]). +t_auth_expire(Cfg) -> + SockType = proplists:get_value(listener_type, Cfg), + Sock = open(SockType), + + Client = #{ + proto_name => <<"demo">>, + proto_ver => <<"v0.1">>, + clientid => <<"test_client_1">> + }, + Password = <<"123456">>, + + ok = meck:new(emqx_access_control, [passthrough, no_history]), + ok = meck:expect( + emqx_access_control, + authenticate, + fun(_) -> + {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}} + end + ), + + ConnBin = frame_connect(Client, Password), + ConnAckBin = frame_connack(0), + + ?assertWaitEvent( + begin + send(Sock, ConnBin), + {ok, ConnAckBin} = recv(Sock, 5000) + end, + #{ + ?snk_kind := conn_process_terminated, + clientid := <<"test_client_1">>, + reason := {shutdown, expired} + }, + 5000 + ). + t_acl_deny(Cfg) -> SockType = proplists:get_value(listener_type, Cfg), Sock = open(SockType), @@ -332,7 +373,6 @@ t_acl_deny(Cfg) -> close(Sock). t_keepalive_timeout(Cfg) -> - ok = snabbkaffe:start_trace(), SockType = proplists:get_value(listener_type, Cfg), Sock = open(SockType), @@ -383,8 +423,7 @@ t_keepalive_timeout(Cfg) -> ?assertEqual(1, length(?of_kind(conn_process_terminated, Trace))), %% socket port should be closed ?assertEqual({error, closed}, recv(Sock, 5000)) - end, - snabbkaffe:stop(). + end. t_hook_connected_disconnected(Cfg) -> SockType = proplists:get_value(listener_type, Cfg), @@ -513,7 +552,6 @@ t_hook_message_delivered(Cfg) -> emqx_hooks:del('message.delivered', {?MODULE, hook_fun5}). t_idle_timeout(Cfg) -> - ok = snabbkaffe:start_trace(), SockType = proplists:get_value(listener_type, Cfg), Sock = open(SockType), @@ -551,8 +589,7 @@ t_idle_timeout(Cfg) -> {ok, #{reason := {shutdown, idle_timeout}}}, ?block_until(#{?snk_kind := conn_process_terminated}, 10000) ) - end, - snabbkaffe:stop(). + end. %%-------------------------------------------------------------------- %% Utils diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src index 155e4dc25..123b60203 100644 --- a/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src +++ b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway_gbt32960, [ {description, "GBT32960 Gateway"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl index 0063ae4e0..9652290d3 100644 --- a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl @@ -72,7 +72,8 @@ -define(TIMER_TABLE, #{ alive_timer => keepalive, - retry_timer => retry_delivery + retry_timer => retry_delivery, + connection_expire_timer => connection_expire }). -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). @@ -468,6 +469,13 @@ handle_timeout( {Outgoings2, NChannel} = dispatch_frame(Channel#channel{inflight = NInflight}), {ok, [{outgoing, Outgoings ++ Outgoings2}], reset_timer(retry_timer, NChannel)} end; +handle_timeout( + _TRef, + connection_expire, + Channel +) -> + NChannel = clean_timer(connection_expire_timer, Channel), + {ok, [{event, disconnected}, {close, expired}], NChannel}; handle_timeout(_TRef, Msg, Channel) -> log(error, #{msg => "unexpected_timeout", content => Msg}, Channel), {ok, Channel}. @@ -591,10 +599,18 @@ ensure_connected( ) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), - Channel#channel{ + schedule_connection_expire(Channel#channel{ conninfo = NConnInfo, conn_state = connected - }. + }). + +schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of + undefined -> + Channel; + Interval -> + ensure_timer(connection_expire_timer, Interval, Channel) + end. process_connect( Frame, diff --git a/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl b/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl index db4d6da94..009ff74eb 100644 --- a/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl +++ b/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl @@ -11,6 +11,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/asserts.hrl"). -define(BYTE, 8 / big - integer). -define(WORD, 16 / big - integer). @@ -52,6 +53,14 @@ end_per_suite(Config) -> emqx_cth_suite:stop(?config(suite_apps, Config)), ok. +init_per_testcase(_, Config) -> + snabbkaffe:start_trace(), + Config. + +end_per_testcase(_, _Config) -> + snabbkaffe:stop(), + ok. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% helper functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% encode(Cmd, Vin, Data) -> @@ -171,6 +180,28 @@ t_case01_login_channel_info(_Config) -> ok = gen_tcp:close(Socket). +t_case01_auth_expire(_Config) -> + ok = meck:new(emqx_access_control, [passthrough, no_history]), + ok = meck:expect( + emqx_access_control, + authenticate, + fun(_) -> + {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}} + end + ), + + ?assertWaitEvent( + begin + {ok, _Socket} = login_first() + end, + #{ + ?snk_kind := conn_process_terminated, + clientid := <<"1G1BL52P7TR115520">>, + reason := {shutdown, expired} + }, + 5000 + ). + t_case02_reportinfo_0x01(_Config) -> % send VEHICLE LOGIN {ok, Socket} = login_first(), diff --git a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src index 36b6bcf4f..66b2db041 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway_lwm2m, [ {description, "LwM2M Gateway"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap, xmerl]}, {env, []}, diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl index 82ea848bb..595041c53 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl @@ -202,6 +202,8 @@ handle_timeout(_, {transport, _} = Msg, Channel) -> call_session(timeout, Msg, Channel); handle_timeout(_, disconnect, Channel) -> {shutdown, normal, Channel}; +handle_timeout(_, connection_expire, Channel) -> + {shutdown, expired, Channel}; handle_timeout(_, _, Channel) -> {ok, Channel}. @@ -353,10 +355,18 @@ ensure_connected( NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), - Channel#channel{ + schedule_connection_expire(Channel#channel{ conninfo = NConnInfo, conn_state = connected - }. + }). + +schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of + undefined -> + Channel; + Interval -> + make_timer(connection_expire, Interval, connection_expire, Channel) + end. %%-------------------------------------------------------------------- %% Ensure disconnected diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl index 6ee08e735..c302c5cd3 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -36,6 +36,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx/include/asserts.hrl"). -record(coap_content, {content_format, payload = <<>>}). @@ -66,6 +67,7 @@ groups() -> [ {test_grp_0_register, [RepeatOpt], [ case01_register, + case01_auth_expire, case01_register_additional_opts, %% TODO now we can't handle partial decode packet %% case01_register_incorrect_opts, @@ -145,6 +147,7 @@ end_per_suite(Config) -> Config. init_per_testcase(TestCase, Config) -> + snabbkaffe:start_trace(), GatewayConfig = case TestCase of case09_auto_observe -> @@ -171,6 +174,7 @@ end_per_testcase(_AllTestCase, Config) -> timer:sleep(300), gen_udp:close(?config(sock, Config)), emqtt:disconnect(?config(emqx_c, Config)), + snabbkaffe:stop(), ok = application:stop(emqx_gateway). default_config() -> @@ -280,6 +284,43 @@ case01_register(Config) -> timer:sleep(50), false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). +case01_auth_expire(Config) -> + ok = meck:new(emqx_access_control, [passthrough, no_history]), + ok = meck:expect( + emqx_access_control, + authenticate, + fun(_) -> + {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}} + end + ), + + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- + UdpSock = ?config(sock, Config), + Epn = "urn:oma:lwm2m:oma:3", + MsgId = 12, + + ?assertWaitEvent( + test_send_coap_request( + UdpSock, + post, + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), + #coap_content{ + content_format = <<"text/plain">>, + payload = <<", , , , ">> + }, + [], + MsgId + ), + #{ + ?snk_kind := conn_process_terminated, + clientid := <<"urn:oma:lwm2m:oma:3">>, + reason := {shutdown, expired} + }, + 5000 + ). + case01_register_additional_opts(Config) -> %%---------------------------------------- %% REGISTER command diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index b840d53a3..501308ea0 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -364,10 +364,18 @@ ensure_connected( ) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), - Channel#channel{ + schedule_connection_expire(Channel#channel{ conninfo = NConnInfo, conn_state = connected - }. + }). + +schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of + undefined -> + Channel; + Interval -> + ensure_timer(connection_expire, Interval, Channel) + end. process_connect( Channel = #channel{ @@ -2122,6 +2130,9 @@ handle_timeout(_TRef, expire_session, Channel) -> shutdown(expired, Channel); handle_timeout(_TRef, expire_asleep, Channel) -> shutdown(asleep_timeout, Channel); +handle_timeout(_TRef, connection_expire, Channel) -> + NChannel = clean_timer(connection_expire, Channel), + handle_out(disconnect, expired, NChannel); handle_timeout(_TRef, Msg, Channel) -> %% NOTE %% We do not expect `emqx_mqttsn_session` to set up any custom timers (i.e with diff --git a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl index c35b93553..9a72c21bf 100644 --- a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl @@ -33,6 +33,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/asserts.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -141,6 +142,14 @@ end_per_suite(Config) -> emqx_common_test_http:delete_default_app(), emqx_cth_suite:stop(?config(suite_apps, Config)). +init_per_testcase(_TestCase, Config) -> + snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TestCase, _Config) -> + snabbkaffe:stop(), + ok. + restart_mqttsn_with_subs_resume_on() -> Conf = emqx:get_raw_config([gateway, mqttsn]), emqx_gateway_conf:update_gateway( @@ -206,6 +215,36 @@ t_connect(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). +t_auth_expire(_) -> + SockName = {'mqttsn:udp:default', 1884}, + ?assertEqual(true, lists:keymember(SockName, 1, esockd:listeners())), + + ok = meck:new(emqx_access_control, [passthrough, no_history]), + ok = meck:expect( + emqx_access_control, + authenticate, + fun(_) -> + {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}} + end + ), + + ?assertWaitEvent( + begin + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, <<"client_id_test1">>), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket) + end, + #{ + ?snk_kind := conn_process_terminated, + clientid := <<"client_id_test1">>, + reason := {shutdown, expired} + }, + 5000 + ). + t_first_disconnect(_) -> SockName = {'mqttsn:udp:default', 1884}, ?assertEqual(true, lists:keymember(SockName, 1, esockd:listeners())), diff --git a/apps/emqx_gateway_ocpp/src/emqx_ocpp_channel.erl b/apps/emqx_gateway_ocpp/src/emqx_ocpp_channel.erl index d20b35d04..8473c9978 100644 --- a/apps/emqx_gateway_ocpp/src/emqx_ocpp_channel.erl +++ b/apps/emqx_gateway_ocpp/src/emqx_ocpp_channel.erl @@ -89,7 +89,8 @@ -type replies() :: reply() | [reply()]. -define(TIMER_TABLE, #{ - alive_timer => keepalive + alive_timer => keepalive, + connection_expire_timer => connection_expire }). -define(INFO_KEYS, [ @@ -315,20 +316,13 @@ enrich_client( expiry_interval => 0, receive_maximum => 1 }, - NClientInfo = fix_mountpoint( + NClientInfo = ClientInfo#{ clientid => ClientId, username => Username - } - ), + }, {ok, Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo}}. -fix_mountpoint(ClientInfo = #{mountpoint := undefined}) -> - ClientInfo; -fix_mountpoint(ClientInfo = #{mountpoint := Mountpoint}) -> - Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), - ClientInfo#{mountpoint := Mountpoint1}. - set_log_meta(#channel{ clientinfo = #{clientid := ClientId}, conninfo = #{peername := Peername} @@ -350,15 +344,14 @@ check_banned(_UserInfo, #channel{clientinfo = ClientInfo}) -> auth_connect( #{password := Password}, - #channel{clientinfo = ClientInfo} = Channel + #channel{ctx = Ctx, clientinfo = ClientInfo} = Channel ) -> #{ clientid := ClientId, username := Username } = ClientInfo, - case emqx_access_control:authenticate(ClientInfo#{password => Password}) of - {ok, AuthResult} -> - NClientInfo = maps:merge(ClientInfo, AuthResult), + case emqx_gateway_ctx:authenticate(Ctx, ClientInfo#{password => Password}) of + {ok, NClientInfo} -> {ok, Channel#channel{clientinfo = NClientInfo}}; {error, Reason} -> ?SLOG(warning, #{ @@ -659,6 +652,9 @@ handle_timeout( {error, timeout} -> handle_out(disconnect, keepalive_timeout, Channel) end; +handle_timeout(_TRef, connection_expire, Channel) -> + %% No take over implemented, so just shutdown + shutdown(expired, Channel); handle_timeout(_TRef, Msg, Channel) -> ?SLOG(error, #{msg => "unexpected_timeout", timeout_msg => Msg}), {ok, Channel}. @@ -796,10 +792,18 @@ ensure_connected( ) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), - Channel#channel{ + schedule_connection_expire(Channel#channel{ conninfo = NConnInfo, conn_state = connected - }. + }). + +schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of + undefined -> + Channel; + Interval -> + ensure_timer(connection_expire_timer, Interval, Channel) + end. ensure_disconnected( Reason, diff --git a/apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl b/apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl index 0932314fe..1b2434a85 100644 --- a/apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl +++ b/apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl @@ -20,6 +20,7 @@ -include("emqx_ocpp.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[OCPP/WS]"). @@ -513,7 +514,8 @@ websocket_close(Reason, State) -> handle_info({sock_closed, Reason}, State). terminate(Reason, _Req, #state{channel = Channel}) -> - ?SLOG(debug, #{msg => "terminated", reason => Reason}), + ClientId = emqx_ocpp_channel:info(clientid, Channel), + ?tp(debug, conn_process_terminated, #{reason => Reason, clientid => ClientId}), emqx_ocpp_channel:terminate(Reason, Channel); terminate(_Reason, _Req, _UnExpectedState) -> ok. diff --git a/apps/emqx_gateway_ocpp/test/emqx_ocpp_SUITE.erl b/apps/emqx_gateway_ocpp/test/emqx_ocpp_SUITE.erl index e63f8891d..ce90d8202 100644 --- a/apps/emqx_gateway_ocpp/test/emqx_ocpp_SUITE.erl +++ b/apps/emqx_gateway_ocpp/test/emqx_ocpp_SUITE.erl @@ -19,6 +19,7 @@ -include("emqx_ocpp.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/asserts.hrl"). -compile(export_all). -compile(nowarn_export_all). @@ -32,8 +33,6 @@ ] ). --define(HEARTBEAT, <<$\n>>). - %% erlfmt-ignore -define(CONF_DEFAULT, <<" gateway.ocpp { @@ -82,6 +81,14 @@ end_per_suite(Config) -> emqx_cth_suite:stop(?config(suite_apps, Config)), ok. +init_per_testcase(_TestCase, Config) -> + snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TestCase, _Config) -> + snabbkaffe:stop(), + ok. + default_config() -> ?CONF_DEFAULT. @@ -188,6 +195,26 @@ t_adjust_keepalive_timer(_Config) -> ?assertEqual(undefined, emqx_gateway_cm:get_chan_info(ocpp, <<"client1">>)), ok. +t_auth_expire(_Config) -> + ok = meck:new(emqx_access_control, [passthrough, no_history]), + ok = meck:expect( + emqx_access_control, + authenticate, + fun(_) -> + {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}} + end + ), + + ?assertWaitEvent( + {ok, _ClientPid} = connect("127.0.0.1", 33033, <<"client1">>), + #{ + ?snk_kind := conn_process_terminated, + clientid := <<"client1">>, + reason := {shutdown, expired} + }, + 5000 + ). + t_listeners_status(_Config) -> {200, [Listener]} = request(get, "/gateways/ocpp/listeners"), ?assertMatch( diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index 71458f15e..ef33128d2 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -93,7 +93,8 @@ -define(TIMER_TABLE, #{ incoming_timer => keepalive, outgoing_timer => keepalive_send, - clean_trans_timer => clean_trans + clean_trans_timer => clean_trans, + connection_expire_timer => connection_expire }). -define(TRANS_TIMEOUT, 60000). @@ -356,10 +357,18 @@ ensure_connected( ) -> NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), - Channel#channel{ + schedule_connection_expire(Channel#channel{ conninfo = NConnInfo, conn_state = connected - }. + }). + +schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) -> + case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of + undefined -> + Channel; + Interval -> + ensure_timer(connection_expire_timer, Interval, Channel) + end. process_connect( Channel = #channel{ @@ -1137,7 +1146,10 @@ handle_timeout(_TRef, clean_trans, Channel = #channel{transaction = Trans}) -> end, Trans ), - {ok, ensure_clean_trans_timer(Channel#channel{transaction = NTrans})}. + {ok, ensure_clean_trans_timer(Channel#channel{transaction = NTrans})}; +handle_timeout(_TRef, connection_expire, Channel) -> + %% No session take over implemented, just shut down + shutdown(expired, Channel). %%-------------------------------------------------------------------- %% Terminate diff --git a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl index 64d95dc42..048e4f7ca 100644 --- a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl @@ -18,6 +18,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/asserts.hrl"). -include("emqx_stomp.hrl"). -compile(export_all). @@ -78,6 +79,14 @@ end_per_suite(Config) -> emqx_cth_suite:stop(?config(suite_apps, Config)), ok. +init_per_testcase(_TestCase, Config) -> + snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TestCase, _Config) -> + snabbkaffe:stop(), + ok. + default_config() -> ?CONF_DEFAULT. @@ -141,6 +150,34 @@ t_connect(_) -> end, with_connection(ProtocolError). +t_auth_expire(_) -> + ok = meck:new(emqx_access_control, [passthrough, no_history]), + ok = meck:expect( + emqx_access_control, + authenticate, + fun(_) -> + {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}} + end + ), + + ConnectWithExpire = fun(Sock) -> + ?assertWaitEvent( + begin + ok = send_connection_frame(Sock, <<"guest">>, <<"guest">>, <<"1000,2000">>), + {ok, Frame} = recv_a_frame(Sock), + ?assertMatch(<<"CONNECTED">>, Frame#stomp_frame.command) + end, + #{ + ?snk_kind := conn_process_terminated, + clientid := _, + reason := {shutdown, expired} + }, + 5000 + ) + end, + with_connection(ConnectWithExpire), + meck:unload(emqx_access_control). + t_heartbeat(_) -> %% Test heart beat with_connection(fun(Sock) -> diff --git a/apps/emqx_license/src/emqx_license.erl b/apps/emqx_license/src/emqx_license.erl index dfb747a96..73f0bdcd5 100644 --- a/apps/emqx_license/src/emqx_license.erl +++ b/apps/emqx_license/src/emqx_license.erl @@ -117,7 +117,9 @@ import_config(#{<<"license">> := Config}) -> {ok, #{root_key => license, changed => Changed1}}; Error -> {error, #{root_key => license, reason => Error}} - end. + end; +import_config(_RawConf) -> + {ok, #{root_key => license, changed => []}}. %%------------------------------------------------------------------------------ %% emqx_config_handler callbacks diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index ef42c0fc9..6b04e71f6 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -89,6 +89,7 @@ emqx_license, emqx_enterprise, emqx_message_validation, + emqx_connector_aggregator, emqx_bridge_kafka, emqx_bridge_pulsar, emqx_bridge_gcp_pubsub, diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 38320780d..a32397bb1 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -1745,6 +1745,18 @@ format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}, Opts) -> format_channel_info(undefined, {ClientId, PSInfo0 = #{}}, _Opts) -> format_persistent_session_info(ClientId, PSInfo0). +format_persistent_session_info( + _ClientId, #{metadata := #{offline_info := #{chan_info := ChanInfo, stats := Stats}}} = PSInfo +) -> + Info0 = format_channel_info(_Node = undefined, {_Key = undefined, ChanInfo, Stats}, #{ + fields => all + }), + Info0#{ + connected => false, + durable => true, + is_persistent => true, + subscriptions_cnt => maps:size(maps:get(subscriptions, PSInfo, #{})) + }; format_persistent_session_info(ClientId, PSInfo0) -> Metadata = maps:get(metadata, PSInfo0, #{}), {ProtoName, ProtoVer} = maps:get(protocol, Metadata), @@ -1762,6 +1774,7 @@ format_persistent_session_info(ClientId, PSInfo0) -> clientid => ClientId, connected => false, connected_at => CreatedAt, + durable => true, ip_address => IpAddress, is_persistent => true, port => Port, diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index b1a8fbce2..cc0e96ed6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -171,21 +171,32 @@ subscriptions(get, #{query_string := QString}) -> {200, Result} end. -format(WhichNode, {{Topic, _Subscriber}, SubOpts}) -> +format(WhichNode, {{Topic, Subscriber}, SubOpts}) -> + FallbackClientId = + case is_binary(Subscriber) of + true -> + Subscriber; + false -> + %% e.g.: could be a pid... + null + end, maps:merge( #{ topic => emqx_topic:maybe_format_share(Topic), - clientid => maps:get(subid, SubOpts, null), - node => WhichNode, + clientid => maps:get(subid, SubOpts, FallbackClientId), + node => convert_null(WhichNode), durable => false }, - maps:with([qos, nl, rap, rh], SubOpts) + maps:with([qos, nl, rap, rh, durable], SubOpts) ). %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +convert_null(undefined) -> null; +convert_null(Val) -> Val. + check_match_topic(#{<<"match_topic">> := MatchTopic}) -> try emqx_topic:parse(MatchTopic) of {#share{}, _} -> {error, invalid_match_topic}; diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 2623e6d4d..39d775a7a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -49,6 +49,7 @@ persistent_session_testcases() -> t_persistent_sessions3, t_persistent_sessions4, t_persistent_sessions5, + t_persistent_sessions_subscriptions1, t_list_clients_v2 ]. client_msgs_testcases() -> @@ -333,7 +334,7 @@ t_persistent_sessions2(Config) -> %% 2) Client connects to the same node and takes over, listed only once. C2 = connect_client(#{port => Port1, clientid => ClientId}), assert_single_client(O#{node => N1, clientid => ClientId, status => connected}), - ok = emqtt:disconnect(C2, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}), + disconnect_and_destroy_session(C2), ?retry( 100, 20, @@ -377,7 +378,7 @@ t_persistent_sessions3(Config) -> list_request(APIPort, "node=" ++ atom_to_list(N1)) ) ), - ok = emqtt:disconnect(C2, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}) + disconnect_and_destroy_session(C2) end, [] ), @@ -417,7 +418,7 @@ t_persistent_sessions4(Config) -> list_request(APIPort, "node=" ++ atom_to_list(N1)) ) ), - ok = emqtt:disconnect(C2, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}) + disconnect_and_destroy_session(C2) end, [] ), @@ -552,6 +553,63 @@ t_persistent_sessions5(Config) -> ), ok. +%% Check that the output of `/clients/:clientid/subscriptions' has the expected keys. +t_persistent_sessions_subscriptions1(Config) -> + [N1, _N2] = ?config(nodes, Config), + APIPort = 18084, + Port1 = get_mqtt_port(N1, tcp), + + ?assertMatch({ok, {{_, 200, _}, _, #{<<"data">> := []}}}, list_request(APIPort)), + + ?check_trace( + begin + ClientId = <<"c1">>, + C1 = connect_client(#{port => Port1, clientid => ClientId}), + {ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(C1, <<"topic/1">>, 1), + ?assertMatch( + {ok, + {{_, 200, _}, _, [ + #{ + <<"durable">> := true, + <<"node">> := <<_/binary>>, + <<"clientid">> := ClientId, + <<"qos">> := 1, + <<"rap">> := 0, + <<"rh">> := 0, + <<"nl">> := 0, + <<"topic">> := <<"topic/1">> + } + ]}}, + get_subscriptions_request(APIPort, ClientId) + ), + + %% Just disconnect + ok = emqtt:disconnect(C1), + ?assertMatch( + {ok, + {{_, 200, _}, _, [ + #{ + <<"durable">> := true, + <<"node">> := null, + <<"clientid">> := ClientId, + <<"qos">> := 1, + <<"rap">> := 0, + <<"rh">> := 0, + <<"nl">> := 0, + <<"topic">> := <<"topic/1">> + } + ]}}, + get_subscriptions_request(APIPort, ClientId) + ), + + C2 = connect_client(#{port => Port1, clientid => ClientId}), + disconnect_and_destroy_session(C2), + ok + end, + [] + ), + ok. + t_clients_bad_value_type(_) -> %% get /clients AuthHeader = [emqx_common_test_http:default_auth_header()], @@ -1800,6 +1858,16 @@ maybe_json_decode(X) -> {error, _} -> X end. +get_subscriptions_request(APIPort, ClientId) -> + Host = "http://127.0.0.1:" ++ integer_to_list(APIPort), + Path = emqx_mgmt_api_test_util:api_path(Host, ["clients", ClientId, "subscriptions"]), + request(get, Path, []). + +get_client_request(Port, ClientId) -> + Host = "http://127.0.0.1:" ++ integer_to_list(Port), + Path = emqx_mgmt_api_test_util:api_path(Host, ["clients", ClientId]), + request(get, Path, []). + list_request(Port) -> list_request(Port, _QueryParams = ""). @@ -1874,6 +1942,19 @@ assert_single_client(Opts) -> {ok, {{_, 200, _}, _, #{<<"connected">> := IsConnected}}}, lookup_request(ClientId, APIPort) ), + ?assertMatch( + {ok, + {{_, 200, _}, _, #{ + <<"connected">> := IsConnected, + <<"is_persistent">> := true, + %% contains statistics from disconnect time + <<"recv_pkt">> := _, + %% contains channel info from disconnect time + <<"listener">> := _, + <<"clean_start">> := _ + }}}, + get_client_request(APIPort, ClientId) + ), ok. connect_client(Opts) -> @@ -1937,3 +2018,6 @@ do_traverse_in_reverse_v2(APIPort, QueryParams0, [Cursor | Rest], DirectOrderCli {ok, {{_, 200, _}, _, #{<<"data">> := Rows}}} = Res0, ClientIds = [ClientId || #{<<"clientid">> := ClientId} <- Rows], do_traverse_in_reverse_v2(APIPort, QueryParams0, Rest, DirectOrderClientIds, ClientIds ++ Acc). + +disconnect_and_destroy_session(Client) -> + ok = emqtt:disconnect(Client, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}). diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index c5f5c475d..4daf1c51a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -290,54 +290,125 @@ t_http_test_json_formatter(_Config) -> end || JSONEntry <- LogEntries ], + ListIterFun = + fun + ListIterFunRec([]) -> + ok; + ListIterFunRec([Item | Rest]) -> + receive + From -> + From ! {list_iter_item, Item} + end, + ListIterFunRec(Rest) + end, + ListIter = spawn_link(fun() -> ListIterFun(DecodedLogEntries) end), + NextFun = + fun() -> + ListIter ! self(), + receive + {list_iter_item, Item} -> + Item + end + end, ?assertMatch( - [ - #{<<"meta">> := #{<<"payload">> := <<"log_this_message">>}}, - #{<<"meta">> := #{<<"payload">> := <<"\nlog\nthis\nmessage">>}}, - #{ - <<"meta">> := #{<<"payload">> := <<"\\\nlog\n_\\n_this\nmessage\\">>} - }, - #{<<"meta">> := #{<<"payload">> := <<"\"log_this_message\"">>}}, - #{<<"meta">> := #{<<"str">> := <<"str">>}}, - #{<<"meta">> := #{<<"term">> := <<"{notjson}">>}}, - #{<<"meta">> := <<_/binary>>}, - #{<<"meta">> := #{<<"integer">> := 42}}, - #{<<"meta">> := #{<<"float">> := 1.2}}, - #{<<"meta">> := <<_/binary>>}, - #{<<"meta">> := <<_/binary>>}, - #{<<"meta">> := <<_/binary>>}, - #{<<"meta">> := #{<<"sub">> := #{}}}, - #{<<"meta">> := #{<<"sub">> := #{<<"key">> := <<"value">>}}}, - #{<<"meta">> := #{<<"true">> := <<"true">>, <<"false">> := <<"false">>}}, - #{ - <<"meta">> := #{ - <<"list">> := #{ - <<"key">> := <<"value">>, - <<"key2">> := <<"value2">> - } - } - }, - #{ - <<"meta">> := #{ - <<"client_ids">> := [<<"a">>, <<"b">>, <<"c">>] - } - }, - #{ - <<"meta">> := #{ - <<"rule_ids">> := [<<"a">>, <<"b">>, <<"c">>] - } - }, - #{ - <<"meta">> := #{ - <<"action_info">> := #{ - <<"type">> := <<"http">>, - <<"name">> := <<"emqx_bridge_http_test_lib">> - } + #{<<"meta">> := #{<<"payload">> := <<"log_this_message">>}}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := #{<<"payload">> := <<"\nlog\nthis\nmessage">>}}, + NextFun() + ), + ?assertMatch( + #{ + <<"meta">> := #{<<"payload">> := <<"\\\nlog\n_\\n_this\nmessage\\">>} + }, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := #{<<"payload">> := <<"\"log_this_message\"">>}}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := #{<<"str">> := <<"str">>}}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := #{<<"term">> := <<"{notjson}">>}}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := <<_/binary>>}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := #{<<"integer">> := 42}}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := #{<<"float">> := 1.2}}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := <<_/binary>>}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := <<_/binary>>}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := <<_/binary>>}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := #{<<"sub">> := #{}}}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := #{<<"sub">> := #{<<"key">> := <<"value">>}}}, + NextFun() + ), + ?assertMatch( + #{<<"meta">> := #{<<"true">> := true, <<"false">> := false}}, + NextFun() + ), + ?assertMatch( + #{ + <<"meta">> := #{ + <<"list">> := #{ + <<"key">> := <<"value">>, + <<"key2">> := <<"value2">> } } - | _ - ], - DecodedLogEntries + }, + NextFun() + ), + ?assertMatch( + #{ + <<"meta">> := #{ + <<"client_ids">> := [<<"a">>, <<"b">>, <<"c">>] + } + }, + NextFun() + ), + ?assertMatch( + #{ + <<"meta">> := #{ + <<"rule_ids">> := [<<"a">>, <<"b">>, <<"c">>] + } + }, + NextFun() + ), + ?assertMatch( + #{ + <<"meta">> := #{ + <<"action_info">> := #{ + <<"type">> := <<"http">>, + <<"name">> := <<"emqx_bridge_http_test_lib">> + } + } + }, + NextFun() ), {ok, Delete} = request_api(delete, api_path("trace/" ++ binary_to_list(Name))), ?assertEqual(<<>>, Delete), @@ -495,7 +566,7 @@ create_trace(Name, Type, TypeValue, Start) -> ?block_until(#{?snk_kind := update_trace_done}) end, fun(Trace) -> - ?assertMatch([#{}], ?of_kind(update_trace_done, Trace)) + ?assertMatch([#{} | _], ?of_kind(update_trace_done, Trace)) end ). diff --git a/apps/emqx_message_validation/src/emqx_message_validation.erl b/apps/emqx_message_validation/src/emqx_message_validation.erl index 7e0d2fd11..8f0b41d2f 100644 --- a/apps/emqx_message_validation/src/emqx_message_validation.erl +++ b/apps/emqx_message_validation/src/emqx_message_validation.erl @@ -69,10 +69,22 @@ remove_handler() -> ok. load() -> - lists:foreach(fun insert/1, emqx:get_config(?VALIDATIONS_CONF_PATH, [])). + Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []), + lists:foreach( + fun({Pos, Validation}) -> + ok = emqx_message_validation_registry:insert(Pos, Validation) + end, + lists:enumerate(Validations) + ). unload() -> - lists:foreach(fun delete/1, emqx:get_config(?VALIDATIONS_CONF_PATH, [])). + Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []), + lists:foreach( + fun(Validation) -> + ok = emqx_message_validation_registry:delete(Validation) + end, + Validations + ). -spec list() -> [validation()]. list() -> @@ -81,7 +93,7 @@ list() -> -spec reorder([validation_name()]) -> {ok, _} | {error, _}. reorder(Order) -> - emqx:update_config( + emqx_conf:update( ?VALIDATIONS_CONF_PATH, {reorder, Order}, #{override_to => cluster} @@ -95,7 +107,7 @@ lookup(Name) -> -spec insert(validation()) -> {ok, _} | {error, _}. insert(Validation) -> - emqx:update_config( + emqx_conf:update( ?VALIDATIONS_CONF_PATH, {append, Validation}, #{override_to => cluster} @@ -104,7 +116,7 @@ insert(Validation) -> -spec update(validation()) -> {ok, _} | {error, _}. update(Validation) -> - emqx:update_config( + emqx_conf:update( ?VALIDATIONS_CONF_PATH, {update, Validation}, #{override_to => cluster} @@ -113,7 +125,7 @@ update(Validation) -> -spec delete(validation_name()) -> {ok, _} | {error, _}. delete(Name) -> - emqx:update_config( + emqx_conf:update( ?VALIDATIONS_CONF_PATH, {delete, Name}, #{override_to => cluster} @@ -247,8 +259,8 @@ evaluate_schema_check(Check, Validation, #message{payload = Data}) -> #{name := Name} = Validation, ExtraArgs = case Check of - #{type := protobuf, message_name := MessageName} -> - [MessageName]; + #{type := protobuf, message_type := MessageType} -> + [MessageType]; _ -> [] end, diff --git a/apps/emqx_message_validation/src/emqx_message_validation_schema.erl b/apps/emqx_message_validation/src/emqx_message_validation_schema.erl index f151cdf62..3c56c5ada 100644 --- a/apps/emqx_message_validation/src/emqx_message_validation_schema.erl +++ b/apps/emqx_message_validation/src/emqx_message_validation_schema.erl @@ -18,8 +18,6 @@ api_schema/1 ]). --export([validate_name/1]). - %%------------------------------------------------------------------------------ %% Type declarations %%------------------------------------------------------------------------------ @@ -55,7 +53,7 @@ fields(validation) -> binary(), #{ required => true, - validator => fun validate_name/1, + validator => fun emqx_resource:validate_name/1, desc => ?DESC("name") } )}, @@ -123,8 +121,8 @@ fields(check_protobuf) -> [ {type, mk(protobuf, #{default => protobuf, desc => ?DESC("check_protobuf_type")})}, {schema, mk(binary(), #{required => true, desc => ?DESC("check_protobuf_schema")})}, - {message_name, - mk(binary(), #{required => true, desc => ?DESC("check_protobuf_message_name")})} + {message_type, + mk(binary(), #{required => true, desc => ?DESC("check_protobuf_message_type")})} ]; fields(check_avro) -> [ @@ -200,16 +198,6 @@ ensure_array(undefined, _) -> undefined; ensure_array(L, _) when is_list(L) -> L; ensure_array(B, _) -> [B]. -validate_name(Name) -> - %% see `MAP_KEY_RE' in hocon_tconf - RE = <<"^[A-Za-z0-9]+[A-Za-z0-9-_]*$">>, - case re:run(Name, RE, [{capture, none}]) of - match -> - ok; - nomatch -> - {error, <<"must conform to regex: ", RE/binary>>} - end. - validate_sql(SQL) -> case emqx_message_validation:parse_sql_check(SQL) of {ok, _Parsed} -> diff --git a/apps/emqx_message_validation/test/emqx_message_validation_http_api_SUITE.erl b/apps/emqx_message_validation/test/emqx_message_validation_http_api_SUITE.erl index 8bd2ae647..879da797c 100644 --- a/apps/emqx_message_validation/test/emqx_message_validation_http_api_SUITE.erl +++ b/apps/emqx_message_validation/test/emqx_message_validation_http_api_SUITE.erl @@ -317,9 +317,9 @@ avro_create_serde(SerdeName) -> on_exit(fun() -> ok = emqx_schema_registry:delete_schema(SerdeName) end), ok. -protobuf_valid_payloads(SerdeName, MessageName) -> +protobuf_valid_payloads(SerdeName, MessageType) -> lists:map( - fun(Payload) -> emqx_schema_registry_serde:encode(SerdeName, Payload, [MessageName]) end, + fun(Payload) -> emqx_schema_registry_serde:encode(SerdeName, Payload, [MessageType]) end, [ #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>}, #{<<"name">> => <<"some name">>, <<"id">> => 10} @@ -1176,11 +1176,11 @@ t_schema_check_avro(_Config) -> t_schema_check_protobuf(_Config) -> SerdeName = <<"myserde">>, - MessageName = <<"Person">>, + MessageType = <<"Person">>, protobuf_create_serde(SerdeName), Name1 = <<"foo">>, - Check1 = schema_check(protobuf, SerdeName, #{<<"message_name">> => MessageName}), + Check1 = schema_check(protobuf, SerdeName, #{<<"message_type">> => MessageType}), Validation1 = validation(Name1, [Check1]), {201, _} = insert(Validation1), @@ -1192,7 +1192,7 @@ t_schema_check_protobuf(_Config) -> ok = publish(C, <<"t/1">>, {raw, Payload}), ?assertReceive({publish, _}) end, - protobuf_valid_payloads(SerdeName, MessageName) + protobuf_valid_payloads(SerdeName, MessageType) ), lists:foreach( fun(Payload) -> @@ -1203,7 +1203,7 @@ t_schema_check_protobuf(_Config) -> ), %% Bad config: unknown message name - Check2 = schema_check(protobuf, SerdeName, #{<<"message_name">> => <<"idontexist">>}), + Check2 = schema_check(protobuf, SerdeName, #{<<"message_type">> => <<"idontexist">>}), Validation2 = validation(Name1, [Check2]), {200, _} = update(Validation2), @@ -1212,7 +1212,7 @@ t_schema_check_protobuf(_Config) -> ok = publish(C, <<"t/1">>, {raw, Payload}), ?assertNotReceive({publish, _}) end, - protobuf_valid_payloads(SerdeName, MessageName) + protobuf_valid_payloads(SerdeName, MessageType) ), ok. diff --git a/apps/emqx_message_validation/test/emqx_message_validation_tests.erl b/apps/emqx_message_validation/test/emqx_message_validation_tests.erl index 7c3dfc9d8..2c33a20ba 100644 --- a/apps/emqx_message_validation/test/emqx_message_validation_tests.erl +++ b/apps/emqx_message_validation/test/emqx_message_validation_tests.erl @@ -195,7 +195,6 @@ invalid_names_test_() -> ?_assertThrow( {_Schema, [ #{ - reason := <<"must conform to regex:", _/binary>>, kind := validation_error, path := "message_validation.validations.1.name" } @@ -209,7 +208,9 @@ invalid_names_test_() -> <<"name!">>, <<"some name">>, <<"nãme"/utf8>>, - <<"test_哈哈"/utf8>> + <<"test_哈哈"/utf8>>, + %% long name + binary:copy(<<"a">>, 256) ] ]. @@ -308,7 +309,7 @@ duplicated_check_test_() -> schema_check( protobuf, <<"a">>, - #{<<"message_name">> => <<"a">>} + #{<<"message_type">> => <<"a">>} ) ]) ]) diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index ff851558a..3ad2fb564 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -30,7 +30,8 @@ on_stop/2, on_query/3, on_batch_query/3, - on_get_status/2 + on_get_status/2, + on_format_query_result/1 ]). %% ecpool connect & reconnect @@ -214,6 +215,13 @@ on_batch_query( }), {error, {unrecoverable_error, invalid_request}}. +on_format_query_result({ok, ColumnNames, Rows}) -> + #{result => ok, column_names => ColumnNames, rows => Rows}; +on_format_query_result({ok, DataList}) -> + #{result => ok, column_names_rows_list => DataList}; +on_format_query_result(Result) -> + Result. + mysql_function(sql) -> query; mysql_function(prepared_query) -> diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index 761c9c0f6..ad674a07c 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -38,7 +38,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - on_get_channel_status/3 + on_get_channel_status/3, + on_format_query_result/1 ]). -export([connect/1]). @@ -695,6 +696,11 @@ handle_result({error, Error}) -> handle_result(Res) -> Res. +on_format_query_result({ok, Cnt}) when is_integer(Cnt) -> + #{result => ok, affected_rows => Cnt}; +on_format_query_result(Res) -> + Res. + handle_batch_result([{ok, Count} | Rest], Acc) -> handle_batch_result(Rest, Acc + Count); handle_batch_result([{error, Error} | _Rest], _Acc) -> diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index eba0d33af..8340bf589 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -86,7 +86,9 @@ forget_allocated_resources/1, deallocate_resource/2, %% Get channel config from resource - call_get_channel_config/3 + call_get_channel_config/3, + % Call the format query result function + call_format_query_result/2 ]). %% Direct calls to the callback module @@ -154,7 +156,8 @@ on_add_channel/4, on_remove_channel/3, on_get_channels/1, - query_mode/1 + query_mode/1, + on_format_query_result/1 ]). %% when calling emqx_resource:start/1 @@ -230,6 +233,14 @@ ResId :: term() ) -> [term()]. +%% When given the result of a on_*query call this function should return a +%% version of the result that is suitable for JSON trace logging. This +%% typically means converting Erlang tuples to maps with appropriate names for +%% the values in the tuple. +-callback on_format_query_result( + QueryResult :: term() +) -> term(). + -define(SAFE_CALL(EXPR), (fun() -> try @@ -551,6 +562,14 @@ call_get_channel_config(ResId, ChannelId, Mod) -> <<"on_get_channels callback function not available for resource id", ResId/binary>>} end. +call_format_query_result(Mod, Result) -> + case erlang:function_exported(Mod, on_format_query_result, 1) of + true -> + Mod:on_format_query_result(Result); + false -> + Result + end. + -spec call_stop(resource_id(), module(), resource_state()) -> term(). call_stop(ResId, Mod, ResourceState) -> ?SAFE_CALL(begin diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index e35453c94..c610df76c 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -68,7 +68,7 @@ {query, FROM, REQUEST, SENT, EXPIRE_AT, TRACE_CTX} ). -define(SIMPLE_QUERY(FROM, REQUEST, TRACE_CTX), ?QUERY(FROM, REQUEST, false, infinity, TRACE_CTX)). --define(REPLY(FROM, SENT, RESULT), {reply, FROM, SENT, RESULT}). +-define(REPLY(FROM, SENT, RESULT, TRACE_CTX), {reply, FROM, SENT, RESULT, TRACE_CTX}). -define(INFLIGHT_ITEM(Ref, BatchOrQuery, IsRetriable, AsyncWorkerMRef), {Ref, BatchOrQuery, IsRetriable, AsyncWorkerMRef} ). @@ -448,8 +448,8 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> Result = call_query(force_sync, Id, Index, Ref, QueryOrBatch, QueryOpts), {ShouldAck, PostFn, DeltaCounters} = case QueryOrBatch of - ?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt, _TraceCtx) -> - Reply = ?REPLY(ReplyTo, HasBeenSent, Result), + ?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt, TraceCtx) -> + Reply = ?REPLY(ReplyTo, HasBeenSent, Result, TraceCtx), reply_caller_defer_metrics(Id, Reply, QueryOpts); [?QUERY(_, _, _, _, _) | _] = Batch -> batch_reply_caller_defer_metrics(Id, Result, Batch, QueryOpts) @@ -662,10 +662,10 @@ do_flush( inflight_tid := InflightTID } = Data0, %% unwrap when not batching (i.e., batch size == 1) - [?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt, _TraceCtx) = Request] = Batch, + [?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt, TraceCtx) = Request] = Batch, QueryOpts = #{inflight_tid => InflightTID, simple_query => false}, Result = call_query(async_if_possible, Id, Index, Ref, Request, QueryOpts), - Reply = ?REPLY(ReplyTo, HasBeenSent, Result), + Reply = ?REPLY(ReplyTo, HasBeenSent, Result, TraceCtx), {ShouldAck, DeltaCounters} = reply_caller(Id, Reply, QueryOpts), Data1 = aggregate_counters(Data0, DeltaCounters), case ShouldAck of @@ -856,15 +856,15 @@ batch_reply_caller_defer_metrics(Id, BatchResult, Batch, QueryOpts) -> expand_batch_reply(BatchResults, Batch) when is_list(BatchResults) -> lists:map( - fun({?QUERY(FROM, _REQUEST, SENT, _EXPIRE_AT, _TraceCtx), Result}) -> - ?REPLY(FROM, SENT, Result) + fun({?QUERY(FROM, _REQUEST, SENT, _EXPIRE_AT, TraceCtx), Result}) -> + ?REPLY(FROM, SENT, Result, TraceCtx) end, lists:zip(Batch, BatchResults) ); expand_batch_reply(BatchResult, Batch) -> lists:map( - fun(?QUERY(FROM, _REQUEST, SENT, _EXPIRE_AT, _TraceCtx)) -> - ?REPLY(FROM, SENT, BatchResult) + fun(?QUERY(FROM, _REQUEST, SENT, _EXPIRE_AT, TraceCtx)) -> + ?REPLY(FROM, SENT, BatchResult, TraceCtx) end, Batch ). @@ -876,12 +876,14 @@ reply_caller(Id, Reply, QueryOpts) -> %% Should only reply to the caller when the decision is final (not %% retriable). See comment on `handle_query_result_pure'. -reply_caller_defer_metrics(Id, ?REPLY(undefined, HasBeenSent, Result), _QueryOpts) -> - handle_query_result_pure(Id, Result, HasBeenSent); -reply_caller_defer_metrics(Id, ?REPLY(ReplyTo, HasBeenSent, Result), QueryOpts) -> +reply_caller_defer_metrics(Id, ?REPLY(undefined, HasBeenSent, Result, TraceCtx), _QueryOpts) -> + handle_query_result_pure(Id, Result, HasBeenSent, TraceCtx); +reply_caller_defer_metrics(Id, ?REPLY(ReplyTo, HasBeenSent, Result, TraceCtx), QueryOpts) -> IsSimpleQuery = maps:get(simple_query, QueryOpts, false), IsUnrecoverableError = is_unrecoverable_error(Result), - {ShouldAck, PostFn, DeltaCounters} = handle_query_result_pure(Id, Result, HasBeenSent), + {ShouldAck, PostFn, DeltaCounters} = handle_query_result_pure( + Id, Result, HasBeenSent, TraceCtx + ), case {ShouldAck, Result, IsUnrecoverableError, IsSimpleQuery} of {ack, {async_return, _}, true, _} -> ok = do_reply_caller(ReplyTo, Result); @@ -921,7 +923,7 @@ batch_reply_dropped(Batch, Result) -> %% This is only called by `simple_{,a}sync_query', so we can bump the %% counters here. handle_query_result(Id, Result, HasBeenSent) -> - {ShouldBlock, PostFn, DeltaCounters} = handle_query_result_pure(Id, Result, HasBeenSent), + {ShouldBlock, PostFn, DeltaCounters} = handle_query_result_pure(Id, Result, HasBeenSent, #{}), PostFn(), bump_counters(Id, DeltaCounters), ShouldBlock. @@ -932,37 +934,49 @@ handle_query_result(Id, Result, HasBeenSent) -> %% * the result is a success (or at least a delayed result) %% We also retry even sync requests. In that case, we shouldn't reply %% the caller until one of those final results above happen. --spec handle_query_result_pure(id(), term(), HasBeenSent :: boolean()) -> +-spec handle_query_result_pure(id(), term(), HasBeenSent :: boolean(), TraceCTX :: map()) -> {ack | nack, function(), counters()}. -handle_query_result_pure(_Id, ?RESOURCE_ERROR_M(exception, Msg), _HasBeenSent) -> +handle_query_result_pure(_Id, ?RESOURCE_ERROR_M(exception, Msg), _HasBeenSent, TraceCTX) -> PostFn = fun() -> - ?SLOG(error, #{msg => "resource_exception", info => emqx_utils:redact(Msg)}), + ?TRACE( + error, + "ERROR", + "resource_exception", + (trace_ctx_map(TraceCTX))#{info => emqx_utils:redact(Msg)} + ), ok end, {nack, PostFn, #{}}; -handle_query_result_pure(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _HasBeenSent) when +handle_query_result_pure(_Id, ?RESOURCE_ERROR_M(NotWorking, _), _HasBeenSent, _TraceCTX) when NotWorking == not_connected; NotWorking == blocked -> {nack, fun() -> ok end, #{}}; -handle_query_result_pure(Id, ?RESOURCE_ERROR_M(not_found, Msg), _HasBeenSent) -> +handle_query_result_pure(Id, ?RESOURCE_ERROR_M(not_found, Msg), _HasBeenSent, TraceCTX) -> PostFn = fun() -> - ?SLOG(error, #{id => Id, msg => "resource_not_found", info => Msg}), + ?TRACE( + error, + "ERROR", + "resource_not_found", + (trace_ctx_map(TraceCTX))#{id => Id, info => Msg} + ), ok end, {ack, PostFn, #{dropped_resource_not_found => 1}}; -handle_query_result_pure(Id, ?RESOURCE_ERROR_M(stopped, Msg), _HasBeenSent) -> +handle_query_result_pure(Id, ?RESOURCE_ERROR_M(stopped, Msg), _HasBeenSent, TraceCTX) -> PostFn = fun() -> - ?SLOG(error, #{id => Id, msg => "resource_stopped", info => Msg}), + ?TRACE(error, "ERROR", "resource_stopped", (trace_ctx_map(TraceCTX))#{id => Id, info => Msg}), ok end, {ack, PostFn, #{dropped_resource_stopped => 1}}; -handle_query_result_pure(Id, ?RESOURCE_ERROR_M(Reason, _), _HasBeenSent) -> +handle_query_result_pure(Id, ?RESOURCE_ERROR_M(Reason, _), _HasBeenSent, TraceCTX) -> PostFn = fun() -> - ?SLOG(error, #{id => Id, msg => "other_resource_error", reason => Reason}), + ?TRACE(error, "ERROR", "other_resource_error", (trace_ctx_map(TraceCTX))#{ + id => Id, reason => Reason + }), ok end, {nack, PostFn, #{}}; -handle_query_result_pure(Id, {error, Reason} = Error, HasBeenSent) -> +handle_query_result_pure(Id, {error, Reason} = Error, HasBeenSent, TraceCTX) -> case is_unrecoverable_error(Error) of true -> PostFn = @@ -979,14 +993,16 @@ handle_query_result_pure(Id, {error, Reason} = Error, HasBeenSent) -> false -> PostFn = fun() -> - ?SLOG(error, #{id => Id, msg => "send_error", reason => Reason}), + ?TRACE(error, "ERROR", "send_error", (trace_ctx_map(TraceCTX))#{ + id => Id, reason => Reason + }), ok end, {nack, PostFn, #{}} end; -handle_query_result_pure(Id, {async_return, Result}, HasBeenSent) -> - handle_query_async_result_pure(Id, Result, HasBeenSent); -handle_query_result_pure(_Id, Result, HasBeenSent) -> +handle_query_result_pure(Id, {async_return, Result}, HasBeenSent, TraceCTX) -> + handle_query_async_result_pure(Id, Result, HasBeenSent, TraceCTX); +handle_query_result_pure(_Id, Result, HasBeenSent, _TraceCTX) -> PostFn = fun() -> assert_ok_result(Result), ok @@ -998,9 +1014,9 @@ handle_query_result_pure(_Id, Result, HasBeenSent) -> end, {ack, PostFn, Counters}. --spec handle_query_async_result_pure(id(), term(), HasBeenSent :: boolean()) -> +-spec handle_query_async_result_pure(id(), term(), HasBeenSent :: boolean(), map()) -> {ack | nack, function(), counters()}. -handle_query_async_result_pure(Id, {error, Reason} = Error, HasBeenSent) -> +handle_query_async_result_pure(Id, {error, Reason} = Error, HasBeenSent, TraceCTX) -> case is_unrecoverable_error(Error) of true -> PostFn = @@ -1016,16 +1032,18 @@ handle_query_async_result_pure(Id, {error, Reason} = Error, HasBeenSent) -> {ack, PostFn, Counters}; false -> PostFn = fun() -> - ?SLOG(error, #{id => Id, msg => "async_send_error", reason => Reason}), + ?TRACE(error, "ERROR", "async_send_error", (trace_ctx_map(TraceCTX))#{ + id => Id, reason => Reason + }), ok end, {nack, PostFn, #{}} end; -handle_query_async_result_pure(_Id, {ok, Pid}, _HasBeenSent) when is_pid(Pid) -> +handle_query_async_result_pure(_Id, {ok, Pid}, _HasBeenSent, _TraceCTX) when is_pid(Pid) -> {ack, fun() -> ok end, #{}}; -handle_query_async_result_pure(_Id, ok, _HasBeenSent) -> +handle_query_async_result_pure(_Id, ok, _HasBeenSent, _TraceCTX) -> {ack, fun() -> ok end, #{}}; -handle_query_async_result_pure(Id, Results, HasBeenSent) when is_list(Results) -> +handle_query_async_result_pure(Id, Results, HasBeenSent, TraceCTX) when is_list(Results) -> All = fun(L) -> case L of {ok, Pid} -> is_pid(Pid); @@ -1037,17 +1055,26 @@ handle_query_async_result_pure(Id, Results, HasBeenSent) when is_list(Results) - {ack, fun() -> ok end, #{}}; false -> PostFn = fun() -> - ?SLOG(error, #{ - id => Id, - msg => "async_batch_send_error", - reason => Results, - has_been_sent => HasBeenSent - }), + ?TRACE( + error, + "ERROR", + "async_batch_send_error", + (trace_ctx_map(TraceCTX))#{ + id => Id, + reason => Results, + has_been_sent => HasBeenSent + } + ), ok end, {nack, PostFn, #{}} end. +trace_ctx_map(undefined) -> + #{}; +trace_ctx_map(Map) -> + Map. + -spec aggregate_counters(data(), counters()) -> data(). aggregate_counters(Data = #{counters := OldCounters}, DeltaCounters) -> Counters = merge_counters(OldCounters, DeltaCounters), @@ -1526,7 +1553,7 @@ do_handle_async_reply( request_ref := Ref, buffer_worker := BufferWorkerPid, inflight_tid := InflightTID, - min_query := ?QUERY(ReplyTo, _, Sent, _ExpireAt, _TraceCtx) = _Query + min_query := ?QUERY(ReplyTo, _, Sent, _ExpireAt, TraceCtx) = _Query }, Result ) -> @@ -1534,7 +1561,7 @@ do_handle_async_reply( %% but received no ACK, NOT the number of messages queued in the %% inflight window. {Action, PostFn, DeltaCounters} = reply_caller_defer_metrics( - Id, ?REPLY(ReplyTo, Sent, Result), QueryOpts + Id, ?REPLY(ReplyTo, Sent, Result, TraceCtx), QueryOpts ), ?tp(handle_async_reply, #{ diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 4d57d4c2a..dcd9024ef 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -104,6 +104,20 @@ pre_process_action_args(_, Args) -> %%-------------------------------------------------------------------- -spec console(map(), map(), map()) -> any(). console(Selected, #{metadata := #{rule_id := RuleId}} = Envs, _Args) -> + case logger:get_process_metadata() of + #{action_id := ActionID} -> + emqx_trace:rendered_action_template( + ActionID, + #{ + selected => Selected, + environment => Envs + } + ); + _ -> + %% We may not have an action ID in the metadata if this is called + %% from a test case or similar + ok + end, ?ULOG( "[rule action] ~ts~n" "\tAction Data: ~p~n" @@ -149,15 +163,24 @@ republish( PubProps0 = render_pub_props(UserPropertiesTemplate, Selected, Env), MQTTProps = render_mqtt_properties(MQTTPropertiesTemplate, Selected, Env), PubProps = maps:merge(PubProps0, MQTTProps), + TraceInfo = #{ + flags => Flags, + topic => Topic, + payload => Payload, + pub_props => PubProps + }, + case logger:get_process_metadata() of + #{action_id := ActionID} -> + emqx_trace:rendered_action_template(ActionID, TraceInfo); + _ -> + %% We may not have an action ID in the metadata if this is called + %% from a test case or similar + ok + end, ?TRACE( "RULE", "republish_message", - #{ - flags => Flags, - topic => Topic, - payload => Payload, - pub_props => PubProps - } + TraceInfo ), safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index d203dd915..3dd8048d7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -274,6 +274,7 @@ schema("/rules/:id/test") -> responses => #{ 400 => error_schema('BAD_REQUEST', "Invalid Parameters"), 412 => error_schema('NOT_MATCH', "SQL Not Match"), + 404 => error_schema('RULE_NOT_FOUND', "The rule could not be found"), 200 => <<"Rule Applied">> } } @@ -419,11 +420,13 @@ param_path_id() -> begin case emqx_rule_sqltester:apply_rule(RuleId, CheckedParams) of {ok, Result} -> - {200, Result}; + {200, emqx_logger_jsonfmt:best_effort_json_obj(Result)}; {error, {parse_error, Reason}} -> {400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}}; {error, nomatch} -> {412, #{code => 'NOT_MATCH', message => <<"SQL Not Match">>}}; + {error, rule_not_found} -> + {404, #{code => 'RULE_NOT_FOUND', message => <<"The rule could not be found">>}}; {error, Reason} -> {400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}} end diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 5ec4bdc6e..9a5de7871 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -415,6 +415,15 @@ handle_action(RuleId, ActId, Selected, Envs) -> rule_metrics, RuleId, 'actions.failed.out_of_service' ), trace_action(ActId, "out_of_service", #{}, warning); + error:?EMQX_TRACE_STOP_ACTION_MATCH = Reason -> + ?EMQX_TRACE_STOP_ACTION(Explanation) = Reason, + trace_action( + ActId, + "action_stopped_after_template_rendering", + #{reason => Explanation} + ), + emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'), + emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown'); Err:Reason:ST -> ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'), ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown'), @@ -475,7 +484,18 @@ do_handle_action(RuleId, #{mod := Mod, func := Func} = Action, Selected, Envs) - trace_action(Action, "call_action_function"), %% the function can also throw 'out_of_service' Args = maps:get(args, Action, []), - Result = Mod:Func(Selected, Envs, Args), + PrevProcessMetadata = + case logger:get_process_metadata() of + undefined -> #{}; + D -> D + end, + Result = + try + logger:update_process_metadata(#{action_id => Action}), + Mod:Func(Selected, Envs, Args) + after + logger:set_process_metadata(PrevProcessMetadata) + end, {_, IncCtx} = do_handle_action_get_trace_inc_metrics_context(RuleId, Action), inc_action_metrics(IncCtx, Result), Result. @@ -737,31 +757,63 @@ do_inc_action_metrics( emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown'); do_inc_action_metrics( #{rule_id := RuleId, action_id := ActId} = TraceContext, - {error, {recoverable_error, _}} + {error, {recoverable_error, _}} = Reason ) -> + FormatterRes = #emqx_trace_format_func_data{ + function = fun trace_formatted_result/1, + data = {ActId, Reason} + }, TraceContext1 = maps:remove(action_id, TraceContext), - trace_action(ActId, "out_of_service", TraceContext1), + trace_action(ActId, "out_of_service", TraceContext1#{reason => FormatterRes}), emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'); do_inc_action_metrics( #{rule_id := RuleId, action_id := ActId} = TraceContext, - {error, {unrecoverable_error, _} = Reason} + {error, {unrecoverable_error, _}} = Reason ) -> TraceContext1 = maps:remove(action_id, TraceContext), - trace_action(ActId, "action_failed", maps:merge(#{reason => Reason}, TraceContext1)), + FormatterRes = #emqx_trace_format_func_data{ + function = fun trace_formatted_result/1, + data = {ActId, Reason} + }, + trace_action(ActId, "action_failed", maps:merge(#{reason => FormatterRes}, TraceContext1)), emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'), emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown'); do_inc_action_metrics(#{rule_id := RuleId, action_id := ActId} = TraceContext, R) -> TraceContext1 = maps:remove(action_id, TraceContext), + FormatterRes = #emqx_trace_format_func_data{ + function = fun trace_formatted_result/1, + data = {ActId, R} + }, case is_ok_result(R) of false -> - trace_action(ActId, "action_failed", maps:merge(#{reason => R}, TraceContext1)), + trace_action( + ActId, + "action_failed", + maps:merge(#{reason => FormatterRes}, TraceContext1) + ), emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed'), emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.unknown'); true -> - trace_action(ActId, "action_success", maps:merge(#{result => R}, TraceContext1)), + trace_action( + ActId, + "action_success", + maps:merge(#{result => FormatterRes}, TraceContext1) + ), emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success') end. +trace_formatted_result({{bridge_v2, Type, _Name}, R}) -> + ConnectorType = emqx_action_info:action_type_to_connector_type(Type), + ResourceModule = emqx_connector_info:resource_callback_module(ConnectorType), + clean_up_error_tuple(emqx_resource:call_format_query_result(ResourceModule, R)); +trace_formatted_result({{bridge, BridgeType, _BridgeName, _ResId}, R}) -> + BridgeV2Type = emqx_action_info:bridge_v1_type_to_action_type(BridgeType), + ConnectorType = emqx_action_info:action_type_to_connector_type(BridgeV2Type), + ResourceModule = emqx_connector_info:resource_callback_module(ConnectorType), + clean_up_error_tuple(emqx_resource:call_format_query_result(ResourceModule, R)); +trace_formatted_result({_, R}) -> + R. + is_ok_result(ok) -> true; is_ok_result({async_return, R}) -> @@ -771,6 +823,15 @@ is_ok_result(R) when is_tuple(R) -> is_ok_result(_) -> false. +clean_up_error_tuple({error, {unrecoverable_error, Reason}}) -> + Reason; +clean_up_error_tuple({error, {recoverable_error, Reason}}) -> + Reason; +clean_up_error_tuple({error, Reason}) -> + Reason; +clean_up_error_tuple(Result) -> + Result. + parse_module_name(Name) when is_binary(Name) -> case ?IS_VALID_SQL_FUNC_PROVIDER_MODULE_NAME(Name) of true -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 83f29eef3..6d393c24a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -26,12 +26,22 @@ apply_rule( RuleId, + Parameters +) -> + case emqx_rule_engine:get_rule(RuleId) of + {ok, Rule} -> + do_apply_rule(Rule, Parameters); + not_found -> + {error, rule_not_found} + end. + +do_apply_rule( + Rule, #{ context := Context, stop_action_after_template_rendering := StopAfterRender } ) -> - {ok, Rule} = emqx_rule_engine:get_rule(RuleId), InTopic = get_in_topic(Context), EventTopics = maps:get(from, Rule, []), case lists:all(fun is_publish_topic/1, EventTopics) of diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index b0ca00a0e..a84ead1c2 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -118,7 +118,8 @@ groups() -> t_event_client_disconnected_normal, t_event_client_disconnected_kicked, t_event_client_disconnected_discarded, - t_event_client_disconnected_takenover + t_event_client_disconnected_takenover, + t_event_client_disconnected_takenover_2 ]}, {telemetry, [], [ t_get_basic_usage_info_0, @@ -983,6 +984,66 @@ t_event_client_disconnected_takenover(_Config) -> delete_rule(TopicRule). +t_event_client_disconnected_takenover_2(_Config) -> + SQL = + "select * " + "from \"$events/client_disconnected\" ", + RepubT = <<"repub/to/disconnected/takenover">>, + + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{ + sql => SQL, + id => ?TMP_RULEID, + actions => [republish_action(RepubT, <<>>)] + } + ), + + {ok, ClientRecv} = emqtt:start_link([ + {clientid, <<"get_repub_client">>}, {username, <<"emqx0">>} + ]), + {ok, _} = emqtt:connect(ClientRecv), + {ok, _, _} = emqtt:subscribe(ClientRecv, RepubT, 0), + ct:sleep(200), + + {ok, Client1} = emqtt:start_link([ + {clientid, <<"emqx">>}, {username, <<"emqx">>}, {clean_start, false} + ]), + {ok, _} = emqtt:connect(Client1), + ok = emqtt:disconnect(Client1), + + %% receive the normal disconnected event + receive + {publish, #{topic := T, payload := Payload}} -> + ?assertEqual(RepubT, T), + ?assertMatch( + #{<<"reason">> := <<"normal">>}, emqx_utils_json:decode(Payload, [return_maps]) + ) + after 1000 -> + ct:fail(wait_for_repub_disconnected_discarded) + end, + + {ok, Client2} = emqtt:start_link([ + {clientid, <<"emqx">>}, {username, <<"emqx">>}, {clean_start, false} + ]), + {ok, _} = emqtt:connect(Client2), + + %% should not receive the takenoverdisconnected event + receive + {publish, #{topic := T1, payload := Payload1}} -> + ?assertEqual(RepubT, T1), + ?assertMatch( + #{<<"reason">> := <<"takenover">>}, emqx_utils_json:decode(Payload1, [return_maps]) + ), + ct:fail(wait_for_repub_disconnected_discarded) + after 1000 -> + ok + end, + + emqtt:stop(ClientRecv), + emqtt:stop(Client2), + + delete_rule(TopicRule). + client_connack_failed() -> {ok, Client} = emqtt:start_link( [ diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_apply_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_apply_SUITE.erl index 52fa1a2e5..9f03a7bda 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_apply_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_apply_SUITE.erl @@ -26,7 +26,24 @@ -define(CONF_DEFAULT, <<"rule_engine {rules {}}">>). all() -> - emqx_common_test_helpers:all(?MODULE). + [ + emqx_common_test_helpers:all(?MODULE), + {group, republish}, + {group, console_print} + ]. + +groups() -> + [ + {republish, [], basic_tests()}, + {console_print, [], basic_tests()} + ]. + +basic_tests() -> + [ + t_basic_apply_rule_trace_ruleid, + t_basic_apply_rule_trace_clientid, + t_basic_apply_rule_trace_ruleid_stop_after_render + ]. init_per_suite(Config) -> application:load(emqx_conf), @@ -50,6 +67,12 @@ init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite(), [{apps, Apps} | Config]. +init_per_group(GroupName, Config) -> + [{group_name, GroupName} | Config]. + +end_per_group(_GroupName, Config) -> + Config. + end_per_suite(Config) -> Apps = ?config(apps, Config), emqx_mgmt_api_test_util:end_suite(), @@ -64,31 +87,62 @@ end_per_testcase(_TestCase, _Config) -> emqx_bridge_v2_testlib:delete_all_bridges(), emqx_bridge_v2_testlib:delete_all_connectors(), emqx_common_test_helpers:call_janitor(), + meck:unload(), ok. t_basic_apply_rule_trace_ruleid(Config) -> - basic_apply_rule_test_helper(Config, ruleid, false). + basic_apply_rule_test_helper(get_action(Config), ruleid, false). t_basic_apply_rule_trace_clientid(Config) -> - basic_apply_rule_test_helper(Config, clientid, false). + basic_apply_rule_test_helper(get_action(Config), clientid, false). t_basic_apply_rule_trace_ruleid_stop_after_render(Config) -> - basic_apply_rule_test_helper(Config, ruleid, true). + basic_apply_rule_test_helper(get_action(Config), ruleid, true). -basic_apply_rule_test_helper(Config, TraceType, StopAfterRender) -> +get_action(Config) -> + case ?config(group_name, Config) of + republish -> + republish_action(); + console_print -> + console_print_action(); + _ -> + make_http_bridge(Config) + end. + +make_http_bridge(Config) -> HTTPServerConfig = ?config(http_server, Config), emqx_bridge_http_test_lib:make_bridge(HTTPServerConfig), #{status := connected} = emqx_bridge_v2:health_check( http, emqx_bridge_http_test_lib:bridge_name() ), + BridgeName = ?config(bridge_name, Config), + emqx_bridge_resource:bridge_id(http, BridgeName). + +republish_action() -> + #{ + <<"args">> => + #{ + <<"mqtt_properties">> => #{}, + <<"payload">> => <<"MY PL">>, + <<"qos">> => 0, + <<"retain">> => false, + <<"topic">> => <<"rule_apply_test_SUITE">>, + <<"user_properties">> => <<>> + }, + <<"function">> => <<"republish">> + }. + +console_print_action() -> + #{<<"function">> => <<"console">>}. + +basic_apply_rule_test_helper(Action, TraceType, StopAfterRender) -> %% Create Rule RuleTopic = iolist_to_binary([<<"my_rule_topic/">>, atom_to_binary(?FUNCTION_NAME)]), SQL = <<"SELECT payload.id as id FROM \"", RuleTopic/binary, "\"">>, {ok, #{<<"id">> := RuleId}} = - emqx_bridge_testlib:create_rule_and_action_http( - http, + emqx_bridge_testlib:create_rule_and_action( + Action, RuleTopic, - Config, #{sql => SQL} ), ClientId = <<"c_emqx">>, @@ -117,10 +171,7 @@ basic_apply_rule_test_helper(Config, TraceType, StopAfterRender) -> <<"context">> => Context, <<"stop_action_after_template_rendering">> => StopAfterRender }, - emqx_trace:check(), - ok = emqx_trace_handler_SUITE:filesync(TraceName, TraceType), Now = erlang:system_time(second) - 10, - {ok, _} = file:read_file(emqx_trace:log_file(TraceName, Now)), ?assertMatch({ok, _}, call_apply_rule_api(RuleId, Params)), ?retry( _Interval0 = 200, @@ -130,9 +181,14 @@ basic_apply_rule_test_helper(Config, TraceType, StopAfterRender) -> io:format("THELOG:~n~s", [Bin]), ?assertNotEqual(nomatch, binary:match(Bin, [<<"rule_activated">>])), ?assertNotEqual(nomatch, binary:match(Bin, [<<"SQL_yielded_result">>])), - ?assertNotEqual(nomatch, binary:match(Bin, [<<"bridge_action">>])), - ?assertNotEqual(nomatch, binary:match(Bin, [<<"action_template_rendered">>])), - ?assertNotEqual(nomatch, binary:match(Bin, [<<"QUERY_ASYNC">>])) + case Action of + A when is_binary(A) -> + ?assertNotEqual(nomatch, binary:match(Bin, [<<"bridge_action">>])), + ?assertNotEqual(nomatch, binary:match(Bin, [<<"QUERY_ASYNC">>])); + _ -> + ?assertNotEqual(nomatch, binary:match(Bin, [<<"call_action_function">>])) + end, + ?assertNotEqual(nomatch, binary:match(Bin, [<<"action_template_rendered">>])) end ), case StopAfterRender of @@ -155,7 +211,8 @@ basic_apply_rule_test_helper(Config, TraceType, StopAfterRender) -> begin Bin = read_rule_trace_file(TraceName, TraceType, Now), io:format("THELOG3:~n~s", [Bin]), - ?assertNotEqual(nomatch, binary:match(Bin, [<<"action_success">>])) + ?assertNotEqual(nomatch, binary:match(Bin, [<<"action_success">>])), + do_final_log_check(Action, Bin) end ) end, @@ -173,7 +230,51 @@ basic_apply_rule_test_helper(Config, TraceType, StopAfterRender) -> ) || #{<<"meta">> := Meta} <- LogEntries ], - emqx_trace:delete(TraceName), + ok. + +do_final_log_check(Action, Bin0) when is_binary(Action) -> + %% The last line in the Bin should be the action_success entry + Bin1 = string:trim(Bin0), + LastEntry = unicode:characters_to_binary(lists:last(string:split(Bin1, <<"\n">>, all))), + LastEntryJSON = emqx_utils_json:decode(LastEntry, [return_maps]), + %% Check that lazy formatting of the action result works correctly + ?assertMatch( + #{ + <<"level">> := <<"debug">>, + <<"meta">> := + #{ + <<"action_info">> := + #{ + <<"name">> := <<"emqx_bridge_http_test_lib">>, + <<"type">> := <<"http">> + }, + <<"clientid">> := <<"c_emqx">>, + <<"result">> := + #{ + <<"response">> := + #{ + <<"body">> := <<"hello">>, + <<"headers">> := + #{ + <<"content-type">> := <<"text/plain">>, + <<"date">> := _, + <<"server">> := _ + }, + <<"status">> := 200 + }, + <<"result">> := <<"ok">> + }, + <<"rule_id">> := _, + <<"rule_trigger_time">> := _, + <<"stop_action_after_render">> := false, + <<"trace_tag">> := <<"ACTION">> + }, + <<"msg">> := <<"action_success">>, + <<"time">> := _ + }, + LastEntryJSON + ); +do_final_log_check(_, _) -> ok. create_trace(TraceName, TraceType, TraceValue) -> @@ -188,26 +289,14 @@ create_trace(TraceName, TraceType, TraceValue) -> end_at => End, formatter => json }, - {ok, _} = emqx_trace:create(Trace). + {ok, _} = CreateRes = emqx_trace:create(Trace), + emqx_common_test_helpers:on_exit(fun() -> + ok = emqx_trace:delete(TraceName) + end), + CreateRes. t_apply_rule_test_batch_separation_stop_after_render(_Config) -> - MeckOpts = [passthrough, no_link, no_history, non_strict], - catch meck:new(emqx_connector_info, MeckOpts), - meck:expect( - emqx_connector_info, - hard_coded_test_connector_info_modules, - 0, - [emqx_rule_engine_test_connector_info] - ), - emqx_connector_info:clean_cache(), - catch meck:new(emqx_action_info, MeckOpts), - meck:expect( - emqx_action_info, - hard_coded_test_action_info_modules, - 0, - [emqx_rule_engine_test_action_info] - ), - emqx_action_info:clean_cache(), + meck_in_test_connector(), {ok, _} = emqx_connector:create(rule_engine_test, ?FUNCTION_NAME, #{}), Name = atom_to_binary(?FUNCTION_NAME), ActionConf = @@ -239,8 +328,6 @@ t_apply_rule_test_batch_separation_stop_after_render(_Config) -> SQL ), create_trace(Name, ruleid, RuleID), - emqx_trace:check(), - ok = emqx_trace_handler_SUITE:filesync(Name, ruleid), Now = erlang:system_time(second) - 10, %% Stop ParmsStopAfterRender = apply_rule_parms(true, Name), @@ -306,14 +393,242 @@ t_apply_rule_test_batch_separation_stop_after_render(_Config) -> ) end ), - %% Cleanup - ok = emqx_trace:delete(Name), - ok = emqx_rule_engine:delete_rule(RuleID), - ok = emqx_bridge_v2:remove(rule_engine_test, ?FUNCTION_NAME), - ok = emqx_connector:remove(rule_engine_test, ?FUNCTION_NAME), - [_, _] = meck:unload(), ok. +t_apply_rule_test_format_action_failed(_Config) -> + MeckOpts = [passthrough, no_link, no_history, non_strict], + catch meck:new(emqx_rule_engine_test_connector, MeckOpts), + meck:expect( + emqx_rule_engine_test_connector, + on_query, + 3, + {error, {unrecoverable_error, <<"MY REASON">>}} + ), + CheckFun = + fun(Bin0) -> + %% The last line in the Bin should be the action_failed entry + ?assertNotEqual(nomatch, binary:match(Bin0, [<<"action_failed">>])), + Bin1 = string:trim(Bin0), + LastEntry = unicode:characters_to_binary(lists:last(string:split(Bin1, <<"\n">>, all))), + LastEntryJSON = emqx_utils_json:decode(LastEntry, [return_maps]), + ?assertMatch( + #{ + <<"level">> := <<"debug">>, + <<"meta">> := #{ + <<"action_info">> := #{ + <<"name">> := _, + <<"type">> := <<"rule_engine_test">> + }, + <<"client_ids">> := [], + <<"clientid">> := _, + <<"reason">> := <<"MY REASON">>, + <<"rule_id">> := _, + <<"rule_ids">> := [], + <<"rule_trigger_time">> := _, + <<"rule_trigger_times">> := [], + <<"stop_action_after_render">> := false, + <<"trace_tag">> := <<"ACTION">> + }, + <<"msg">> := <<"action_failed">>, + <<"time">> := _ + }, + LastEntryJSON + ) + end, + do_apply_rule_test_format_action_failed_test(1, CheckFun). + +t_apply_rule_test_format_action_out_of_service_query(_Config) -> + Reason = <<"MY_RECOVERABLE_REASON">>, + CheckFun = out_of_service_check_fun(<<"send_error">>, Reason), + meck_test_connector_recoverable_errors(Reason), + do_apply_rule_test_format_action_failed_test(1, CheckFun). + +t_apply_rule_test_format_action_out_of_service_batch_query(_Config) -> + Reason = <<"MY_RECOVERABLE_REASON">>, + CheckFun = out_of_service_check_fun(<<"send_error">>, Reason), + meck_test_connector_recoverable_errors(Reason), + do_apply_rule_test_format_action_failed_test(10, CheckFun). + +t_apply_rule_test_format_action_out_of_service_async_query(_Config) -> + Reason = <<"MY_RECOVERABLE_REASON">>, + CheckFun = out_of_service_check_fun(<<"async_send_error">>, Reason), + meck_test_connector_recoverable_errors(Reason), + meck:expect( + emqx_rule_engine_test_connector, + callback_mode, + 0, + async_if_possible + ), + do_apply_rule_test_format_action_failed_test(1, CheckFun). + +t_apply_rule_test_format_action_out_of_service_async_batch_query(_Config) -> + Reason = <<"MY_RECOVERABLE_REASON">>, + CheckFun = out_of_service_check_fun(<<"async_send_error">>, Reason), + meck_test_connector_recoverable_errors(Reason), + meck:expect( + emqx_rule_engine_test_connector, + callback_mode, + 0, + async_if_possible + ), + do_apply_rule_test_format_action_failed_test(10, CheckFun). + +out_of_service_check_fun(SendErrorMsg, Reason) -> + fun(Bin0) -> + %% The last line in the Bin should be the action_failed entry + ?assertNotEqual(nomatch, binary:match(Bin0, [<<"action_failed">>])), + io:format("LOG:\n~s", [Bin0]), + Bin1 = string:trim(Bin0), + LastEntry = unicode:characters_to_binary(lists:last(string:split(Bin1, <<"\n">>, all))), + LastEntryJSON = emqx_utils_json:decode(LastEntry, [return_maps]), + ?assertMatch( + #{ + <<"level">> := <<"debug">>, + <<"meta">> := + #{ + <<"action_info">> := + #{ + <<"name">> := _, + <<"type">> := <<"rule_engine_test">> + }, + <<"clientid">> := _, + <<"reason">> := <<"request_expired">>, + <<"rule_id">> := _, + <<"rule_trigger_time">> := _, + <<"stop_action_after_render">> := false, + <<"trace_tag">> := <<"ACTION">> + }, + <<"msg">> := <<"action_failed">>, + <<"time">> := _ + }, + LastEntryJSON + ), + %% We should have at least one entry containing Reason + [ReasonLine | _] = find_lines_with(Bin1, Reason), + ReasonEntryJSON = emqx_utils_json:decode(ReasonLine, [return_maps]), + ?assertMatch( + #{ + <<"level">> := <<"debug">>, + <<"meta">> := + #{ + <<"client_ids">> := [], + <<"clientid">> := _, + <<"id">> := _, + <<"reason">> := + #{ + <<"additional_info">> := _, + <<"error_type">> := <<"recoverable_error">>, + <<"msg">> := <<"MY_RECOVERABLE_REASON">> + }, + <<"rule_id">> := _, + <<"rule_ids">> := [], + <<"rule_trigger_time">> := _, + <<"rule_trigger_times">> := [], + <<"stop_action_after_render">> := false, + <<"trace_tag">> := <<"ERROR">> + }, + <<"msg">> := SendErrorMsg, + <<"time">> := _ + }, + ReasonEntryJSON + ) + end. + +meck_test_connector_recoverable_errors(Reason) -> + MeckOpts = [passthrough, no_link, no_history, non_strict], + catch meck:new(emqx_rule_engine_test_connector, MeckOpts), + meck:expect( + emqx_rule_engine_test_connector, + on_query, + 3, + {error, {recoverable_error, Reason}} + ), + meck:expect( + emqx_rule_engine_test_connector, + on_batch_query, + 3, + {error, {recoverable_error, Reason}} + ), + meck:expect( + emqx_rule_engine_test_connector, + on_query_async, + 4, + {error, {recoverable_error, Reason}} + ), + meck:expect( + emqx_rule_engine_test_connector, + on_batch_query_async, + 4, + {error, {recoverable_error, Reason}} + ). + +find_lines_with(Data, InLineText) -> + % Split the binary data into lines + Lines = re:split(Data, "\n", [{return, binary}]), + + % Use a list comprehension to filter lines containing 'Reason' + [Line || Line <- Lines, re:run(Line, InLineText, [multiline, {capture, none}]) =/= nomatch]. + +do_apply_rule_test_format_action_failed_test(BatchSize, CheckLastTraceEntryFun) -> + meck_in_test_connector(), + {ok, _} = emqx_connector:create(rule_engine_test, ?FUNCTION_NAME, #{}), + Name = atom_to_binary(?FUNCTION_NAME), + ActionConf = + #{ + <<"connector">> => Name, + <<"parameters">> => #{<<"values">> => #{}}, + <<"resource_opts">> => #{ + <<"batch_size">> => BatchSize, + <<"batch_time">> => 10, + <<"request_ttl">> => 200 + } + }, + {ok, _} = emqx_bridge_v2:create( + rule_engine_test, + ?FUNCTION_NAME, + ActionConf + ), + SQL = <<"SELECT payload.is_stop_after_render as stop_after_render FROM \"", Name/binary, "\"">>, + {ok, RuleID} = create_rule_with_action( + rule_engine_test, + ?FUNCTION_NAME, + SQL + ), + create_trace(Name, ruleid, RuleID), + Now = erlang:system_time(second) - 10, + %% Stop + ParmsNoStopAfterRender = apply_rule_parms(false, Name), + {ok, _} = call_apply_rule_api(RuleID, ParmsNoStopAfterRender), + %% Just check that the log file is created as expected + ?retry( + _Interval0 = 200, + _NAttempts0 = 100, + begin + Bin = read_rule_trace_file(Name, ruleid, Now), + CheckLastTraceEntryFun(Bin) + end + ), + ok. + +meck_in_test_connector() -> + MeckOpts = [passthrough, no_link, no_history, non_strict], + catch meck:new(emqx_connector_info, MeckOpts), + meck:expect( + emqx_connector_info, + hard_coded_test_connector_info_modules, + 0, + [emqx_rule_engine_test_connector_info] + ), + emqx_connector_info:clean_cache(), + catch meck:new(emqx_action_info, MeckOpts), + meck:expect( + emqx_action_info, + hard_coded_test_action_info_modules, + 0, + [emqx_rule_engine_test_action_info] + ), + emqx_action_info:clean_cache(). + apply_rule_parms(StopAfterRender, Name) -> Payload = #{<<"is_stop_after_render">> => StopAfterRender}, Context = #{ @@ -342,6 +657,9 @@ create_rule_with_action(ActionType, ActionName, SQL) -> case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of {ok, Res0} -> #{<<"id">> := RuleId} = emqx_utils_json:decode(Res0, [return_maps]), + emqx_common_test_helpers:on_exit(fun() -> + emqx_rule_engine:delete_rule(RuleId) + end), {ok, RuleId}; Error -> Error diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl index c22c5fbd5..6a0d6b3ec 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl @@ -29,7 +29,9 @@ on_start/2, on_stop/2, on_query/3, + on_query_async/4, on_batch_query/3, + on_batch_query_async/4, on_get_status/2, on_add_channel/4, on_remove_channel/3, @@ -85,6 +87,14 @@ on_query( ) -> ok. +on_query_async( + _InstId, + _Query, + _State, + _Callback +) -> + ok. + on_batch_query( _InstId, [{ChannelId, _Req} | _] = Msg, @@ -96,5 +106,13 @@ on_batch_query( emqx_trace:rendered_action_template(ChannelId, #{nothing_to_render => ok}), ok. +on_batch_query_async( + _InstId, + _Batch, + _State, + _Callback +) -> + ok. + on_get_status(_InstId, _State) -> connected. diff --git a/changes/ce/feat-12947.en.md b/changes/ce/feat-12947.en.md new file mode 100644 index 000000000..470a61f80 --- /dev/null +++ b/changes/ce/feat-12947.en.md @@ -0,0 +1,10 @@ +## Breaking changes + +For JWT authentication, support new `disconnect_after_expire` option. When enabled, the client will be disconnected after the JWT token expires. + +This option is enabled by default, so the default behavior is changed. +Previously, the clients with actual JWTs could connect to the broker and stay connected +even after the JWT token expired. +Now, the client will be disconnected after the JWT token expires. + +To preserve the previous behavior, set `disconnect_after_expire` to `false`. diff --git a/changes/ce/fix-12962.en.md b/changes/ce/fix-12962.en.md new file mode 100644 index 000000000..7319709e4 --- /dev/null +++ b/changes/ce/fix-12962.en.md @@ -0,0 +1,4 @@ +TLS clients can now verify server hostname against wildcard certificate. + +For example, if a certificate is issued for host `*.example.com`, +TLS clients is able to verify server hostnames like `srv1.example.com`. diff --git a/changes/ce/fix-12976.md b/changes/ce/fix-12976.md new file mode 100644 index 000000000..82cf82b0d --- /dev/null +++ b/changes/ce/fix-12976.md @@ -0,0 +1 @@ +Fix the `client.disconnected` event being triggered when taking over a session that the socket has been disconnected before. diff --git a/mix.exs b/mix.exs index 2696cfe55..e1de43e81 100644 --- a/mix.exs +++ b/mix.exs @@ -158,6 +158,7 @@ defmodule EMQXUmbrella.MixProject do # need to remove those when listing `/apps/`... defp enterprise_umbrella_apps(_release_type) do MapSet.new([ + :emqx_connector_aggregator, :emqx_bridge_kafka, :emqx_bridge_confluent, :emqx_bridge_gcp_pubsub, diff --git a/rebar.config.erl b/rebar.config.erl index e2e86bc29..399f34b18 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -76,6 +76,7 @@ is_cover_enabled() -> is_enterprise(ce) -> false; is_enterprise(ee) -> true. +is_community_umbrella_app("apps/emqx_connector_aggregator") -> false; is_community_umbrella_app("apps/emqx_bridge_kafka") -> false; is_community_umbrella_app("apps/emqx_bridge_confluent") -> false; is_community_umbrella_app("apps/emqx_bridge_gcp_pubsub") -> false; diff --git a/rel/i18n/emqx_authn_jwt_schema.hocon b/rel/i18n/emqx_authn_jwt_schema.hocon index 67b301b86..f99b8c7f1 100644 --- a/rel/i18n/emqx_authn_jwt_schema.hocon +++ b/rel/i18n/emqx_authn_jwt_schema.hocon @@ -146,4 +146,10 @@ Authentication will verify that the value of claims in the JWT (taken from the P verify_claims.label: """Verify Claims""" +disconnect_after_expire.desc: +"""Disconnect the client after the token expires.""" + +disconnect_after_expire.label: +"""Disconnect After Expire""" + } diff --git a/rel/i18n/emqx_message_validation_schema.hocon b/rel/i18n/emqx_message_validation_schema.hocon index 670a16b80..1aeec7233 100644 --- a/rel/i18n/emqx_message_validation_schema.hocon +++ b/rel/i18n/emqx_message_validation_schema.hocon @@ -30,9 +30,9 @@ emqx_message_validation_schema { check_protobuf_schema.label: """Schema name""" - check_protobuf_message_name.desc: + check_protobuf_message_type.desc: """Message name to use during check.""" - check_protobuf_message_name.label: + check_protobuf_message_type.label: """Message name""" check_sql_type.desc: