Merge pull request #12997 from thalesmg/sync-r57-m-20240508

sync `release-57` to `master`

apps/emqx/include/emqx_trace.hrl

@@ -35,6 +35,11 @@
     end_at :: integer() | undefined | '_'
 }).
 
+-record(emqx_trace_format_func_data, {
+    function :: fun((any()) -> any()),
+    data :: any()
+}).
+
 -define(SHARD, ?COMMON_SHARD).
 -define(MAX_SIZE, 30).
 

apps/emqx/src/emqx_broker_helper.erl

@@ -110,7 +110,7 @@ reclaim_seq(Topic) ->
 
 stats_fun() ->
     safe_update_stats(subscriber_val(), 'subscribers.count', 'subscribers.max'),
-    safe_update_stats(table_size(?SUBSCRIPTION), 'subscriptions.count', 'subscriptions.max'),
+    safe_update_stats(subscription_count(), 'subscriptions.count', 'subscriptions.max'),
     safe_update_stats(table_size(?SUBOPTION), 'suboptions.count', 'suboptions.max').
 
 safe_update_stats(undefined, _Stat, _MaxStat) ->
@@ -118,6 +118,16 @@ safe_update_stats(undefined, _Stat, _MaxStat) ->
 safe_update_stats(Val, Stat, MaxStat) when is_integer(Val) ->
     emqx_stats:setstat(Stat, MaxStat, Val).
 
+subscription_count() ->
+    NonPSCount = table_size(?SUBSCRIPTION),
+    PSCount = emqx_persistent_session_bookkeeper:get_subscription_count(),
+    case is_integer(NonPSCount) of
+        true ->
+            NonPSCount + PSCount;
+        false ->
+            PSCount
+    end.
+
 subscriber_val() ->
     sum_subscriber(table_size(?SUBSCRIBER), table_size(?SHARED_SUBSCRIBER)).
 

apps/emqx/src/emqx_channel.erl

@@ -1075,7 +1075,7 @@ handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel = ?IS_MQTT_V5) -
     Packet = ?DISCONNECT_PACKET(ReasonCode, Props),
     {ok, [?REPLY_OUTGOING(Packet), ?REPLY_CLOSE(ReasonName)], Channel};
 handle_out(disconnect, {_ReasonCode, ReasonName, _Props}, Channel) ->
-    {ok, {close, ReasonName}, Channel};
+    {ok, ?REPLY_CLOSE(ReasonName), Channel};
 handle_out(auth, {ReasonCode, Properties}, Channel) ->
     {ok, ?AUTH_PACKET(ReasonCode, Properties), Channel};
 handle_out(Type, Data, Channel) ->
@@ -1406,6 +1406,16 @@ handle_timeout(
         {_, Quota2} ->
             {ok, clean_timer(TimerName, Channel#channel{quota = Quota2})}
     end;
+handle_timeout(
+    _TRef,
+    connection_expire,
+    #channel{conn_state = ConnState} = Channel0
+) ->
+    Channel1 = clean_timer(connection_expire, Channel0),
+    case ConnState of
+        disconnected -> {ok, Channel1};
+        _ -> handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel1)
+    end;
 handle_timeout(TRef, Msg, Channel) ->
     case emqx_hooks:run_fold('client.timeout', [TRef, Msg], []) of
         [] ->
@@ -1810,18 +1820,23 @@ log_auth_failure(Reason) ->
 %% Merge authentication result into ClientInfo
 %% Authentication result may include:
 %% 1. `is_superuser': The superuser flag from various backends
-%% 2. `acl': ACL rules from JWT, HTTP auth backend
+%% 2. `expire_at`: Authentication validity deadline, the client will be disconnected after this time
-%% 3. `client_attrs': Extra client attributes from JWT, HTTP auth backend
+%% 3. `acl': ACL rules from JWT, HTTP auth backend
-%% 4. Maybe more non-standard fields used by hook callbacks
+%% 4. `client_attrs': Extra client attributes from JWT, HTTP auth backend
+%% 5. Maybe more non-standard fields used by hook callbacks
 merge_auth_result(ClientInfo, AuthResult0) when is_map(ClientInfo) andalso is_map(AuthResult0) ->
     IsSuperuser = maps:get(is_superuser, AuthResult0, false),
-    AuthResult = maps:without([client_attrs], AuthResult0),
+    ExpireAt = maps:get(expire_at, AuthResult0, undefined),
+    AuthResult = maps:without([client_attrs, expire_at], AuthResult0),
     Attrs0 = maps:get(client_attrs, ClientInfo, #{}),
     Attrs1 = maps:get(client_attrs, AuthResult0, #{}),
     Attrs = maps:merge(Attrs0, Attrs1),
     NewClientInfo = maps:merge(
         ClientInfo#{client_attrs => Attrs},
-        AuthResult#{is_superuser => IsSuperuser}
+        AuthResult#{
+            is_superuser => IsSuperuser,
+            auth_expire_at => ExpireAt
+        }
     ),
     fix_mountpoint(NewClientInfo).
 
@@ -2228,10 +2243,16 @@ ensure_connected(
 ) ->
     NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
     ok = run_hooks('client.connected', [ClientInfo, NConnInfo]),
-    Channel#channel{
+    schedule_connection_expire(Channel#channel{
         conninfo = trim_conninfo(NConnInfo),
         conn_state = connected
-    }.
+    }).
+
+schedule_connection_expire(Channel = #channel{clientinfo = #{auth_expire_at := undefined}}) ->
+    Channel;
+schedule_connection_expire(Channel = #channel{clientinfo = #{auth_expire_at := ExpireAt}}) ->
+    Interval = max(0, ExpireAt - erlang:system_time(millisecond)),
+    ensure_timer(connection_expire, Interval, Channel).
 
 trim_conninfo(ConnInfo) ->
     maps:without(
@@ -2615,10 +2636,15 @@ disconnect_and_shutdown(
 ->
     NChannel = ensure_disconnected(Reason, Channel),
     shutdown(Reason, Reply, ?DISCONNECT_PACKET(reason_code(Reason)), NChannel);
-%% mqtt v3/v4 sessions, mqtt v5 other conn_state sessions
+%% mqtt v3/v4 connected sessions
-disconnect_and_shutdown(Reason, Reply, Channel) ->
+disconnect_and_shutdown(Reason, Reply, Channel = #channel{conn_state = ConnState}) when
+    ConnState =:= connected orelse ConnState =:= reauthenticating
+->
     NChannel = ensure_disconnected(Reason, Channel),
-    shutdown(Reason, Reply, NChannel).
+    shutdown(Reason, Reply, NChannel);
+%% other conn_state sessions
+disconnect_and_shutdown(Reason, Reply, Channel) ->
+    shutdown(Reason, Reply, Channel).
 
 -compile({inline, [sp/1, flag/1]}).
 sp(true) -> 1;

apps/emqx/src/emqx_cm_sup.erl

@@ -53,6 +53,7 @@ init([]) ->
     RegistryKeeper = child_spec(emqx_cm_registry_keeper, 5000, worker),
     Manager = child_spec(emqx_cm, 5000, worker),
     DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor),
+    DSSessionBookkeeper = child_spec(emqx_persistent_session_bookkeeper, 5_000, worker),
     Children =
         [
             Banned,
@@ -62,7 +63,8 @@ init([]) ->
             Registry,
             RegistryKeeper,
             Manager,
-            DSSessionGCSup
+            DSSessionGCSup,
+            DSSessionBookkeeper
         ],
     {ok, {SupFlags, Children}}.
 

apps/emqx/src/emqx_listeners.erl

@@ -1036,8 +1036,8 @@ to_quicer_listener_opts(Opts) ->
     SSLOpts = maps:from_list(ssl_opts(Opts)),
     Opts1 = maps:filter(
         fun
-            (cacertfile, undefined) -> fasle;
+            (cacertfile, undefined) -> false;
-            (password, undefined) -> fasle;
+            (password, undefined) -> false;
             (_, _) -> true
         end,
         Opts

apps/emqx/src/emqx_logger_jsonfmt.erl

@@ -229,7 +229,7 @@ best_effort_json_obj(Map, Config) ->
             do_format_msg("~p", [Map], Config)
     end.
 
-json(A, _) when is_atom(A) -> atom_to_binary(A, utf8);
+json(A, _) when is_atom(A) -> A;
 json(I, _) when is_integer(I) -> I;
 json(F, _) when is_float(F) -> F;
 json(P, C) when is_pid(P) -> json(pid_to_list(P), C);

apps/emqx/src/emqx_persistent_session_bookkeeper.erl

@@ -0,0 +1,107 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_persistent_session_bookkeeper).
+
+-behaviour(gen_server).
+
+%% API
+-export([
+    start_link/0,
+    get_subscription_count/0
+]).
+
+%% `gen_server' API
+-export([
+    init/1,
+    handle_continue/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+%%------------------------------------------------------------------------------
+%% Type declarations
+%%------------------------------------------------------------------------------
+
+%% call/cast/info events
+-record(tally_subs, {}).
+-record(get_subscription_count, {}).
+
+%%------------------------------------------------------------------------------
+%% API
+%%------------------------------------------------------------------------------
+
+-spec start_link() -> gen_server:start_ret().
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, _InitOpts = #{}, _Opts = []).
+
+%% @doc Gets a cached view of the cluster-global count of persistent subscriptions.
+-spec get_subscription_count() -> non_neg_integer().
+get_subscription_count() ->
+    case emqx_persistent_message:is_persistence_enabled() of
+        true ->
+            gen_server:call(?MODULE, #get_subscription_count{}, infinity);
+        false ->
+            0
+    end.
+
+%%------------------------------------------------------------------------------
+%% `gen_server' API
+%%------------------------------------------------------------------------------
+
+init(_Opts) ->
+    case emqx_persistent_message:is_persistence_enabled() of
+        true ->
+            State = #{subs_count => 0},
+            {ok, State, {continue, #tally_subs{}}};
+        false ->
+            ignore
+    end.
+
+handle_continue(#tally_subs{}, State0) ->
+    State = tally_persistent_subscriptions(State0),
+    ensure_subs_tally_timer(),
+    {noreply, State}.
+
+handle_call(#get_subscription_count{}, _From, State) ->
+    #{subs_count := N} = State,
+    {reply, N, State};
+handle_call(_Call, _From, State) ->
+    {reply, {error, bad_call}, State}.
+
+handle_cast(_Cast, State) ->
+    {noreply, State}.
+
+handle_info(#tally_subs{}, State0) ->
+    State = tally_persistent_subscriptions(State0),
+    ensure_subs_tally_timer(),
+    {noreply, State};
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+%%------------------------------------------------------------------------------
+%% Internal fns
+%%------------------------------------------------------------------------------
+
+tally_persistent_subscriptions(State0) ->
+    N = emqx_persistent_session_ds_state:total_subscription_count(),
+    State0#{subs_count := N}.
+
+ensure_subs_tally_timer() ->
+    Timeout = emqx_config:get([session_persistence, subscription_count_refresh_interval]),
+    _ = erlang:send_after(Timeout, self(), #tally_subs{}),
+    ok.

apps/emqx/src/emqx_persistent_session_ds.erl

@@ -658,16 +658,17 @@ replay_batch(Srs0, Session0, ClientInfo) ->
 %%--------------------------------------------------------------------
 
 -spec disconnect(session(), emqx_types:conninfo()) -> {shutdown, session()}.
-disconnect(Session = #{s := S0}, ConnInfo) ->
+disconnect(Session = #{id := Id, s := S0}, ConnInfo) ->
-    S1 = emqx_persistent_session_ds_state:set_last_alive_at(now_ms(), S0),
+    S1 = maybe_set_offline_info(S0, Id),
-    S2 =
+    S2 = emqx_persistent_session_ds_state:set_last_alive_at(now_ms(), S1),
+    S3 =
         case ConnInfo of
             #{expiry_interval := EI} when is_number(EI) ->
-                emqx_persistent_session_ds_state:set_expiry_interval(EI, S1);
+                emqx_persistent_session_ds_state:set_expiry_interval(EI, S2);
             _ ->
-                S1
+                S2
         end,
-    S = emqx_persistent_session_ds_state:commit(S2),
+    S = emqx_persistent_session_ds_state:commit(S3),
     {shutdown, Session#{s => S}}.
 
 -spec terminate(Reason :: term(), session()) -> ok.
@@ -702,7 +703,7 @@ list_client_subscriptions(ClientId) ->
                         maps:fold(
                             fun(Topic, #{current_state := CS}, Acc) ->
                                 #{subopts := SubOpts} = maps:get(CS, SStates),
-                                Elem = {Topic, SubOpts},
+                                Elem = {Topic, SubOpts#{durable => true}},
                                 [Elem | Acc]
                             end,
                             [],
@@ -1175,6 +1176,19 @@ try_get_live_session(ClientId) ->
             not_found
     end.
 
+-spec maybe_set_offline_info(emqx_persistent_session_ds_state:t(), emqx_types:clientid()) ->
+    emqx_persistent_session_ds_state:t().
+maybe_set_offline_info(S, Id) ->
+    case emqx_cm:lookup_client({clientid, Id}) of
+        [{_Key, ChannelInfo, Stats}] ->
+            emqx_persistent_session_ds_state:set_offline_info(
+                #{chan_info => ChannelInfo, stats => Stats},
+                S
+            );
+        _ ->
+            S
+    end.
+
 %%--------------------------------------------------------------------
 %% SeqNo tracking
 %% --------------------------------------------------------------------

apps/emqx/src/emqx_persistent_session_ds.hrl

@@ -81,5 +81,6 @@
 -define(will_message, will_message).
 -define(clientinfo, clientinfo).
 -define(protocol, protocol).
+-define(offline_info, offline_info).
 
 -endif.

apps/emqx/src/emqx_persistent_session_ds_state.erl

@@ -35,6 +35,7 @@
 -export([get_expiry_interval/1, set_expiry_interval/2]).
 -export([get_clientinfo/1, set_clientinfo/2]).
 -export([get_will_message/1, set_will_message/2, clear_will_message/1, clear_will_message_now/1]).
+-export([set_offline_info/2]).
 -export([get_peername/1, set_peername/2]).
 -export([get_protocol/1, set_protocol/2]).
 -export([new_id/1]).
@@ -53,6 +54,7 @@
     cold_get_subscription/2,
     fold_subscriptions/3,
     n_subscriptions/1,
+    total_subscription_count/0,
     put_subscription/3,
     del_subscription/2
 ]).
@@ -372,6 +374,10 @@ clear_will_message_now(SessionId) when is_binary(SessionId) ->
 clear_will_message(Rec) ->
     set_will_message(undefined, Rec).
 
+-spec set_offline_info(_Info :: map(), t()) -> t().
+set_offline_info(Info, Rec) ->
+    set_meta(?offline_info, Info, Rec).
+
 -spec new_id(t()) -> {emqx_persistent_session_ds:subscription_id(), t()}.
 new_id(Rec) ->
     LastId =
@@ -401,6 +407,12 @@ fold_subscriptions(Fun, Acc, Rec) ->
 n_subscriptions(Rec) ->
     gen_size(?subscriptions, Rec).
 
+-spec total_subscription_count() -> non_neg_integer().
+total_subscription_count() ->
+    mria:async_dirty(?DS_MRIA_SHARD, fun() ->
+        mnesia:foldl(fun(#kv{}, Acc) -> Acc + 1 end, 0, ?subscription_tab)
+    end).
+
 -spec put_subscription(
     emqx_persistent_session_ds:topic_filter(),
     emqx_persistent_session_ds_subs:subscription(),

apps/emqx/src/emqx_router_helper.erl

@@ -189,7 +189,17 @@ code_change(_OldVsn, State, _Extra) ->
 %%--------------------------------------------------------------------
 
 stats_fun() ->
-    emqx_stats:setstat('topics.count', 'topics.max', emqx_router:stats(n_routes)).
+    PSRouteCount = persistent_route_count(),
+    NonPSRouteCount = emqx_router:stats(n_routes),
+    emqx_stats:setstat('topics.count', 'topics.max', PSRouteCount + NonPSRouteCount).
+
+persistent_route_count() ->
+    case emqx_persistent_message:is_persistence_enabled() of
+        true ->
+            emqx_persistent_session_ds_router:stats(n_routes);
+        false ->
+            0
+    end.
 
 cleanup_routes(Node) ->
     emqx_router:cleanup_routes(Node).

apps/emqx/src/emqx_schema.erl

@@ -1713,6 +1713,14 @@ fields("session_persistence") ->
                     desc => ?DESC(session_ds_session_gc_batch_size)
                 }
             )},
+        {"subscription_count_refresh_interval",
+            sc(
+                timeout_duration(),
+                #{
+                    default => <<"5s">>,
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )},
         {"message_retention_period",
             sc(
                 timeout_duration(),

apps/emqx/src/emqx_tls_lib.erl

@@ -545,13 +545,19 @@ to_client_opts(Type, Opts) ->
                     {depth, Get(depth)},
                     {password, ensure_str(Get(password))},
                     {secure_renegotiate, Get(secure_renegotiate)}
-                ],
+                ] ++ hostname_check(Verify),
                 Versions
             );
         false ->
             []
     end.
 
+hostname_check(verify_none) ->
+    [];
+hostname_check(verify_peer) ->
+    %% allow wildcard certificates
+    [{customize_hostname_check, [{match_fun, public_key:pkix_verify_hostname_match_fun(https)}]}].
+
 resolve_cert_path_for_read_strict(Path) ->
     case resolve_cert_path_for_read(Path) of
         undefined ->

apps/emqx/src/emqx_trace/emqx_trace.erl

@@ -31,7 +31,8 @@
     log/4,
     rendered_action_template/2,
     make_rendered_action_template_trace_context/1,
-    rendered_action_template_with_ctx/2
+    rendered_action_template_with_ctx/2,
+    is_rule_trace_active/0
 ]).
 
 -export([
@@ -96,6 +97,16 @@ unsubscribe(Topic, SubOpts) ->
     ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}).
 
 rendered_action_template(<<"action:", _/binary>> = ActionID, RenderResult) ->
+    do_rendered_action_template(ActionID, RenderResult);
+rendered_action_template(#{mod := _, func := _} = ActionID, RenderResult) ->
+    do_rendered_action_template(ActionID, RenderResult);
+rendered_action_template(_ActionID, _RenderResult) ->
+    %% We do nothing if we don't get a valid Action ID. This can happen when
+    %% called from connectors that are used for actions as well as authz and
+    %% authn.
+    ok.
+
+do_rendered_action_template(ActionID, RenderResult) ->
     TraceResult = ?TRACE(
         "QUERY_RENDER",
         "action_template_rendered",
@@ -108,23 +119,25 @@ rendered_action_template(<<"action:", _/binary>> = ActionID, RenderResult) ->
         #{stop_action_after_render := true} ->
             %% We throw an unrecoverable error to stop action before the
             %% resource is called/modified
-            StopMsg = lists:flatten(
+            ActionIDStr =
+                case ActionID of
+                    Bin when is_binary(Bin) ->
+                        Bin;
+                    Term ->
+                        ActionIDFormatted = io_lib:format("~tw", [Term]),
+                        unicode:characters_to_binary(ActionIDFormatted)
+                end,
+            StopMsg =
                 io_lib:format(
                     "Action ~ts stopped after template rendering due to test setting.",
-                    [ActionID]
+                    [ActionIDStr]
-                )
                 ),
             MsgBin = unicode:characters_to_binary(StopMsg),
             error(?EMQX_TRACE_STOP_ACTION(MsgBin));
         _ ->
             ok
     end,
-    TraceResult;
+    TraceResult.
-rendered_action_template(_ActionID, _RenderResult) ->
-    %% We do nothing if we don't get a valid Action ID. This can happen when
-    %% called from connectors that are used for actions as well as authz and
-    %% authn.
-    ok.
 
 %% The following two functions are used for connectors that don't do the
 %% rendering in the main process (the one that called on_*query). In this case
@@ -165,6 +178,16 @@ rendered_action_template_with_ctx(
         logger:set_process_metadata(OldMetaData)
     end.
 
+is_rule_trace_active() ->
+    case logger:get_process_metadata() of
+        #{rule_id := RID} when is_binary(RID) ->
+            true;
+        #{rule_ids := RIDs} when map_size(RIDs) > 0 ->
+            true;
+        _ ->
+            false
+    end.
+
 log(List, Msg, Meta) ->
     log(debug, List, Msg, Meta).
 
@@ -382,7 +405,14 @@ code_change(_, State, _Extra) ->
     {ok, State}.
 
 insert_new_trace(Trace) ->
-    transaction(fun emqx_trace_dl:insert_new_trace/1, [Trace]).
+    case transaction(fun emqx_trace_dl:insert_new_trace/1, [Trace]) of
+        {error, _} = Error ->
+            Error;
+        Res ->
+            %% We call this to ensure the trace is active when we return
+            check(),
+            Res
+    end.
 
 update_trace(Traces) ->
     Now = now_second(),

apps/emqx/src/emqx_trace/emqx_trace_formatter.erl

@@ -15,9 +15,11 @@
 %%--------------------------------------------------------------------
 -module(emqx_trace_formatter).
 -include("emqx_mqtt.hrl").
+-include("emqx_trace.hrl").
 
 -export([format/2]).
 -export([format_meta_map/1]).
+-export([evaluate_lazy_values/1]).
 
 %% logger_formatter:config/0 is not exported.
 -type config() :: map().
@@ -28,18 +30,35 @@
     LogEvent :: logger:log_event(),
     Config :: config().
 format(
-    #{level := debug, meta := Meta = #{trace_tag := Tag}, msg := Msg},
+    #{level := debug, meta := Meta0 = #{trace_tag := Tag}, msg := Msg},
     #{payload_encode := PEncode}
 ) ->
+    Meta1 = evaluate_lazy_values(Meta0),
     Time = emqx_utils_calendar:now_to_rfc3339(microsecond),
-    ClientId = to_iolist(maps:get(clientid, Meta, "")),
+    ClientId = to_iolist(maps:get(clientid, Meta1, "")),
-    Peername = maps:get(peername, Meta, ""),
+    Peername = maps:get(peername, Meta1, ""),
-    MetaBin = format_meta(Meta, PEncode),
+    MetaBin = format_meta(Meta1, PEncode),
     Msg1 = to_iolist(Msg),
     Tag1 = to_iolist(Tag),
     [Time, " [", Tag1, "] ", ClientId, "@", Peername, " msg: ", Msg1, ", ", MetaBin, "\n"];
 format(Event, Config) ->
-    emqx_logger_textfmt:format(Event, Config).
+    emqx_logger_textfmt:format(evaluate_lazy_values(Event), Config).
+
+evaluate_lazy_values(Map) when is_map(Map) ->
+    maps:map(fun evaluate_lazy_values_kv/2, Map);
+evaluate_lazy_values(V) ->
+    V.
+
+evaluate_lazy_values_kv(_K, #emqx_trace_format_func_data{function = Formatter, data = V}) ->
+    try
+        NewV = Formatter(V),
+        evaluate_lazy_values(NewV)
+    catch
+        _:_ ->
+            V
+    end;
+evaluate_lazy_values_kv(_K, V) ->
+    evaluate_lazy_values(V).
 
 format_meta_map(Meta) ->
     Encode = emqx_trace_handler:payload_encode(),

apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl

@@ -16,6 +16,7 @@
 -module(emqx_trace_json_formatter).
 
 -include("emqx_mqtt.hrl").
+-include("emqx_trace.hrl").
 
 -export([format/2]).
 
@@ -30,15 +31,16 @@
     LogEvent :: logger:log_event(),
     Config :: config().
 format(
-    LogMap,
+    LogMap0,
     #{payload_encode := PEncode}
 ) ->
+    LogMap1 = emqx_trace_formatter:evaluate_lazy_values(LogMap0),
     %% We just make some basic transformations on the input LogMap and then do
     %% an external call to create the JSON text
     Time = emqx_utils_calendar:now_to_rfc3339(microsecond),
-    LogMap1 = LogMap#{time => Time},
+    LogMap2 = LogMap1#{time => Time},
-    LogMap2 = prepare_log_map(LogMap1, PEncode),
+    LogMap3 = prepare_log_map(LogMap2, PEncode),
-    [emqx_logger_jsonfmt:best_effort_json(LogMap2, [force_utf8]), "\n"].
+    [emqx_logger_jsonfmt:best_effort_json(LogMap3, [force_utf8]), "\n"].
 
 %%%-----------------------------------------------------------------
 %%% Helper Functions
@@ -48,21 +50,26 @@ prepare_log_map(LogMap, PEncode) ->
     NewKeyValuePairs = [prepare_key_value(K, V, PEncode) || {K, V} <- maps:to_list(LogMap)],
     maps:from_list(NewKeyValuePairs).
 
-prepare_key_value(K, {Formatter, V}, PEncode) when is_function(Formatter, 1) ->
+prepare_key_value(host, {I1, I2, I3, I4} = IP, _PEncode) when
-    %% A cusom formatter is provided with the value
+    is_integer(I1),
-    try
+    is_integer(I2),
-        NewV = Formatter(V),
+    is_integer(I3),
-        prepare_key_value(K, NewV, PEncode)
+    is_integer(I4)
-    catch
-        _:_ ->
-            {K, V}
-    end;
-prepare_key_value(K, {ok, Status, Headers, Body}, PEncode) when
-    is_integer(Status), is_list(Headers), is_binary(Body)
 ->
-    %% This is unlikely anything else then info about a HTTP request so we make
+    %% We assume this is an IP address
-    %% it more structured
+    {host, unicode:characters_to_binary(inet:ntoa(IP))};
-    prepare_key_value(K, #{status => Status, headers => Headers, body => Body}, PEncode);
+prepare_key_value(host, {I1, I2, I3, I4, I5, I6, I7, I8} = IP, _PEncode) when
+    is_integer(I1),
+    is_integer(I2),
+    is_integer(I3),
+    is_integer(I4),
+    is_integer(I5),
+    is_integer(I6),
+    is_integer(I7),
+    is_integer(I8)
+->
+    %% We assume this is an IP address
+    {host, unicode:characters_to_binary(inet:ntoa(IP))};
 prepare_key_value(payload = K, V, PEncode) ->
     NewV =
         try
@@ -81,6 +88,21 @@ prepare_key_value(packet = K, V, PEncode) ->
                 V
         end,
     {K, NewV};
+prepare_key_value(K, {recoverable_error, Msg} = OrgV, PEncode) ->
+    try
+        prepare_key_value(
+            K,
+            #{
+                error_type => recoverable_error,
+                msg => Msg,
+                additional_info => <<"The operation may be retried.">>
+            },
+            PEncode
+        )
+    catch
+        _:_ ->
+            {K, OrgV}
+    end;
 prepare_key_value(rule_ids = K, V, _PEncode) ->
     NewV =
         try
@@ -137,6 +159,8 @@ format_map_set_to_list(Map) ->
     ],
     lists:sort(Items).
 
+format_action_info(#{mod := _Mod, func := _Func} = FuncCall) ->
+    FuncCall;
 format_action_info(V) ->
     [<<"action">>, Type, Name | _] = binary:split(V, <<":">>, [global]),
     #{

apps/emqx/test/emqx_channel_SUITE.erl

@@ -1061,6 +1061,7 @@ clientinfo(InitProps) ->
             clientid => <<"clientid">>,
             username => <<"username">>,
             is_superuser => false,
+            auth_expire_at => undefined,
             is_bridge => false,
             mountpoint => undefined
         },

apps/emqx/test/emqx_config_SUITE.erl

@@ -475,6 +475,7 @@ zone_global_defaults() ->
                 message_retention_period => 86400000,
                 renew_streams_interval => 5000,
                 session_gc_batch_size => 100,
-                session_gc_interval => 600000
+                session_gc_interval => 600000,
+                subscription_count_refresh_interval => 5000
             }
     }.

apps/emqx/test/emqx_connection_expire_SUITE.erl

@@ -0,0 +1,54 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_connection_expire_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+all() -> emqx_common_test_helpers:all(?MODULE).
+
+%%--------------------------------------------------------------------
+%% CT callbacks
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+    Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}),
+    [{apps, Apps} | Config].
+
+end_per_suite(Config) ->
+    emqx_cth_suite:stop(proplists:get_value(apps, Config)).
+
+t_disonnect_by_auth_info(_) ->
+    _ = process_flag(trap_exit, true),
+
+    _ = meck:new(emqx_access_control, [passthrough, no_history]),
+    _ = meck:expect(emqx_access_control, authenticate, fun(_) ->
+        {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
+    end),
+
+    {ok, C} = emqtt:start_link([{proto_ver, v5}]),
+    {ok, _} = emqtt:connect(C),
+
+    receive
+        {disconnected, ?RC_NOT_AUTHORIZED, #{}} -> ok
+    after 5000 ->
+        ct:fail("Client should be disconnected by timeout")
+    end.

apps/emqx/test/emqx_tls_lib_tests.erl

@@ -240,7 +240,7 @@ to_client_opts_test() ->
     Versions13Only = ['tlsv1.3'],
     Options = #{
         enable => true,
-        verify => "Verify",
+        verify => verify_none,
         server_name_indication => "SNI",
         ciphers => "Ciphers",
         depth => "depth",
@@ -249,9 +249,16 @@ to_client_opts_test() ->
         secure_renegotiate => "secure_renegotiate",
         reuse_sessions => "reuse_sessions"
     },
-    Expected1 = lists:usort(maps:keys(Options) -- [enable]),
+    Expected0 = lists:usort(maps:keys(Options) -- [enable]),
+    Expected1 = lists:sort(Expected0 ++ [customize_hostname_check]),
     ?assertEqual(
-        Expected1, lists:usort(proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options)))
+        Expected0, lists:usort(proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options)))
+    ),
+    ?assertEqual(
+        Expected1,
+        lists:usort(
+            proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options#{verify => verify_peer}))
+        )
     ),
     Expected2 =
         lists:usort(

apps/emqx_auth/src/emqx_authn/emqx_authn_chains.erl

@@ -142,6 +142,8 @@ end).
 -type state() :: #{atom() => term()}.
 -type extra() :: #{
     is_superuser := boolean(),
+    %% millisecond timestamp
+    expire_at => pos_integer(),
     atom() => term()
 }.
 -type user_info() :: #{

apps/emqx_auth_jwt/src/emqx_authn_jwt.erl

@@ -78,6 +78,7 @@ authenticate(
     Credential,
     #{
         verify_claims := VerifyClaims0,
+        disconnect_after_expire := DisconnectAfterExpire,
         jwk := JWK,
         acl_claim_name := AclClaimName,
         from := From
@@ -86,11 +87,12 @@ authenticate(
     JWT = maps:get(From, Credential),
     JWKs = [JWK],
     VerifyClaims = render_expected(VerifyClaims0, Credential),
-    verify(JWT, JWKs, VerifyClaims, AclClaimName);
+    verify(JWT, JWKs, VerifyClaims, AclClaimName, DisconnectAfterExpire);
 authenticate(
     Credential,
     #{
         verify_claims := VerifyClaims0,
+        disconnect_after_expire := DisconnectAfterExpire,
         jwk_resource := ResourceId,
         acl_claim_name := AclClaimName,
         from := From
@@ -106,7 +108,7 @@ authenticate(
         {ok, JWKs} ->
             JWT = maps:get(From, Credential),
             VerifyClaims = render_expected(VerifyClaims0, Credential),
-            verify(JWT, JWKs, VerifyClaims, AclClaimName)
+            verify(JWT, JWKs, VerifyClaims, AclClaimName, DisconnectAfterExpire)
     end.
 
 destroy(#{jwk_resource := ResourceId}) ->
@@ -125,6 +127,7 @@ create2(#{
     secret := Secret0,
     secret_base64_encoded := Base64Encoded,
     verify_claims := VerifyClaims,
+    disconnect_after_expire := DisconnectAfterExpire,
     acl_claim_name := AclClaimName,
     from := From
 }) ->
@@ -136,6 +139,7 @@ create2(#{
             {ok, #{
                 jwk => JWK,
                 verify_claims => VerifyClaims,
+                disconnect_after_expire => DisconnectAfterExpire,
                 acl_claim_name => AclClaimName,
                 from => From
             }}
@@ -145,6 +149,7 @@ create2(#{
     algorithm := 'public-key',
     public_key := PublicKey,
     verify_claims := VerifyClaims,
+    disconnect_after_expire := DisconnectAfterExpire,
     acl_claim_name := AclClaimName,
     from := From
 }) ->
@@ -152,6 +157,7 @@ create2(#{
     {ok, #{
         jwk => JWK,
         verify_claims => VerifyClaims,
+        disconnect_after_expire => DisconnectAfterExpire,
         acl_claim_name => AclClaimName,
         from => From
     }};
@@ -159,6 +165,7 @@ create2(
     #{
         use_jwks := true,
         verify_claims := VerifyClaims,
+        disconnect_after_expire := DisconnectAfterExpire,
         acl_claim_name := AclClaimName,
         from := From
     } = Config
@@ -173,6 +180,7 @@ create2(
     {ok, #{
         jwk_resource => ResourceId,
         verify_claims => VerifyClaims,
+        disconnect_after_expire => DisconnectAfterExpire,
         acl_claim_name => AclClaimName,
         from => From
     }}.
@@ -211,23 +219,12 @@ render_expected([{Name, ExpectedTemplate} | More], Variables) ->
     Expected = emqx_authn_utils:render_str(ExpectedTemplate, Variables),
     [{Name, Expected} | render_expected(More, Variables)].
 
-verify(undefined, _, _, _) ->
+verify(undefined, _, _, _, _) ->
     ignore;
-verify(JWT, JWKs, VerifyClaims, AclClaimName) ->
+verify(JWT, JWKs, VerifyClaims, AclClaimName, DisconnectAfterExpire) ->
     case do_verify(JWT, JWKs, VerifyClaims) of
         {ok, Extra} ->
-            IsSuperuser = emqx_authn_utils:is_superuser(Extra),
+            extra_to_auth_data(Extra, JWT, AclClaimName, DisconnectAfterExpire);
-            Attrs = emqx_authn_utils:client_attrs(Extra),
-            try
-                ACL = acl(Extra, AclClaimName),
-                Result = maps:merge(IsSuperuser, maps:merge(ACL, Attrs)),
-                {ok, Result}
-            catch
-                throw:{bad_acl_rule, Reason} ->
-                    %% it's a invalid token, so ok to log
-                    ?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{jwt => JWT}),
-                    {error, bad_username_or_password}
-            end;
         {error, {missing_claim, Claim}} ->
             %% it's a invalid token, so it's ok to log
             ?TRACE_AUTHN_PROVIDER("missing_jwt_claim", #{jwt => JWT, claim => Claim}),
@@ -242,6 +239,28 @@ verify(JWT, JWKs, VerifyClaims, AclClaimName) ->
             {error, bad_username_or_password}
     end.
 
+extra_to_auth_data(Extra, JWT, AclClaimName, DisconnectAfterExpire) ->
+    IsSuperuser = emqx_authn_utils:is_superuser(Extra),
+    Attrs = emqx_authn_utils:client_attrs(Extra),
+    ExpireAt = expire_at(DisconnectAfterExpire, Extra),
+    try
+        ACL = acl(Extra, AclClaimName),
+        Result = merge_maps([ExpireAt, IsSuperuser, ACL, Attrs]),
+        {ok, Result}
+    catch
+        throw:{bad_acl_rule, Reason} ->
+            %% it's a invalid token, so ok to log
+            ?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{jwt => JWT}),
+            {error, bad_username_or_password}
+    end.
+
+expire_at(false, _Extra) ->
+    #{};
+expire_at(true, #{<<"exp">> := ExpireTime}) ->
+    #{expire_at => erlang:convert_time_unit(ExpireTime, second, millisecond)};
+expire_at(true, #{}) ->
+    #{}.
+
 acl(Claims, AclClaimName) ->
     case Claims of
         #{AclClaimName := Rules} ->
@@ -379,3 +398,6 @@ parse_rule(Rule) ->
         {error, Reason} ->
             throw({bad_acl_rule, Reason})
     end.
+
+merge_maps([]) -> #{};
+merge_maps([Map | Maps]) -> maps:merge(Map, merge_maps(Maps)).

apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl

@@ -127,6 +127,7 @@ common_fields() ->
             desc => ?DESC(acl_claim_name)
         }},
         {verify_claims, fun verify_claims/1},
+        {disconnect_after_expire, fun disconnect_after_expire/1},
         {from, fun from/1}
     ] ++ emqx_authn_schema:common_fields().
 
@@ -188,6 +189,11 @@ verify_claims(required) ->
 verify_claims(_) ->
     undefined.
 
+disconnect_after_expire(type) -> boolean();
+disconnect_after_expire(desc) -> ?DESC(?FUNCTION_NAME);
+disconnect_after_expire(default) -> true;
+disconnect_after_expire(_) -> undefined.
+
 do_check_verify_claims([]) ->
     true;
 %% _Expected can't be invalid since tuples may come only from converter

apps/emqx_auth_jwt/test/emqx_authn_jwt_SUITE.erl

@@ -55,7 +55,8 @@ t_hmac_based(_) ->
         algorithm => 'hmac-based',
         secret => Secret,
         secret_base64_encoded => false,
-        verify_claims => [{<<"username">>, <<"${username}">>}]
+        verify_claims => [{<<"username">>, <<"${username}">>}],
+        disconnect_after_expire => false
     },
     {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
 
@@ -179,7 +180,8 @@ t_public_key(_) ->
         use_jwks => false,
         algorithm => 'public-key',
         public_key => PublicKey,
-        verify_claims => []
+        verify_claims => [],
+        disconnect_after_expire => false
     },
     {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
 
@@ -207,7 +209,8 @@ t_jwt_in_username(_) ->
         algorithm => 'hmac-based',
         secret => Secret,
         secret_base64_encoded => false,
-        verify_claims => []
+        verify_claims => [],
+        disconnect_after_expire => false
     },
     {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
 
@@ -229,7 +232,8 @@ t_complex_template(_) ->
         algorithm => 'hmac-based',
         secret => Secret,
         secret_base64_encoded => false,
-        verify_claims => [{<<"id">>, <<"${username}-${clientid}">>}]
+        verify_claims => [{<<"id">>, <<"${username}-${clientid}">>}],
+        disconnect_after_expire => false
     },
     {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
 
@@ -269,7 +273,7 @@ t_jwks_renewal(_Config) ->
         algorithm => 'public-key',
         ssl => #{enable => false},
         verify_claims => [],
-
+        disconnect_after_expire => false,
         use_jwks => true,
         endpoint => "https://127.0.0.1:" ++ integer_to_list(?JWKS_PORT + 1) ++ ?JWKS_PATH,
         refresh_interval => 1000,
@@ -366,7 +370,8 @@ t_verify_claims(_) ->
         algorithm => 'hmac-based',
         secret => Secret,
         secret_base64_encoded => false,
-        verify_claims => [{<<"foo">>, <<"bar">>}]
+        verify_claims => [{<<"foo">>, <<"bar">>}],
+        disconnect_after_expire => false
     },
     {ok, State0} = emqx_authn_jwt:create(?AUTHN_ID, Config0),
 
@@ -456,7 +461,8 @@ t_verify_claim_clientid(_) ->
         algorithm => 'hmac-based',
         secret => Secret,
         secret_base64_encoded => false,
-        verify_claims => [{<<"cl">>, <<"${clientid}">>}]
+        verify_claims => [{<<"cl">>, <<"${clientid}">>}],
+        disconnect_after_expire => false
     },
     {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config),
 

apps/emqx_auth_jwt/test/emqx_authn_jwt_expire_SUITE.erl

@@ -0,0 +1,96 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authn_jwt_expire_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("emqx_auth/include/emqx_authn.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-define(PATH, [authentication]).
+
+all() -> emqx_common_test_helpers:all(?MODULE).
+
+init_per_testcase(_, Config) ->
+    _ = emqx_authn_test_lib:delete_authenticators(?PATH, ?GLOBAL),
+    Config.
+
+end_per_testcase(_, _Config) ->
+    _ = emqx_authn_test_lib:delete_authenticators(?PATH, ?GLOBAL),
+    ok.
+
+init_per_suite(Config) ->
+    Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_auth, emqx_auth_jwt], #{
+        work_dir => ?config(priv_dir, Config)
+    }),
+    [{apps, Apps} | Config].
+
+end_per_suite(Config) ->
+    emqx_authn_test_lib:delete_authenticators(?PATH, ?GLOBAL),
+    ok = emqx_cth_suite:stop(?config(apps, Config)),
+    ok.
+
+%%--------------------------------------------------------------------
+%% CT cases
+%%--------------------------------------------------------------------
+
+t_jwt_expire(_Config) ->
+    _ = process_flag(trap_exit, true),
+
+    {ok, _} = emqx:update_config(
+        ?PATH,
+        {create_authenticator, ?GLOBAL, auth_config()}
+    ),
+
+    {ok, [#{provider := emqx_authn_jwt}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
+
+    Expire = erlang:system_time(second) + 3,
+
+    Payload = #{
+        <<"username">> => <<"myuser">>,
+        <<"exp">> => Expire
+    },
+    JWS = emqx_authn_jwt_SUITE:generate_jws('hmac-based', Payload, <<"secret">>),
+
+    {ok, C} = emqtt:start_link([{username, <<"myuser">>}, {password, JWS}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(C),
+
+    receive
+        {disconnected, ?RC_NOT_AUTHORIZED, #{}} ->
+            ?assert(erlang:system_time(second) >= Expire)
+    after 5000 ->
+        ct:fail("Client should be disconnected by timeout")
+    end.
+
+%%--------------------------------------------------------------------
+%% Helper functions
+%%--------------------------------------------------------------------
+
+auth_config() ->
+    #{
+        <<"use_jwks">> => false,
+        <<"algorithm">> => <<"hmac-based">>,
+        <<"acl_claim_name">> => <<"acl">>,
+        <<"secret">> => <<"secret">>,
+        <<"mechanism">> => <<"jwt">>,
+        <<"verify_claims">> => #{<<"username">> => <<"${username}">>}
+        %% Should be enabled by default
+        %% <<"disconnect_after_expire">> => true
+    }.

apps/emqx_auth_jwt/test/emqx_authz_jwt_SUITE.erl

@@ -455,11 +455,12 @@ t_invalid_rule(_Config) ->
 authn_config() ->
     #{
         <<"mechanism">> => <<"jwt">>,
-        <<"use_jwks">> => <<"false">>,
+        <<"use_jwks">> => false,
         <<"algorithm">> => <<"hmac-based">>,
         <<"secret">> => ?SECRET,
-        <<"secret_base64_encoded">> => <<"false">>,
+        <<"secret_base64_encoded">> => false,
         <<"acl_claim_name">> => <<"acl">>,
+        <<"disconnect_after_expire">> => false,
         <<"verify_claims">> => #{
             <<"username">> => ?PH_USERNAME
         }

apps/emqx_bridge/test/emqx_bridge_testlib.erl

@@ -252,11 +252,14 @@ create_rule_and_action_http(BridgeType, RuleTopic, Config) ->
 create_rule_and_action_http(BridgeType, RuleTopic, Config, Opts) ->
     BridgeName = ?config(bridge_name, Config),
     BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
+    create_rule_and_action(BridgeId, RuleTopic, Opts).
+
+create_rule_and_action(Action, RuleTopic, Opts) ->
     SQL = maps:get(sql, Opts, <<"SELECT * FROM \"", RuleTopic/binary, "\"">>),
     Params = #{
         enable => true,
         sql => SQL,
-        actions => [BridgeId]
+        actions => [Action]
     },
     Path = emqx_mgmt_api_test_util:api_path(["rules"]),
     AuthHeader = emqx_mgmt_api_test_util:auth_header_(),

apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl

@@ -29,7 +29,8 @@
     on_query_async/4,
     on_batch_query/3,
     on_batch_query_async/4,
-    on_get_status/2
+    on_get_status/2,
+    on_format_query_result/1
 ]).
 
 %% callbacks of ecpool
@@ -459,6 +460,11 @@ handle_result({error, Error}) ->
 handle_result(Res) ->
     Res.
 
+on_format_query_result({ok, Result}) ->
+    #{result => ok, info => Result};
+on_format_query_result(Result) ->
+    Result.
+
 %%--------------------------------------------------------------------
 %% utils
 

apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl

@@ -38,7 +38,8 @@
     on_get_channels/1,
     on_query/3,
     on_batch_query/3,
-    on_get_status/2
+    on_get_status/2,
+    on_format_query_result/1
 ]).
 
 %% callbacks for ecpool
@@ -519,6 +520,13 @@ transform_and_log_clickhouse_result(ClickhouseErrorResult, ResourceID, SQL) ->
             to_error_tuple(ClickhouseErrorResult)
     end.
 
+on_format_query_result(ok) ->
+    #{result => ok, message => <<"">>};
+on_format_query_result({ok, Message}) ->
+    #{result => ok, message => Message};
+on_format_query_result(Result) ->
+    Result.
+
 to_recoverable_error({error, Reason}) ->
     {error, {recoverable_error, Reason}};
 to_recoverable_error(Error) ->

apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl

@@ -26,7 +26,8 @@
     on_add_channel/4,
     on_remove_channel/3,
     on_get_channels/1,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 -export([
@@ -184,6 +185,11 @@ on_batch_query(InstanceId, [{_ChannelId, _} | _] = Query, State) ->
 on_batch_query(_InstanceId, Query, _State) ->
     {error, {unrecoverable_error, {invalid_request, Query}}}.
 
+on_format_query_result({ok, Result}) ->
+    #{result => ok, info => Result};
+on_format_query_result(Result) ->
+    Result.
+
 health_check_timeout() ->
     2500.
 

apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl

@@ -27,6 +27,8 @@
 -export([execute/2]).
 -endif.
 
+-include_lib("emqx/include/emqx_trace.hrl").
+
 %%%===================================================================
 %%% API
 %%%===================================================================
@@ -107,7 +109,10 @@ do_query(Table, Query0, Templates, TraceRenderedCTX) ->
         Query = apply_template(Query0, Templates),
         emqx_trace:rendered_action_template_with_ctx(TraceRenderedCTX, #{
             table => Table,
-            query => {fun trace_format_query/1, Query}
+            query => #emqx_trace_format_func_data{
+                function = fun trace_format_query/1,
+                data = Query
+            }
         }),
         execute(Query, Table)
     catch

apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl

@@ -23,7 +23,8 @@
     on_add_channel/4,
     on_remove_channel/3,
     on_get_channels/1,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 -export([
@@ -286,6 +287,9 @@ on_query_async(
         InstanceId, {ChannelId, Msg}, ReplyFunAndArgs, State
     ).
 
+on_format_query_result(Result) ->
+    emqx_bridge_http_connector:on_format_query_result(Result).
+
 on_add_channel(
     InstanceId,
     #{channels := Channels} = State0,

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl

@@ -53,7 +53,8 @@
     on_add_channel/4,
     on_remove_channel/3,
     on_get_channels/1,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 -export([reply_delegator/2]).
@@ -489,6 +490,11 @@ handle_result({error, Reason} = Result, _Request, QueryMode, ResourceId) ->
 handle_result({ok, _} = Result, _Request, _QueryMode, _ResourceId) ->
     Result.
 
+on_format_query_result({ok, Info}) ->
+    #{result => ok, info => Info};
+on_format_query_result(Result) ->
+    Result.
+
 reply_delegator(ReplyFunAndArgs, Response) ->
     case Response of
         {error, Reason} when

apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl

@@ -27,7 +27,8 @@
     on_batch_query/3,
     on_query_async/4,
     on_batch_query_async/4,
-    on_get_status/2
+    on_get_status/2,
+    on_format_query_result/1
 ]).
 -export([reply_callback/2]).
 
@@ -453,6 +454,11 @@ do_query(InstId, Channel, Client, Points) ->
             end
     end.
 
+on_format_query_result({ok, {affected_rows, Rows}}) ->
+    #{result => ok, affected_rows => Rows};
+on_format_query_result(Result) ->
+    Result.
+
 do_async_query(InstId, Channel, Client, Points, ReplyFunAndArgs) ->
     ?SLOG(info, #{
         msg => "greptimedb_write_point_async",

apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl

@@ -20,6 +20,7 @@
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("emqx/include/emqx_trace.hrl").
 
 -behaviour(emqx_resource).
 
@@ -35,7 +36,8 @@
     on_add_channel/4,
     on_remove_channel/3,
     on_get_channels/1,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 -export([reply_delegator/3]).
@@ -231,6 +233,7 @@ on_start(
         host => Host,
         port => Port,
         connect_timeout => ConnectTimeout,
+        scheme => Scheme,
         request => preprocess_request(maps:get(request, Config, undefined))
     },
     case start_pool(InstId, PoolOpts) of
@@ -358,7 +361,7 @@ on_query(InstId, {Method, Request, Timeout}, State) ->
 on_query(
     InstId,
     {ActionId, KeyOrNum, Method, Request, Timeout, Retry},
-    #{host := Host} = State
+    #{host := Host, port := Port, scheme := Scheme} = State
 ) ->
     ?TRACE(
         "QUERY",
@@ -372,7 +375,7 @@ on_query(
         }
     ),
     NRequest = formalize_request(Method, Request),
-    trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout),
+    trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout),
     Worker = resolve_pool_worker(State, KeyOrNum),
     Result0 = ehttpc:request(
         Worker,
@@ -468,7 +471,7 @@ on_query_async(
     InstId,
     {ActionId, KeyOrNum, Method, Request, Timeout},
     ReplyFunAndArgs,
-    #{host := Host} = State
+    #{host := Host, port := Port, scheme := Scheme} = State
 ) ->
     Worker = resolve_pool_worker(State, KeyOrNum),
     ?TRACE(
@@ -482,7 +485,7 @@ on_query_async(
         }
     ),
     NRequest = formalize_request(Method, Request),
-    trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout),
+    trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout),
     MaxAttempts = maps:get(max_attempts, State, 3),
     Context = #{
         attempt => 1,
@@ -491,7 +494,8 @@ on_query_async(
         key_or_num => KeyOrNum,
         method => Method,
         request => NRequest,
-        timeout => Timeout
+        timeout => Timeout,
+        trace_metadata => logger:get_process_metadata()
     },
     ok = ehttpc:request_async(
         Worker,
@@ -502,17 +506,25 @@ on_query_async(
     ),
     {ok, Worker}.
 
-trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) ->
+trace_rendered_action_template(ActionId, Scheme, Host, Port, Method, NRequest, Timeout) ->
     case NRequest of
         {Path, Headers} ->
             emqx_trace:rendered_action_template(
                 ActionId,
                 #{
                     host => Host,
+                    port => Port,
                     path => Path,
                     method => Method,
-                    headers => {fun emqx_utils_redact:redact_headers/1, Headers},
+                    headers => #emqx_trace_format_func_data{
-                    timeout => Timeout
+                        function = fun emqx_utils_redact:redact_headers/1,
+                        data = Headers
+                    },
+                    timeout => Timeout,
+                    url => #emqx_trace_format_func_data{
+                        function = fun render_url/1,
+                        data = {Scheme, Host, Port, Path}
+                    }
                 }
             );
         {Path, Headers, Body} ->
@@ -520,15 +532,42 @@ trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) ->
                 ActionId,
                 #{
                     host => Host,
+                    port => Port,
                     path => Path,
                     method => Method,
-                    headers => {fun emqx_utils_redact:redact_headers/1, Headers},
+                    headers => #emqx_trace_format_func_data{
+                        function = fun emqx_utils_redact:redact_headers/1,
+                        data = Headers
+                    },
                     timeout => Timeout,
-                    body => {fun log_format_body/1, Body}
+                    body => #emqx_trace_format_func_data{
+                        function = fun log_format_body/1,
+                        data = Body
+                    },
+                    url => #emqx_trace_format_func_data{
+                        function = fun render_url/1,
+                        data = {Scheme, Host, Port, Path}
+                    }
                 }
             )
     end.
 
+render_url({Scheme, Host, Port, Path}) ->
+    SchemeStr =
+        case Scheme of
+            http ->
+                <<"http://">>;
+            https ->
+                <<"https://">>
+        end,
+    unicode:characters_to_binary([
+        SchemeStr,
+        Host,
+        <<":">>,
+        erlang:integer_to_binary(Port),
+        Path
+    ]).
+
 log_format_body(Body) ->
     unicode:characters_to_binary(Body).
 
@@ -604,6 +643,26 @@ on_get_channel_status(
     %% XXX: Reuse the connector status
     on_get_status(InstId, State).
 
+on_format_query_result({ok, Status, Headers, Body}) ->
+    #{
+        result => ok,
+        response => #{
+            status => Status,
+            headers => Headers,
+            body => Body
+        }
+    };
+on_format_query_result({ok, Status, Headers}) ->
+    #{
+        result => ok,
+        response => #{
+            status => Status,
+            headers => Headers
+        }
+    };
+on_format_query_result(Result) ->
+    Result.
+
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
@@ -809,9 +868,15 @@ to_bin(Str) when is_list(Str) ->
 to_bin(Atom) when is_atom(Atom) ->
     atom_to_binary(Atom, utf8).
 
-reply_delegator(Context, ReplyFunAndArgs, Result0) ->
+reply_delegator(
+    #{trace_metadata := TraceMetadata} = Context,
+    ReplyFunAndArgs,
+    Result0
+) ->
     spawn(fun() ->
+        logger:set_process_metadata(TraceMetadata),
         Result = transform_result(Result0),
+        logger:unset_process_metadata(),
         maybe_retry(Result, Context, ReplyFunAndArgs)
     end).
 

apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl

@@ -27,7 +27,8 @@
     on_batch_query/3,
     on_query_async/4,
     on_batch_query_async/4,
-    on_get_status/2
+    on_get_status/2,
+    on_format_query_result/1
 ]).
 -export([reply_callback/2]).
 
@@ -209,6 +210,9 @@ on_batch_query_async(
             {error, {unrecoverable_error, Reason}}
     end.
 
+on_format_query_result(Result) ->
+    emqx_bridge_http_connector:on_format_query_result(Result).
+
 on_get_status(_InstId, #{client := Client}) ->
     case influxdb:is_alive(Client) andalso ok =:= influxdb:check_auth(Client) of
         true ->

apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl

@@ -26,7 +26,8 @@
     on_add_channel/4,
     on_remove_channel/3,
     on_get_channels/1,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 -export([
@@ -388,6 +389,9 @@ on_batch_query(
             Error
     end.
 
+on_format_query_result(Result) ->
+    emqx_bridge_http_connector:on_format_query_result(Result).
+
 on_add_channel(
     InstanceId,
     #{iotdb_version := Version, channels := Channels} = OldState0,

apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl

@@ -39,7 +39,8 @@
     on_add_channel/4,
     on_remove_channel/3,
     on_get_channels/1,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 -export([
@@ -318,6 +319,11 @@ handle_result({error, Reason} = Error, Requests, InstanceId) ->
     }),
     Error.
 
+on_format_query_result({ok, Result}) ->
+    #{result => ok, info => Result};
+on_format_query_result(Result) ->
+    Result.
+
 parse_template(Config) ->
     #{payload_template := PayloadTemplate, partition_key := PartitionKeyTemplate} = Config,
     Templates = #{send_message => PayloadTemplate, partition_key => PartitionKeyTemplate},

apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl

@@ -18,7 +18,8 @@
     on_get_status/2,
     on_query/3,
     on_start/2,
-    on_stop/2
+    on_stop/2,
+    on_format_query_result/1
 ]).
 
 %%========================================================================================
@@ -85,6 +86,11 @@ on_query(InstanceId, {Channel, Message0}, #{channels := Channels, connector_stat
 on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) ->
     emqx_mongodb:on_query(InstanceId, Request, ConnectorState).
 
+on_format_query_result({{Result, Info}, Documents}) ->
+    #{result => Result, info => Info, documents => Documents};
+on_format_query_result(Result) ->
+    Result.
+
 on_remove_channel(_InstanceId, #{channels := Channels} = State, ChannelId) ->
     NewState = State#{channels => maps:remove(ChannelId, Channels)},
     {ok, NewState}.

apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl

@@ -27,7 +27,8 @@
     on_add_channel/4,
     on_remove_channel/3,
     on_get_channels/1,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 -export([connector_examples/1]).
@@ -175,6 +176,11 @@ on_batch_query(
             Error
     end.
 
+on_format_query_result({ok, StatusCode, BodyMap}) ->
+    #{result => ok, status_code => StatusCode, body => BodyMap};
+on_format_query_result(Result) ->
+    Result.
+
 on_get_status(_InstanceId, #{server := Server}) ->
     Result =
         case opentsdb_connectivity(Server) of

apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl

@@ -20,7 +20,8 @@
     on_get_status/2,
     on_get_channel_status/3,
     on_query/3,
-    on_query_async/4
+    on_query_async/4,
+    on_format_query_result/1
 ]).
 
 -type pulsar_client_id() :: atom().
@@ -234,6 +235,11 @@ on_query_async2(ChannelId, Producers, Message, MessageTmpl, AsyncReplyFn) ->
     }),
     pulsar:send(Producers, [PulsarMessage], #{callback_fn => AsyncReplyFn}).
 
+on_format_query_result({ok, Info}) ->
+    #{result => ok, info => Info};
+on_format_query_result(Result) ->
+    Result.
+
 %%-------------------------------------------------------------------------------------
 %% Internal fns
 %%-------------------------------------------------------------------------------------

apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl

@@ -20,7 +20,8 @@
     on_query/3,
     on_batch_query/3,
     on_get_status/2,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 %% -------------------------------------------------------------------------------------------------
@@ -161,6 +162,11 @@ on_batch_query(
             Error
     end.
 
+on_format_query_result({ok, Msg}) ->
+    #{result => ok, message => Msg};
+on_format_query_result(Res) ->
+    Res.
+
 %% -------------------------------------------------------------------------------------------------
 %% private helpers
 %% -------------------------------------------------------------------------------------------------

apps/emqx_bridge_s3/rebar.config

@@ -2,5 +2,6 @@
 
 {erl_opts, [debug_info]}.
 {deps, [
-    {emqx_resource, {path, "../../apps/emqx_resource"}}
+    {emqx_resource, {path, "../../apps/emqx_resource"}},
+    {emqx_connector_aggregator, {path, "../../apps/emqx_connector_aggregator"}}
 ]}.

apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src

@@ -7,6 +7,7 @@
         stdlib,
         erlcloud,
         emqx_resource,
+        emqx_connector_aggregator,
         emqx_s3
     ]},
     {env, [
@@ -18,7 +19,6 @@
             emqx_bridge_s3_connector_info
         ]}
     ]},
-    {mod, {emqx_bridge_s3_app, []}},
     {modules, []},
     {links, []}
 ]}.

apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl

@@ -1,212 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%--------------------------------------------------------------------
-
-%% This module takes aggregated records from a buffer and delivers them to S3,
-%% wrapped in a configurable container (though currently there's only CSV).
--module(emqx_bridge_s3_aggreg_delivery).
-
--include_lib("snabbkaffe/include/trace.hrl").
--include("emqx_bridge_s3_aggregator.hrl").
-
--export([start_link/3]).
-
-%% Internal exports
--export([
-    init/4,
-    loop/3
-]).
-
--behaviour(emqx_template).
--export([lookup/2]).
-
-%% Sys
--export([
-    system_continue/3,
-    system_terminate/4,
-    format_status/2
-]).
-
--record(delivery, {
-    name :: _Name,
-    container :: emqx_bridge_s3_aggreg_csv:container(),
-    reader :: emqx_bridge_s3_aggreg_buffer:reader(),
-    upload :: emqx_s3_upload:t(),
-    empty :: boolean()
-}).
-
--type state() :: #delivery{}.
-
-%%
-
-start_link(Name, Buffer, Opts) ->
-    proc_lib:start_link(?MODULE, init, [self(), Name, Buffer, Opts]).
-
-%%
-
--spec init(pid(), _Name, buffer(), _Opts :: map()) -> no_return().
-init(Parent, Name, Buffer, Opts) ->
-    ?tp(s3_aggreg_delivery_started, #{action => Name, buffer => Buffer}),
-    Reader = open_buffer(Buffer),
-    Delivery = init_delivery(Name, Reader, Buffer, Opts#{action => Name}),
-    _ = erlang:process_flag(trap_exit, true),
-    ok = proc_lib:init_ack({ok, self()}),
-    loop(Delivery, Parent, []).
-
-init_delivery(Name, Reader, Buffer, Opts = #{container := ContainerOpts}) ->
-    #delivery{
-        name = Name,
-        container = mk_container(ContainerOpts),
-        reader = Reader,
-        upload = mk_upload(Buffer, Opts),
-        empty = true
-    }.
-
-open_buffer(#buffer{filename = Filename}) ->
-    case file:open(Filename, [read, binary, raw]) of
-        {ok, FD} ->
-            {_Meta, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(FD),
-            Reader;
-        {error, Reason} ->
-            error({buffer_open_failed, Reason})
-    end.
-
-mk_container(#{type := csv, column_order := OrderOpt}) ->
-    %% TODO: Deduplicate?
-    ColumnOrder = lists:map(fun emqx_utils_conv:bin/1, OrderOpt),
-    emqx_bridge_s3_aggreg_csv:new(#{column_order => ColumnOrder}).
-
-mk_upload(
-    Buffer,
-    Opts = #{
-        bucket := Bucket,
-        upload_options := UploadOpts,
-        client_config := Config,
-        uploader_config := UploaderConfig
-    }
-) ->
-    Client = emqx_s3_client:create(Bucket, Config),
-    Key = mk_object_key(Buffer, Opts),
-    emqx_s3_upload:new(Client, Key, UploadOpts, UploaderConfig).
-
-mk_object_key(Buffer, #{action := Name, key := Template}) ->
-    emqx_template:render_strict(Template, {?MODULE, {Name, Buffer}}).
-
-%%
-
--spec loop(state(), pid(), [sys:debug_option()]) -> no_return().
-loop(Delivery, Parent, Debug) ->
-    %% NOTE: This function is mocked in tests.
-    receive
-        Msg -> handle_msg(Msg, Delivery, Parent, Debug)
-    after 0 ->
-        process_delivery(Delivery, Parent, Debug)
-    end.
-
-process_delivery(Delivery0 = #delivery{reader = Reader0}, Parent, Debug) ->
-    case emqx_bridge_s3_aggreg_buffer:read(Reader0) of
-        {Records = [#{} | _], Reader} ->
-            Delivery1 = Delivery0#delivery{reader = Reader},
-            Delivery2 = process_append_records(Records, Delivery1),
-            Delivery = process_write(Delivery2),
-            loop(Delivery, Parent, Debug);
-        {[], Reader} ->
-            Delivery = Delivery0#delivery{reader = Reader},
-            loop(Delivery, Parent, Debug);
-        eof ->
-            process_complete(Delivery0);
-        {Unexpected, _Reader} ->
-            exit({buffer_unexpected_record, Unexpected})
-    end.
-
-process_append_records(Records, Delivery = #delivery{container = Container0, upload = Upload0}) ->
-    {Writes, Container} = emqx_bridge_s3_aggreg_csv:fill(Records, Container0),
-    {ok, Upload} = emqx_s3_upload:append(Writes, Upload0),
-    Delivery#delivery{
-        container = Container,
-        upload = Upload,
-        empty = false
-    }.
-
-process_write(Delivery = #delivery{upload = Upload0}) ->
-    case emqx_s3_upload:write(Upload0) of
-        {ok, Upload} ->
-            Delivery#delivery{upload = Upload};
-        {cont, Upload} ->
-            process_write(Delivery#delivery{upload = Upload});
-        {error, Reason} ->
-            _ = emqx_s3_upload:abort(Upload0),
-            exit({upload_failed, Reason})
-    end.
-
-process_complete(#delivery{name = Name, empty = true}) ->
-    ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => empty}),
-    exit({shutdown, {skipped, empty}});
-process_complete(#delivery{name = Name, container = Container, upload = Upload0}) ->
-    Trailer = emqx_bridge_s3_aggreg_csv:close(Container),
-    {ok, Upload} = emqx_s3_upload:append(Trailer, Upload0),
-    case emqx_s3_upload:complete(Upload) of
-        {ok, Completed} ->
-            ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => Completed}),
-            ok;
-        {error, Reason} ->
-            _ = emqx_s3_upload:abort(Upload),
-            exit({upload_failed, Reason})
-    end.
-
-%%
-
-handle_msg({system, From, Msg}, Delivery, Parent, Debug) ->
-    sys:handle_system_msg(Msg, From, Parent, ?MODULE, Debug, Delivery);
-handle_msg({'EXIT', Parent, Reason}, Delivery, Parent, Debug) ->
-    system_terminate(Reason, Parent, Debug, Delivery);
-handle_msg(_Msg, Delivery, Parent, Debug) ->
-    loop(Parent, Debug, Delivery).
-
--spec system_continue(pid(), [sys:debug_option()], state()) -> no_return().
-system_continue(Parent, Debug, Delivery) ->
-    loop(Delivery, Parent, Debug).
-
--spec system_terminate(_Reason, pid(), [sys:debug_option()], state()) -> _.
-system_terminate(_Reason, _Parent, _Debug, #delivery{upload = Upload}) ->
-    emqx_s3_upload:abort(Upload).
-
--spec format_status(normal, Args :: [term()]) -> _StateFormatted.
-format_status(_Normal, [_PDict, _SysState, _Parent, _Debug, Delivery]) ->
-    Delivery#delivery{
-        upload = emqx_s3_upload:format(Delivery#delivery.upload)
-    }.
-
-%%
-
--spec lookup(emqx_template:accessor(), {_Name, buffer()}) ->
-    {ok, integer() | string()} | {error, undefined}.
-lookup([<<"action">>], {Name, _Buffer}) ->
-    {ok, mk_fs_safe_string(Name)};
-lookup(Accessor, {_Name, Buffer = #buffer{}}) ->
-    lookup_buffer_var(Accessor, Buffer);
-lookup(_Accessor, _Context) ->
-    {error, undefined}.
-
-lookup_buffer_var([<<"datetime">>, Format], #buffer{since = Since}) ->
-    {ok, format_timestamp(Since, Format)};
-lookup_buffer_var([<<"datetime_until">>, Format], #buffer{until = Until}) ->
-    {ok, format_timestamp(Until, Format)};
-lookup_buffer_var([<<"sequence">>], #buffer{seq = Seq}) ->
-    {ok, Seq};
-lookup_buffer_var([<<"node">>], #buffer{}) ->
-    {ok, mk_fs_safe_string(atom_to_binary(erlang:node()))};
-lookup_buffer_var(_Binding, _Context) ->
-    {error, undefined}.
-
-format_timestamp(Timestamp, <<"rfc3339utc">>) ->
-    String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}, {offset, "Z"}]),
-    mk_fs_safe_string(String);
-format_timestamp(Timestamp, <<"rfc3339">>) ->
-    String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}]),
-    mk_fs_safe_string(String);
-format_timestamp(Timestamp, <<"unix">>) ->
-    Timestamp.
-
-mk_fs_safe_string(String) ->
-    unicode:characters_to_binary(string:replace(String, ":", "_", all)).

apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl

@@ -1,16 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%--------------------------------------------------------------------
-
--module(emqx_bridge_s3_app).
-
--behaviour(application).
--export([start/2, stop/1]).
-
-%%
-
-start(_StartType, _StartArgs) ->
-    emqx_bridge_s3_sup:start_link().
-
-stop(_State) ->
-    ok.

apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl

@@ -7,6 +7,8 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/trace.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
+-include_lib("emqx/include/emqx_trace.hrl").
+-include_lib("emqx_connector_aggregator/include/emqx_connector_aggregator.hrl").
 -include("emqx_bridge_s3.hrl").
 
 -behaviour(emqx_resource).
@@ -23,6 +25,19 @@
     on_get_channel_status/3
 ]).
 
+-behaviour(emqx_connector_aggreg_delivery).
+-export([
+    init_transfer_state/2,
+    process_append/2,
+    process_write/1,
+    process_complete/1,
+    process_format_status/1,
+    process_terminate/1
+]).
+
+-behaviour(emqx_template).
+-export([lookup/2]).
+
 -type config() :: #{
     access_key_id => string(),
     secret_access_key => emqx_secret:t(string()),
@@ -204,13 +219,14 @@ start_channel(State, #{
         key => emqx_bridge_s3_aggreg_upload:mk_key_template(Parameters),
         container => Container,
         upload_options => emqx_bridge_s3_aggreg_upload:mk_upload_options(Parameters),
+        callback_module => ?MODULE,
         client_config => maps:get(client_config, State),
         uploader_config => maps:with([min_part_size, max_part_size], Parameters)
     },
-    _ = emqx_bridge_s3_sup:delete_child({Type, Name}),
+    _ = emqx_connector_aggreg_sup:delete_child({Type, Name}),
-    {ok, SupPid} = emqx_bridge_s3_sup:start_child(#{
+    {ok, SupPid} = emqx_connector_aggreg_sup:start_child(#{
         id => {Type, Name},
-        start => {emqx_bridge_s3_aggreg_upload_sup, start_link, [Name, AggregOpts, DeliveryOpts]},
+        start => {emqx_connector_aggreg_upload_sup, start_link, [Name, AggregOpts, DeliveryOpts]},
         type => supervisor,
         restart => permanent
     }),
@@ -219,7 +235,7 @@ start_channel(State, #{
         name => Name,
         bucket => Bucket,
         supervisor => SupPid,
-        on_stop => fun() -> emqx_bridge_s3_sup:delete_child({Type, Name}) end
+        on_stop => fun() -> emqx_connector_aggreg_sup:delete_child({Type, Name}) end
     }.
 
 upload_options(Parameters) ->
@@ -241,7 +257,7 @@ channel_status(#{type := ?ACTION_UPLOAD}, _State) ->
 channel_status(#{type := ?ACTION_AGGREGATED_UPLOAD, name := Name, bucket := Bucket}, State) ->
     %% NOTE: This will effectively trigger uploads of buffers yet to be uploaded.
     Timestamp = erlang:system_time(second),
-    ok = emqx_bridge_s3_aggregator:tick(Name, Timestamp),
+    ok = emqx_connector_aggregator:tick(Name, Timestamp),
     ok = check_bucket_accessible(Bucket, State),
     ok = check_aggreg_upload_errors(Name),
     ?status_connected.
@@ -263,7 +279,7 @@ check_bucket_accessible(Bucket, #{client_config := Config}) ->
     end.
 
 check_aggreg_upload_errors(Name) ->
-    case emqx_bridge_s3_aggregator:take_error(Name) of
+    case emqx_connector_aggregator:take_error(Name) of
         [Error] ->
             %% TODO
             %% This approach means that, for example, 3 upload failures will cause
@@ -320,7 +336,10 @@ run_simple_upload(
     emqx_trace:rendered_action_template(ChannelID, #{
         bucket => Bucket,
         key => Key,
-        content => Content
+        content => #emqx_trace_format_func_data{
+            function = fun unicode:characters_to_binary/1,
+            data = Content
+        }
     }),
     case emqx_s3_client:put_object(Client, Key, UploadOpts, Content) of
         ok ->
@@ -336,7 +355,7 @@ run_simple_upload(
 
 run_aggregated_upload(InstId, Records, #{name := Name}) ->
     Timestamp = erlang:system_time(second),
-    case emqx_bridge_s3_aggregator:push_records(Name, Timestamp, Records) of
+    case emqx_connector_aggregator:push_records(Name, Timestamp, Records) of
         ok ->
             ?tp(s3_bridge_aggreg_push_ok, #{instance_id => InstId, name => Name}),
             ok;
@@ -372,3 +391,84 @@ render_content(Template, Data) ->
 
 iolist_to_string(IOList) ->
     unicode:characters_to_list(IOList).
+
+%% `emqx_connector_aggreg_delivery` APIs
+
+-spec init_transfer_state(buffer_map(), map()) -> emqx_s3_upload:t().
+init_transfer_state(BufferMap, Opts) ->
+    #{
+        bucket := Bucket,
+        upload_options := UploadOpts,
+        client_config := Config,
+        uploader_config := UploaderConfig
+    } = Opts,
+    Client = emqx_s3_client:create(Bucket, Config),
+    Key = mk_object_key(BufferMap, Opts),
+    emqx_s3_upload:new(Client, Key, UploadOpts, UploaderConfig).
+
+mk_object_key(BufferMap, #{action := Name, key := Template}) ->
+    emqx_template:render_strict(Template, {?MODULE, {Name, BufferMap}}).
+
+process_append(Writes, Upload0) ->
+    {ok, Upload} = emqx_s3_upload:append(Writes, Upload0),
+    Upload.
+
+process_write(Upload0) ->
+    case emqx_s3_upload:write(Upload0) of
+        {ok, Upload} ->
+            {ok, Upload};
+        {cont, Upload} ->
+            process_write(Upload);
+        {error, Reason} ->
+            _ = emqx_s3_upload:abort(Upload0),
+            {error, Reason}
+    end.
+
+process_complete(Upload) ->
+    case emqx_s3_upload:complete(Upload) of
+        {ok, Completed} ->
+            {ok, Completed};
+        {error, Reason} ->
+            _ = emqx_s3_upload:abort(Upload),
+            exit({upload_failed, Reason})
+    end.
+
+process_format_status(Upload) ->
+    emqx_s3_upload:format(Upload).
+
+process_terminate(Upload) ->
+    emqx_s3_upload:abort(Upload).
+
+%% `emqx_template` APIs
+
+-spec lookup(emqx_template:accessor(), {_Name, buffer_map()}) ->
+    {ok, integer() | string()} | {error, undefined}.
+lookup([<<"action">>], {Name, _Buffer}) ->
+    {ok, mk_fs_safe_string(Name)};
+lookup(Accessor, {_Name, Buffer = #{}}) ->
+    lookup_buffer_var(Accessor, Buffer);
+lookup(_Accessor, _Context) ->
+    {error, undefined}.
+
+lookup_buffer_var([<<"datetime">>, Format], #{since := Since}) ->
+    {ok, format_timestamp(Since, Format)};
+lookup_buffer_var([<<"datetime_until">>, Format], #{until := Until}) ->
+    {ok, format_timestamp(Until, Format)};
+lookup_buffer_var([<<"sequence">>], #{seq := Seq}) ->
+    {ok, Seq};
+lookup_buffer_var([<<"node">>], #{}) ->
+    {ok, mk_fs_safe_string(atom_to_binary(erlang:node()))};
+lookup_buffer_var(_Binding, _Context) ->
+    {error, undefined}.
+
+format_timestamp(Timestamp, <<"rfc3339utc">>) ->
+    String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}, {offset, "Z"}]),
+    mk_fs_safe_string(String);
+format_timestamp(Timestamp, <<"rfc3339">>) ->
+    String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}]),
+    mk_fs_safe_string(String);
+format_timestamp(Timestamp, <<"unix">>) ->
+    Timestamp.
+
+mk_fs_safe_string(String) ->
+    unicode:characters_to_binary(string:replace(String, ":", "_", all)).

apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl

@@ -170,7 +170,7 @@ t_aggreg_upload(Config) ->
     ]),
     ok = send_messages(BridgeName, MessageEvents),
     %% Wait until the delivery is completed.
-    ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
+    ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
     %% Check the uploaded objects.
     _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
     ?assertMatch(
@@ -217,7 +217,7 @@ t_aggreg_upload_rule(Config) ->
         emqx_message:make(?FUNCTION_NAME, T3 = <<"s3/empty">>, P3 = <<>>),
         emqx_message:make(?FUNCTION_NAME, <<"not/s3">>, <<"should not be here">>)
     ]),
-    ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
+    ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
     %% Check the uploaded objects.
     _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
     _CSV = [Header | Rows] = fetch_parse_csv(Bucket, Key),
@@ -258,15 +258,15 @@ t_aggreg_upload_restart(Config) ->
         {<<"C3">>, T3 = <<"t/42">>, P3 = <<"">>}
     ]),
     ok = send_messages(BridgeName, MessageEvents),
-    {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}),
+    {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_records_written, action := BridgeName}),
     %% Restart the bridge.
     {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName),
     {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName),
     %% Send some more messages.
     ok = send_messages(BridgeName, MessageEvents),
-    {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}),
+    {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_records_written, action := BridgeName}),
     %% Wait until the delivery is completed.
-    {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
+    {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
     %% Check there's still only one upload.
     _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
     _Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key),
@@ -300,18 +300,18 @@ t_aggreg_upload_restart_corrupted(Config) ->
     %% Ensure that they span multiple batch queries.
     ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages1), 1),
     {ok, _} = ?block_until(
-        #{?snk_kind := s3_aggreg_records_written, action := BridgeName},
+        #{?snk_kind := connector_aggreg_records_written, action := BridgeName},
         infinity,
         0
     ),
     %% Find out the buffer file.
     {ok, #{filename := Filename}} = ?block_until(
-        #{?snk_kind := s3_aggreg_buffer_allocated, action := BridgeName}
+        #{?snk_kind := connector_aggreg_buffer_allocated, action := BridgeName}
     ),
     %% Stop the bridge, corrupt the buffer file, and restart the bridge.
     {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName),
     BufferFileSize = filelib:file_size(Filename),
-    ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, BufferFileSize div 2),
+    ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, BufferFileSize div 2),
     {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName),
     %% Send some more messages.
     Messages2 = [
@@ -320,7 +320,7 @@ t_aggreg_upload_restart_corrupted(Config) ->
     ],
     ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages2), 0),
     %% Wait until the delivery is completed.
-    {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
+    {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
     %% Check that upload contains part of the first batch and all of the second batch.
     _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
     CSV = [_Header | Rows] = fetch_parse_csv(Bucket, Key),
@@ -362,7 +362,7 @@ t_aggreg_pending_upload_restart(Config) ->
     %% Restart the bridge.
     {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName),
     %% Wait until the delivery is completed.
-    {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}),
+    {ok, _} = ?block_until(#{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}),
     %% Check that delivery contains all the messages.
     _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket),
     [_Header | Rows] = fetch_parse_csv(Bucket, Key),
@@ -392,7 +392,9 @@ t_aggreg_next_rotate(Config) ->
     NSent = receive_sender_reports(Senders),
     %% Wait for the last delivery to complete.
     ok = timer:sleep(round(?CONF_TIME_INTERVAL * 0.5)),
-    ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}, infinity, 0),
+    ?block_until(
+        #{?snk_kind := connector_aggreg_delivery_completed, action := BridgeName}, infinity, 0
+    ),
     %% There should be at least 2 time windows of aggregated records.
     Uploads = [K || #{key := K} <- emqx_bridge_s3_test_helpers:list_objects(Bucket)],
     DTs = [DT || K <- Uploads, [_Action, _Node, DT | _] <- [string:split(K, "/", all)]],

apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl

@@ -48,11 +48,3 @@ list_pending_uploads(Bucket, Key) ->
     {ok, Props} = erlcloud_s3:list_multipart_uploads(Bucket, [{prefix, Key}], [], AwsConfig),
     Uploads = proplists:get_value(uploads, Props),
     lists:map(fun maps:from_list/1, Uploads).
-
-%% File utilities
-
-truncate_at(Filename, Pos) ->
-    {ok, FD} = file:open(Filename, [read, write, binary]),
-    {ok, Pos} = file:position(FD, Pos),
-    ok = file:truncate(FD),
-    ok = file:close(FD).

apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl

@@ -39,7 +39,8 @@
     on_add_channel/4,
     on_remove_channel/3,
     on_get_channels/1,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 %% callbacks for ecpool
@@ -320,6 +321,11 @@ on_batch_query(ResourceId, BatchRequests, State) ->
     ),
     do_query(ResourceId, BatchRequests, ?SYNC_QUERY_MODE, State).
 
+on_format_query_result({ok, Rows}) ->
+    #{result => ok, rows => Rows};
+on_format_query_result(Result) ->
+    Result.
+
 on_get_status(_InstanceId, #{pool_name := PoolName} = _State) ->
     Health = emqx_resource_pool:health_check_workers(
         PoolName,

apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl

@@ -28,7 +28,8 @@
     on_add_channel/4,
     on_remove_channel/3,
     on_get_channels/1,
-    on_get_channel_status/3
+    on_get_channel_status/3,
+    on_format_query_result/1
 ]).
 
 -export([connector_examples/1]).
@@ -215,6 +216,11 @@ on_batch_query(InstanceId, BatchReq, State) ->
     ?SLOG(error, LogMeta#{msg => "invalid_request"}),
     {error, {unrecoverable_error, invalid_request}}.
 
+on_format_query_result({ok, ResultMap}) ->
+    #{result => ok, info => ResultMap};
+on_format_query_result(Result) ->
+    Result.
+
 on_get_status(_InstanceId, #{pool_name := PoolName} = State) ->
     case
         emqx_resource_pool:health_check_workers(

apps/emqx_connector_aggregator/BSL.txt

@@ -0,0 +1,94 @@
+Business Source License 1.1
+
+Licensor:             Hangzhou EMQ Technologies Co., Ltd.
+Licensed Work:        EMQX Enterprise Edition
+                      The Licensed Work is (c) 2024
+                      Hangzhou EMQ Technologies Co., Ltd.
+Additional Use Grant: Students and educators are granted right to copy,
+                      modify, and create derivative work for research
+                      or education.
+Change Date:          2028-05-06
+Change License:       Apache License, Version 2.0
+
+For information about alternative licensing arrangements for the Software,
+please contact Licensor: https://www.emqx.com/en/contact
+
+Notice
+
+The Business Source License (this document, or the “License”) is not an Open
+Source license. However, the Licensed Work will eventually be made available
+under an Open Source License, as stated in this License.
+
+License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
+“Business Source License” is a trademark of MariaDB Corporation Ab.
+
+-----------------------------------------------------------------------------
+
+Business Source License 1.1
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited
+production use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.
+
+MariaDB hereby grants you permission to use this License’s text to license
+your works, and to refer to it using the trademark “Business Source License”,
+as long as you comply with the Covenants of Licensor below.
+
+Covenants of Licensor
+
+In consideration of the right to use this License’s text and the “Business
+Source License” name and trademark, Licensor covenants to MariaDB, and to all
+other recipients of the licensed work to be provided by Licensor:
+
+1. To specify as the Change License the GPL Version 2.0 or any later version,
+   or a license that is compatible with GPL Version 2.0 or a later version,
+   where “compatible” means that software provided under the Change License can
+   be included in a program with software provided under GPL Version 2.0 or a
+   later version. Licensor may specify additional Change Licenses without
+   limitation.
+
+2. To either: (a) specify an additional grant of rights to use that does not
+   impose any additional restriction on the right granted in this License, as
+   the Additional Use Grant; or (b) insert the text “None”.
+
+3. To specify a Change Date.
+
+4. Not to modify this License in any other way.

apps/emqx_connector_aggregator/README.md

@@ -0,0 +1,11 @@
+# EMQX Connector Aggregator
+
+This application provides common logic for connector and action implementations of EMQX to aggregate multiple incoming messsages into a container file before sending it to a blob storage backend.
+
+## Contributing
+
+Please see our [contributing.md](../../CONTRIBUTING.md).
+
+## License
+
+EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt).

apps/emqx_connector_aggregator/include/emqx_connector_aggregator.hrl

@@ -3,8 +3,8 @@
 %%--------------------------------------------------------------------
 
 -record(buffer, {
-    since :: emqx_bridge_s3_aggregator:timestamp(),
+    since :: emqx_connector_aggregator:timestamp(),
-    until :: emqx_bridge_s3_aggregator:timestamp(),
+    until :: emqx_connector_aggregator:timestamp(),
     seq :: non_neg_integer(),
     filename :: file:filename(),
     fd :: file:io_device() | undefined,
@@ -13,3 +13,11 @@
 }).
 
 -type buffer() :: #buffer{}.
+
+-type buffer_map() :: #{
+    since := emqx_connector_aggregator:timestamp(),
+    until := emqx_connector_aggregator:timestamp(),
+    seq := non_neg_integer(),
+    filename := file:filename(),
+    max_records := pos_integer() | undefined
+}.

apps/emqx_connector_aggregator/rebar.config

@@ -0,0 +1,7 @@
+%% -*- mode: erlang; -*-
+
+{deps, [
+    {emqx, {path, "../../apps/emqx"}}
+]}.
+
+{project_plugins, [erlfmt]}.

apps/emqx_connector_aggregator/src/emqx_connector_aggreg_app.erl

@@ -0,0 +1,25 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_connector_aggreg_app).
+
+-behaviour(application).
+-export([start/2, stop/1]).
+
+%%------------------------------------------------------------------------------
+%% Type declarations
+%%------------------------------------------------------------------------------
+
+%%------------------------------------------------------------------------------
+%% `application` API
+%%------------------------------------------------------------------------------
+
+start(_StartType, _StartArgs) ->
+    emqx_connector_aggreg_sup:start_link().
+
+stop(_State) ->
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Internal fns
+%%------------------------------------------------------------------------------

apps/emqx_connector_aggregator/src/emqx_connector_aggreg_buffer.erl

@@ -17,7 +17,7 @@
 %% ...
 %% ```
 %% ^ ETF = Erlang External Term Format (i.e. `erlang:term_to_binary/1`).
--module(emqx_bridge_s3_aggreg_buffer).
+-module(emqx_connector_aggreg_buffer).
 
 -export([
     new_writer/2,

apps/emqx_connector_aggregator/src/emqx_connector_aggreg_csv.erl

@@ -2,8 +2,8 @@
 %% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
-%% CSV container implementation for `emqx_bridge_s3_aggregator`.
+%% CSV container implementation for `emqx_connector_aggregator`.
--module(emqx_bridge_s3_aggreg_csv).
+-module(emqx_connector_aggreg_csv).
 
 %% Container API
 -export([
@@ -33,7 +33,7 @@
     column_order => [column()]
 }.
 
--type record() :: emqx_bridge_s3_aggregator:record().
+-type record() :: emqx_connector_aggregator:record().
 -type column() :: binary().
 
 %%

apps/emqx_connector_aggregator/src/emqx_connector_aggreg_delivery.erl

@@ -0,0 +1,195 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+%% This module takes aggregated records from a buffer and delivers them to a blob storage
+%% backend, wrapped in a configurable container (though currently there's only CSV).
+-module(emqx_connector_aggreg_delivery).
+
+-include_lib("snabbkaffe/include/trace.hrl").
+-include("emqx_connector_aggregator.hrl").
+
+-export([start_link/3]).
+
+%% Internal exports
+-export([
+    init/4,
+    loop/3
+]).
+
+%% Sys
+-export([
+    system_continue/3,
+    system_terminate/4,
+    format_status/2
+]).
+
+-record(delivery, {
+    name :: _Name,
+    callback_module :: module(),
+    container :: emqx_connector_aggreg_csv:container(),
+    reader :: emqx_connector_aggreg_buffer:reader(),
+    transfer :: transfer_state(),
+    empty :: boolean()
+}).
+
+-type state() :: #delivery{}.
+
+-type init_opts() :: #{
+    callback_module := module(),
+    any() => term()
+}.
+
+-type transfer_state() :: term().
+
+%% @doc Initialize the transfer state, such as blob storage path, transfer options, client
+%% credentials, etc. .
+-callback init_transfer_state(buffer_map(), map()) -> transfer_state().
+
+%% @doc Append data to the transfer before sending.  Usually should not fail.
+-callback process_append(iodata(), transfer_state()) -> transfer_state().
+
+%% @doc Push appended transfer data to its destination (e.g.: upload a part of a
+%% multi-part upload).  May fail.
+-callback process_write(transfer_state()) -> {ok, transfer_state()} | {error, term()}.
+
+%% @doc Finalize the transfer and clean up any resources.  May return a term summarizing
+%% the transfer.
+-callback process_complete(transfer_state()) -> {ok, term()}.
+
+%%
+
+start_link(Name, Buffer, Opts) ->
+    proc_lib:start_link(?MODULE, init, [self(), Name, Buffer, Opts]).
+
+%%
+
+-spec init(pid(), _Name, buffer(), init_opts()) -> no_return().
+init(Parent, Name, Buffer, Opts) ->
+    ?tp(connector_aggreg_delivery_started, #{action => Name, buffer => Buffer}),
+    Reader = open_buffer(Buffer),
+    Delivery = init_delivery(Name, Reader, Buffer, Opts#{action => Name}),
+    _ = erlang:process_flag(trap_exit, true),
+    ok = proc_lib:init_ack({ok, self()}),
+    loop(Delivery, Parent, []).
+
+init_delivery(
+    Name,
+    Reader,
+    Buffer,
+    Opts = #{
+        container := ContainerOpts,
+        callback_module := Mod
+    }
+) ->
+    BufferMap = emqx_connector_aggregator:buffer_to_map(Buffer),
+    #delivery{
+        name = Name,
+        callback_module = Mod,
+        container = mk_container(ContainerOpts),
+        reader = Reader,
+        transfer = Mod:init_transfer_state(BufferMap, Opts),
+        empty = true
+    }.
+
+open_buffer(#buffer{filename = Filename}) ->
+    case file:open(Filename, [read, binary, raw]) of
+        {ok, FD} ->
+            {_Meta, Reader} = emqx_connector_aggreg_buffer:new_reader(FD),
+            Reader;
+        {error, Reason} ->
+            error(#{reason => buffer_open_failed, file => Filename, posix => Reason})
+    end.
+
+mk_container(#{type := csv, column_order := OrderOpt}) ->
+    %% TODO: Deduplicate?
+    ColumnOrder = lists:map(fun emqx_utils_conv:bin/1, OrderOpt),
+    emqx_connector_aggreg_csv:new(#{column_order => ColumnOrder}).
+
+%%
+
+-spec loop(state(), pid(), [sys:debug_option()]) -> no_return().
+loop(Delivery, Parent, Debug) ->
+    %% NOTE: This function is mocked in tests.
+    receive
+        Msg -> handle_msg(Msg, Delivery, Parent, Debug)
+    after 0 ->
+        process_delivery(Delivery, Parent, Debug)
+    end.
+
+process_delivery(Delivery0 = #delivery{reader = Reader0}, Parent, Debug) ->
+    case emqx_connector_aggreg_buffer:read(Reader0) of
+        {Records = [#{} | _], Reader} ->
+            Delivery1 = Delivery0#delivery{reader = Reader},
+            Delivery2 = process_append_records(Records, Delivery1),
+            Delivery = process_write(Delivery2),
+            ?MODULE:loop(Delivery, Parent, Debug);
+        {[], Reader} ->
+            Delivery = Delivery0#delivery{reader = Reader},
+            ?MODULE:loop(Delivery, Parent, Debug);
+        eof ->
+            process_complete(Delivery0);
+        {Unexpected, _Reader} ->
+            exit({buffer_unexpected_record, Unexpected})
+    end.
+
+process_append_records(
+    Records,
+    Delivery = #delivery{
+        callback_module = Mod,
+        container = Container0,
+        transfer = Transfer0
+    }
+) ->
+    {Writes, Container} = emqx_connector_aggreg_csv:fill(Records, Container0),
+    Transfer = Mod:process_append(Writes, Transfer0),
+    Delivery#delivery{
+        container = Container,
+        transfer = Transfer,
+        empty = false
+    }.
+
+process_write(Delivery = #delivery{callback_module = Mod, transfer = Transfer0}) ->
+    case Mod:process_write(Transfer0) of
+        {ok, Transfer} ->
+            Delivery#delivery{transfer = Transfer};
+        {error, Reason} ->
+            %% Todo: handle more gracefully?  Retry?
+            error({transfer_failed, Reason})
+    end.
+
+process_complete(#delivery{name = Name, empty = true}) ->
+    ?tp(connector_aggreg_delivery_completed, #{action => Name, transfer => empty}),
+    exit({shutdown, {skipped, empty}});
+process_complete(#delivery{
+    name = Name, callback_module = Mod, container = Container, transfer = Transfer0
+}) ->
+    Trailer = emqx_connector_aggreg_csv:close(Container),
+    Transfer = Mod:process_append(Trailer, Transfer0),
+    {ok, Completed} = Mod:process_complete(Transfer),
+    ?tp(connector_aggreg_delivery_completed, #{action => Name, transfer => Completed}),
+    ok.
+
+%%
+
+handle_msg({system, From, Msg}, Delivery, Parent, Debug) ->
+    sys:handle_system_msg(Msg, From, Parent, ?MODULE, Debug, Delivery);
+handle_msg({'EXIT', Parent, Reason}, Delivery, Parent, Debug) ->
+    system_terminate(Reason, Parent, Debug, Delivery);
+handle_msg(_Msg, Delivery, Parent, Debug) ->
+    ?MODULE:loop(Parent, Debug, Delivery).
+
+-spec system_continue(pid(), [sys:debug_option()], state()) -> no_return().
+system_continue(Parent, Debug, Delivery) ->
+    ?MODULE:loop(Delivery, Parent, Debug).
+
+-spec system_terminate(_Reason, pid(), [sys:debug_option()], state()) -> _.
+system_terminate(_Reason, _Parent, _Debug, #delivery{callback_module = Mod, transfer = Transfer}) ->
+    Mod:process_terminate(Transfer).
+
+-spec format_status(normal, Args :: [term()]) -> _StateFormatted.
+format_status(_Normal, [_PDict, _SysState, _Parent, _Debug, Delivery]) ->
+    #delivery{callback_module = Mod} = Delivery,
+    Delivery#delivery{
+        transfer = Mod:process_format_status(Delivery#delivery.transfer)
+    }.

apps/emqx_connector_aggregator/src/emqx_connector_aggreg_sup.erl

@@ -2,7 +2,7 @@
 %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_bridge_s3_sup).
+-module(emqx_connector_aggreg_sup).
 
 -export([
     start_link/0,

apps/emqx_connector_aggregator/src/emqx_connector_aggreg_upload_sup.erl

@@ -2,7 +2,7 @@
 %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_bridge_s3_aggreg_upload_sup).
+-module(emqx_connector_aggreg_upload_sup).
 
 -export([
     start_link/3,
@@ -33,7 +33,7 @@ start_delivery(Name, Buffer) ->
     supervisor:start_child(?SUPREF(Name), [Buffer]).
 
 start_delivery_proc(Name, DeliveryOpts, Buffer) ->
-    emqx_bridge_s3_aggreg_delivery:start_link(Name, Buffer, DeliveryOpts).
+    emqx_connector_aggreg_delivery:start_link(Name, Buffer, DeliveryOpts).
 
 %%
 
@@ -45,7 +45,7 @@ init({root, Name, AggregOpts, DeliveryOpts}) ->
     },
     AggregatorChildSpec = #{
         id => aggregator,
-        start => {emqx_bridge_s3_aggregator, start_link, [Name, AggregOpts]},
+        start => {emqx_connector_aggregator, start_link, [Name, AggregOpts]},
         type => worker,
         restart => permanent
     },

apps/emqx_connector_aggregator/src/emqx_connector_aggregator.app.src

@@ -0,0 +1,13 @@
+{application, emqx_connector_aggregator, [
+    {description, "EMQX Enterprise Connector Data Aggregator"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib
+    ]},
+    {env, []},
+    {mod, {emqx_connector_aggreg_app, []}},
+    {modules, []},
+    {links, []}
+]}.

apps/emqx_connector_aggregator/src/emqx_connector_aggregator.erl

@@ -5,18 +5,19 @@
 %% This module manages buffers for aggregating records and offloads them
 %% to separate "delivery" processes when they are full or time interval
 %% is over.
--module(emqx_bridge_s3_aggregator).
+-module(emqx_connector_aggregator).
 
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/trace.hrl").
 
--include("emqx_bridge_s3_aggregator.hrl").
+-include("emqx_connector_aggregator.hrl").
 
 -export([
     start_link/2,
     push_records/3,
     tick/2,
-    take_error/1
+    take_error/1,
+    buffer_to_map/1
 ]).
 
 -behaviour(gen_server).
@@ -72,6 +73,15 @@ tick(Name, Timestamp) ->
 take_error(Name) ->
     gen_server:call(?SRVREF(Name), take_error).
 
+buffer_to_map(#buffer{} = Buffer) ->
+    #{
+        since => Buffer#buffer.since,
+        until => Buffer#buffer.until,
+        seq => Buffer#buffer.seq,
+        filename => Buffer#buffer.filename,
+        max_records => Buffer#buffer.max_records
+    }.
+
 %%
 
 write_records_limited(Name, Buffer = #buffer{max_records = undefined}, Records) ->
@@ -90,9 +100,9 @@ write_records_limited(Name, Buffer = #buffer{max_records = MaxRecords}, Records)
     end.
 
 write_records(Name, Buffer = #buffer{fd = Writer}, Records) ->
-    case emqx_bridge_s3_aggreg_buffer:write(Records, Writer) of
+    case emqx_connector_aggreg_buffer:write(Records, Writer) of
         ok ->
-            ?tp(s3_aggreg_records_written, #{action => Name, records => Records}),
+            ?tp(connector_aggreg_records_written, #{action => Name, records => Records}),
             ok;
         {error, terminated} ->
             BufferNext = rotate_buffer(Name, Buffer),
@@ -250,9 +260,9 @@ compute_since(Timestamp, PrevSince, Interval) ->
 allocate_buffer(Since, Seq, St = #st{name = Name}) ->
     Buffer = #buffer{filename = Filename, cnt_records = Counter} = mk_buffer(Since, Seq, St),
     {ok, FD} = file:open(Filename, [write, binary]),
-    Writer = emqx_bridge_s3_aggreg_buffer:new_writer(FD, _Meta = []),
+    Writer = emqx_connector_aggreg_buffer:new_writer(FD, _Meta = []),
     _ = add_counter(Counter),
-    ?tp(s3_aggreg_buffer_allocated, #{action => Name, filename => Filename}),
+    ?tp(connector_aggreg_buffer_allocated, #{action => Name, filename => Filename}),
     Buffer#buffer{fd = Writer}.
 
 recover_buffer(Buffer = #buffer{filename = Filename, cnt_records = Counter}) ->
@@ -274,7 +284,7 @@ recover_buffer(Buffer = #buffer{filename = Filename, cnt_records = Counter}) ->
     end.
 
 recover_buffer_writer(FD, Filename) ->
-    try emqx_bridge_s3_aggreg_buffer:new_reader(FD) of
+    try emqx_connector_aggreg_buffer:new_reader(FD) of
         {_Meta, Reader} -> recover_buffer_writer(FD, Filename, Reader, 0)
     catch
         error:Reason ->
@@ -282,7 +292,7 @@ recover_buffer_writer(FD, Filename) ->
     end.
 
 recover_buffer_writer(FD, Filename, Reader0, NWritten) ->
-    try emqx_bridge_s3_aggreg_buffer:read(Reader0) of
+    try emqx_connector_aggreg_buffer:read(Reader0) of
         {Records, Reader} when is_list(Records) ->
             recover_buffer_writer(FD, Filename, Reader, NWritten + length(Records));
         {Unexpected, _Reader} ->
@@ -303,7 +313,7 @@ recover_buffer_writer(FD, Filename, Reader0, NWritten) ->
                     "Buffer is truncated or corrupted somewhere in the middle. "
                     "Corrupted records will be discarded."
             }),
-            Writer = emqx_bridge_s3_aggreg_buffer:takeover(Reader0),
+            Writer = emqx_connector_aggreg_buffer:takeover(Reader0),
             {ok, Writer, NWritten}
     end.
 
@@ -362,7 +372,7 @@ lookup_current_buffer(Name) ->
 %%
 
 enqueue_delivery(Buffer, St = #st{name = Name, deliveries = Ds}) ->
-    {ok, Pid} = emqx_bridge_s3_aggreg_upload_sup:start_delivery(Name, Buffer),
+    {ok, Pid} = emqx_connector_aggreg_upload_sup:start_delivery(Name, Buffer),
     MRef = erlang:monitor(process, Pid),
     St#st{deliveries = Ds#{MRef => Buffer}}.
 

apps/emqx_connector_aggregator/test/emqx_connector_aggreg_buffer_SUITE.erl

@@ -2,7 +2,7 @@
 %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_bridge_s3_aggreg_buffer_SUITE).
+-module(emqx_connector_aggreg_buffer_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
@@ -29,7 +29,7 @@ t_write_read_cycle(Config) ->
     Filename = mk_filename(?FUNCTION_NAME, Config),
     Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}},
     {ok, WFD} = file:open(Filename, [write, binary]),
-    Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata),
+    Writer = emqx_connector_aggreg_buffer:new_writer(WFD, Metadata),
     Terms = [
         [],
         [[[[[[[[]]]]]]]],
@@ -43,12 +43,12 @@ t_write_read_cycle(Config) ->
         {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})}
     ],
     ok = lists:foreach(
-        fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end,
+        fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer)) end,
         Terms
     ),
     ok = file:close(WFD),
     {ok, RFD} = file:open(Filename, [read, binary, raw]),
-    {MetadataRead, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD),
+    {MetadataRead, Reader} = emqx_connector_aggreg_buffer:new_reader(RFD),
     ?assertEqual(Metadata, MetadataRead),
     TermsRead = read_until_eof(Reader),
     ?assertEqual(Terms, TermsRead).
@@ -60,7 +60,7 @@ t_read_empty(Config) ->
     {ok, RFD} = file:open(Filename, [read, binary]),
     ?assertError(
         {buffer_incomplete, header},
-        emqx_bridge_s3_aggreg_buffer:new_reader(RFD)
+        emqx_connector_aggreg_buffer:new_reader(RFD)
     ).
 
 t_read_garbage(Config) ->
@@ -71,14 +71,14 @@ t_read_garbage(Config) ->
     {ok, RFD} = file:open(Filename, [read, binary]),
     ?assertError(
         badarg,
-        emqx_bridge_s3_aggreg_buffer:new_reader(RFD)
+        emqx_connector_aggreg_buffer:new_reader(RFD)
     ).
 
 t_read_truncated(Config) ->
     Filename = mk_filename(?FUNCTION_NAME, Config),
     {ok, WFD} = file:open(Filename, [write, binary]),
     Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}},
-    Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata),
+    Writer = emqx_connector_aggreg_buffer:new_writer(WFD, Metadata),
     Terms = [
         [[[[[[[[[[[]]]]]]]]]]],
         lists:seq(1, 100000),
@@ -88,36 +88,36 @@ t_read_truncated(Config) ->
     LastTerm =
         {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})},
     ok = lists:foreach(
-        fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end,
+        fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer)) end,
         Terms
     ),
     {ok, WPos} = file:position(WFD, cur),
-    ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(LastTerm, Writer)),
+    ?assertEqual(ok, emqx_connector_aggreg_buffer:write(LastTerm, Writer)),
     ok = file:close(WFD),
-    ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos + 1),
+    ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, WPos + 1),
     {ok, RFD1} = file:open(Filename, [read, binary]),
-    {Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD1),
+    {Metadata, Reader0} = emqx_connector_aggreg_buffer:new_reader(RFD1),
     {ReadTerms1, Reader1} = read_terms(length(Terms), Reader0),
     ?assertEqual(Terms, ReadTerms1),
     ?assertError(
         badarg,
-        emqx_bridge_s3_aggreg_buffer:read(Reader1)
+        emqx_connector_aggreg_buffer:read(Reader1)
     ),
-    ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2),
+    ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, WPos div 2),
     {ok, RFD2} = file:open(Filename, [read, binary]),
-    {Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD2),
+    {Metadata, Reader2} = emqx_connector_aggreg_buffer:new_reader(RFD2),
     {ReadTerms2, Reader3} = read_terms(_FitsInto = 3, Reader2),
     ?assertEqual(lists:sublist(Terms, 3), ReadTerms2),
     ?assertError(
         badarg,
-        emqx_bridge_s3_aggreg_buffer:read(Reader3)
+        emqx_connector_aggreg_buffer:read(Reader3)
     ).
 
 t_read_truncated_takeover_write(Config) ->
     Filename = mk_filename(?FUNCTION_NAME, Config),
     {ok, WFD} = file:open(Filename, [write, binary]),
     Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}},
-    Writer1 = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata),
+    Writer1 = emqx_connector_aggreg_buffer:new_writer(WFD, Metadata),
     Terms1 = [
         [[[[[[[[[[[]]]]]]]]]]],
         lists:seq(1, 10000),
@@ -129,14 +129,14 @@ t_read_truncated_takeover_write(Config) ->
         {<<"application/x-octet-stream">>, rand:bytes(102400)}
     ],
     ok = lists:foreach(
-        fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer1)) end,
+        fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer1)) end,
         Terms1
     ),
     {ok, WPos} = file:position(WFD, cur),
     ok = file:close(WFD),
-    ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2),
+    ok = emqx_connector_aggregator_test_helpers:truncate_at(Filename, WPos div 2),
     {ok, RWFD} = file:open(Filename, [read, write, binary]),
-    {Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RWFD),
+    {Metadata, Reader0} = emqx_connector_aggreg_buffer:new_reader(RWFD),
     {ReadTerms1, Reader1} = read_terms(_Survived = 3, Reader0),
     ?assertEqual(
         lists:sublist(Terms1, 3),
@@ -144,16 +144,16 @@ t_read_truncated_takeover_write(Config) ->
     ),
     ?assertError(
         badarg,
-        emqx_bridge_s3_aggreg_buffer:read(Reader1)
+        emqx_connector_aggreg_buffer:read(Reader1)
     ),
-    Writer2 = emqx_bridge_s3_aggreg_buffer:takeover(Reader1),
+    Writer2 = emqx_connector_aggreg_buffer:takeover(Reader1),
     ok = lists:foreach(
-        fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer2)) end,
+        fun(T) -> ?assertEqual(ok, emqx_connector_aggreg_buffer:write(T, Writer2)) end,
         Terms2
     ),
     ok = file:close(RWFD),
     {ok, RFD} = file:open(Filename, [read, binary]),
-    {Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD),
+    {Metadata, Reader2} = emqx_connector_aggreg_buffer:new_reader(RFD),
     ReadTerms2 = read_until_eof(Reader2),
     ?assertEqual(
         lists:sublist(Terms1, 3) ++ Terms2,
@@ -168,12 +168,12 @@ mk_filename(Name, Config) ->
 read_terms(0, Reader) ->
     {[], Reader};
 read_terms(N, Reader0) ->
-    {Term, Reader1} = emqx_bridge_s3_aggreg_buffer:read(Reader0),
+    {Term, Reader1} = emqx_connector_aggreg_buffer:read(Reader0),
     {Terms, Reader} = read_terms(N - 1, Reader1),
     {[Term | Terms], Reader}.
 
 read_until_eof(Reader0) ->
-    case emqx_bridge_s3_aggreg_buffer:read(Reader0) of
+    case emqx_connector_aggreg_buffer:read(Reader0) of
         {Term, Reader} ->
             [Term | read_until_eof(Reader)];
         eof ->

apps/emqx_connector_aggregator/test/emqx_connector_aggreg_csv_tests.erl

@@ -2,12 +2,12 @@
 %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_bridge_s3_aggreg_csv_tests).
+-module(emqx_connector_aggreg_csv_tests).
 
 -include_lib("eunit/include/eunit.hrl").
 
 encoding_test() ->
-    CSV = emqx_bridge_s3_aggreg_csv:new(#{}),
+    CSV = emqx_connector_aggreg_csv:new(#{}),
     ?assertEqual(
         "A,B,Ç\n"
         "1.2345,string,0.0\n"
@@ -28,7 +28,7 @@ encoding_test() ->
 
 column_order_test() ->
     Order = [<<"ID">>, <<"TS">>],
-    CSV = emqx_bridge_s3_aggreg_csv:new(#{column_order => Order}),
+    CSV = emqx_connector_aggreg_csv:new(#{column_order => Order}),
     ?assertEqual(
         "ID,TS,A,B,D\n"
         "1,2024-01-01,12.34,str,\"[]\"\n"
@@ -63,10 +63,10 @@ fill_close(CSV, LRecords) ->
     string(fill_close_(CSV, LRecords)).
 
 fill_close_(CSV0, [Records | LRest]) ->
-    {Writes, CSV} = emqx_bridge_s3_aggreg_csv:fill(Records, CSV0),
+    {Writes, CSV} = emqx_connector_aggreg_csv:fill(Records, CSV0),
     [Writes | fill_close_(CSV, LRest)];
 fill_close_(CSV, []) ->
-    [emqx_bridge_s3_aggreg_csv:close(CSV)].
+    [emqx_connector_aggreg_csv:close(CSV)].
 
 string(Writes) ->
     unicode:characters_to_list(Writes).

apps/emqx_connector_aggregator/test/emqx_connector_aggregator_test_helpers.erl

@@ -0,0 +1,25 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_connector_aggregator_test_helpers).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+%% API
+-export([]).
+
+%%------------------------------------------------------------------------------
+%% File utilities
+%%------------------------------------------------------------------------------
+
+truncate_at(Filename, Pos) ->
+    {ok, FD} = file:open(Filename, [read, write, binary]),
+    {ok, Pos} = file:position(FD, Pos),
+    ok = file:truncate(FD),
+    ok = file:close(FD).
+
+%%------------------------------------------------------------------------------
+%% Internal fns
+%%------------------------------------------------------------------------------

apps/emqx_dashboard/src/emqx_dashboard_api.erl

@@ -311,7 +311,7 @@ user(delete, #{bindings := #{username := Username0}, headers := Headers} = Req)
     end.
 
 is_self_auth(?SSO_USERNAME(_, _), _) ->
-    fasle;
+    false;
 is_self_auth(Username, #{<<"authorization">> := Token}) ->
     is_self_auth(Username, Token);
 is_self_auth(Username, #{<<"Authorization">> := Token}) ->

apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl

@@ -20,11 +20,13 @@
 -compile(export_all).
 
 -import(emqx_dashboard_SUITE, [auth_header_/0]).
+-import(emqx_common_test_helpers, [on_exit/1]).
 
 -include("emqx_dashboard.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 -define(SERVER, "http://127.0.0.1:18083").
 -define(BASE_PATH, "/api/v5").
@@ -52,10 +54,47 @@
 %%--------------------------------------------------------------------
 
 all() ->
-    emqx_common_test_helpers:all(?MODULE).
+    [
+        {group, common},
+        {group, persistent_sessions}
+    ].
+
+groups() ->
+    AllTCs = emqx_common_test_helpers:all(?MODULE),
+    PSTCs = persistent_session_testcases(),
+    [
+        {common, [], AllTCs -- PSTCs},
+        {persistent_sessions, [], PSTCs}
+    ].
+
+persistent_session_testcases() ->
+    [
+        t_persistent_session_stats
+    ].
 
 init_per_suite(Config) ->
-    emqx_common_test_helpers:clear_screen(),
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_group(persistent_sessions = Group, Config) ->
+    Apps = emqx_cth_suite:start(
+        [
+            emqx_conf,
+            {emqx, "session_persistence {enable = true}"},
+            {emqx_retainer, ?BASE_RETAINER_CONF},
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard(
+                "dashboard.listeners.http { enable = true, bind = 18083 }\n"
+                "dashboard.sample_interval = 1s"
+            )
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Group, Config)}
+    ),
+    {ok, _} = emqx_common_test_http:create_default_app(),
+    [{apps, Apps} | Config];
+init_per_group(common = Group, Config) ->
     Apps = emqx_cth_suite:start(
         [
             emqx,
@@ -67,12 +106,12 @@ init_per_suite(Config) ->
                 "dashboard.sample_interval = 1s"
             )
         ],
-        #{work_dir => emqx_cth_suite:work_dir(Config)}
+        #{work_dir => emqx_cth_suite:work_dir(Group, Config)}
     ),
     {ok, _} = emqx_common_test_http:create_default_app(),
     [{apps, Apps} | Config].
 
-end_per_suite(Config) ->
+end_per_group(_Group, Config) ->
     Apps = ?config(apps, Config),
     emqx_cth_suite:stop(Apps),
     ok.
@@ -84,6 +123,7 @@ init_per_testcase(_TestCase, Config) ->
 
 end_per_testcase(_TestCase, _Config) ->
     ok = snabbkaffe:stop(),
+    emqx_common_test_helpers:call_janitor(),
     ok.
 
 %%--------------------------------------------------------------------
@@ -272,6 +312,51 @@ t_monitor_api_error(_) ->
         request(["monitor"], "latest=-1"),
     ok.
 
+%% Verifies that subscriptions from persistent sessions are correctly accounted for.
+t_persistent_session_stats(_Config) ->
+    %% pre-condition
+    true = emqx_persistent_message:is_persistence_enabled(),
+
+    NonPSClient = start_and_connect(#{
+        clientid => <<"non-ps">>,
+        expiry_interval => 0
+    }),
+    PSClient = start_and_connect(#{
+        clientid => <<"ps">>,
+        expiry_interval => 30
+    }),
+    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"non/ps/topic/+">>, 2),
+    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"non/ps/topic">>, 2),
+    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"common/topic/+">>, 2),
+    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(NonPSClient, <<"common/topic">>, 2),
+    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"ps/topic/+">>, 2),
+    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"ps/topic">>, 2),
+    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"common/topic/+">>, 2),
+    {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(PSClient, <<"common/topic">>, 2),
+    {ok, _} =
+        snabbkaffe:block_until(
+            ?match_n_events(2, #{?snk_kind := dashboard_monitor_flushed}),
+            infinity
+        ),
+    ?retry(1_000, 10, begin
+        ?assertMatch(
+            {ok, #{
+                %% N.B.: we currently don't perform any deduplication between persistent
+                %% and non-persistent routes, so we count `commont/topic' twice and get 8
+                %% instead of 6 here.
+                <<"topics">> := 8,
+                <<"subscriptions">> := 8
+            }},
+            request(["monitor_current"])
+        )
+    end),
+    %% Sanity checks
+    PSRouteCount = emqx_persistent_session_ds_router:stats(n_routes),
+    ?assert(PSRouteCount > 0, #{ps_route_count => PSRouteCount}),
+    PSSubCount = emqx_persistent_session_bookkeeper:get_subscription_count(),
+    ?assert(PSSubCount > 0, #{ps_sub_count => PSSubCount}),
+    ok.
+
 request(Path) ->
     request(Path, "").
 
@@ -340,3 +425,22 @@ waiting_emqx_stats_and_monitor_update(WaitKey) ->
     %% manually call monitor update
     _ = emqx_dashboard_monitor:current_rate_cluster(),
     ok.
+
+start_and_connect(Opts) ->
+    Defaults = #{clean_start => false, expiry_interval => 30},
+    #{
+        clientid := ClientId,
+        clean_start := CleanStart,
+        expiry_interval := EI
+    } = maps:merge(Defaults, Opts),
+    {ok, Client} = emqtt:start_link([
+        {clientid, ClientId},
+        {clean_start, CleanStart},
+        {proto_ver, v5},
+        {properties, #{'Session-Expiry-Interval' => EI}}
+    ]),
+    on_exit(fun() ->
+        catch emqtt:disconnect(Client, ?RC_NORMAL_DISCONNECTION, #{'Session-Expiry-Interval' => 0})
+    end),
+    {ok, _} = emqtt:connect(Client),
+    Client.

apps/emqx_durable_storage/src/emqx_ds_replication_layer_shard.erl

@@ -82,6 +82,8 @@ server_name(DB, Shard, Site) ->
 
 %%
 
+-spec servers(emqx_ds:db(), emqx_ds_replication_layer:shard_id(), Order) -> [server(), ...] when
+    Order :: leader_preferred | undefined.
 servers(DB, Shard, _Order = leader_preferred) ->
     get_servers_leader_preferred(DB, Shard);
 servers(DB, Shard, _Order = undefined) ->
@@ -98,7 +100,7 @@ get_servers_leader_preferred(DB, Shard) ->
             Servers = ra_leaderboard:lookup_members(ClusterName),
             [Leader | lists:delete(Leader, Servers)];
         undefined ->
-            get_shard_servers(DB, Shard)
+            get_online_servers(DB, Shard)
     end.
 
 get_server_local_preferred(DB, Shard) ->
@@ -111,7 +113,7 @@ get_server_local_preferred(DB, Shard) ->
             %% TODO
             %% Leader is unkonwn if there are no servers of this group on the
             %% local node. We want to pick a replica in that case as well.
-            pick_random(get_shard_servers(DB, Shard))
+            pick_random(get_online_servers(DB, Shard))
     end.
 
 lookup_leader(DB, Shard) ->
@@ -121,6 +123,21 @@ lookup_leader(DB, Shard) ->
     ClusterName = get_cluster_name(DB, Shard),
     ra_leaderboard:lookup_leader(ClusterName).
 
+get_online_servers(DB, Shard) ->
+    filter_online(get_shard_servers(DB, Shard)).
+
+filter_online(Servers) ->
+    case lists:filter(fun is_server_online/1, Servers) of
+        [] ->
+            %% NOTE: Must return non-empty list.
+            Servers;
+        Online ->
+            Online
+    end.
+
+is_server_online({_Name, Node}) ->
+    Node == node() orelse lists:member(Node, nodes()).
+
 pick_local(Servers) ->
     case lists:keyfind(node(), 2, Servers) of
         Local when is_tuple(Local) ->

apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl

@@ -518,7 +518,7 @@ handle_msg({inet_reply, _Sock, ok}, State = #state{active_n = ActiveN}) ->
 handle_msg({inet_reply, _Sock, {error, Reason}}, State) ->
     handle_info({sock_error, Reason}, State);
 handle_msg({close, Reason}, State) ->
-    ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}),
+    ?tp(debug, force_socket_close, #{reason => Reason}),
     handle_info({sock_closed, Reason}, close_socket(State));
 handle_msg(
     {event, connected},

apps/emqx_gateway/src/emqx_gateway_ctx.erl

@@ -39,6 +39,7 @@
 %% Authentication circle
 -export([
     authenticate/2,
+    connection_expire_interval/2,
     open_session/5,
     open_session/6,
     insert_channel_info/4,
@@ -78,6 +79,13 @@ authenticate(_Ctx, ClientInfo0) ->
             {error, Reason}
     end.
 
+-spec connection_expire_interval(context(), emqx_types:clientinfo()) ->
+    undefined | non_neg_integer().
+connection_expire_interval(_Ctx, #{auth_expire_at := undefined}) ->
+    undefined;
+connection_expire_interval(_Ctx, #{auth_expire_at := ExpireAt}) ->
+    max(0, ExpireAt - erlang:system_time(millisecond)).
+
 %% @doc Register the session to the cluster.
 %%
 %%  This function should be called after the client has authenticated
@@ -157,6 +165,9 @@ set_chan_stats(_Ctx = #{gwname := GwName}, ClientId, Stats) ->
 connection_closed(_Ctx = #{gwname := GwName}, ClientId) ->
     emqx_gateway_cm:connection_closed(GwName, ClientId).
 
+%%--------------------------------------------------------------------
+%% Message circle
+
 -spec authorize(
     context(),
     emqx_types:clientinfo(),
@@ -167,6 +178,9 @@ connection_closed(_Ctx = #{gwname := GwName}, ClientId) ->
 authorize(_Ctx, ClientInfo, Action, Topic) ->
     emqx_access_control:authorize(ClientInfo, Action, Topic).
 
+%%--------------------------------------------------------------------
+%% Metrics & Stats
+
 metrics_inc(_Ctx = #{gwname := GwName}, Name) ->
     emqx_gateway_metrics:inc(GwName, Name).
 
@@ -183,6 +197,8 @@ eval_mountpoint(ClientInfo = #{mountpoint := MountPoint}) ->
     MountPoint1 = emqx_mountpoint:replvar(MountPoint, ClientInfo),
     ClientInfo#{mountpoint := MountPoint1}.
 
-merge_auth_result(ClientInfo, AuthResult) when is_map(ClientInfo) andalso is_map(AuthResult) ->
+merge_auth_result(ClientInfo, AuthResult0) when is_map(ClientInfo) andalso is_map(AuthResult0) ->
-    IsSuperuser = maps:get(is_superuser, AuthResult, false),
+    IsSuperuser = maps:get(is_superuser, AuthResult0, false),
-    maps:merge(ClientInfo, AuthResult#{is_superuser => IsSuperuser}).
+    ExpireAt = maps:get(expire_at, AuthResult0, undefined),
+    AuthResult1 = maps:without([expire_at], AuthResult0),
+    maps:merge(ClientInfo#{auth_expire_at => ExpireAt}, AuthResult1#{is_superuser => IsSuperuser}).

apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl

@@ -82,4 +82,4 @@ t_authenticate(_) ->
     ?assertMatch({ok, #{is_superuser := true}}, emqx_gateway_ctx:authenticate(Ctx, Info4)),
     ok.
 
-default_result(Info) -> Info#{zone => default, is_superuser => false}.
+default_result(Info) -> Info#{zone => default, is_superuser => false, auth_expire_at => undefined}.

apps/emqx_gateway_coap/src/emqx_coap_channel.erl

@@ -214,6 +214,8 @@ handle_timeout(_, {transport, Msg}, Channel) ->
     call_session(timeout, Msg, Channel);
 handle_timeout(_, disconnect, Channel) ->
     {shutdown, normal, Channel};
+handle_timeout(_, connection_expire, Channel) ->
+    {shutdown, expired, Channel};
 handle_timeout(_, _, Channel) ->
     {ok, Channel}.
 
@@ -595,6 +597,14 @@ process_connect(
             iter(Iter, reply({error, bad_request}, Msg, Result), Channel)
     end.
 
+schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
+    case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
+        undefined ->
+            Channel;
+        Interval ->
+            ensure_timer(connection_expire_timer, Interval, connection_expire, Channel)
+    end.
+
 run_hooks(Ctx, Name, Args) ->
     emqx_gateway_ctx:metrics_inc(Ctx, Name),
     emqx_hooks:run(Name, Args).
@@ -619,7 +629,7 @@ ensure_connected(
     NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
     _ = run_hooks(Ctx, 'client.connack', [NConnInfo, connection_accepted, #{}]),
     ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
-    Channel#channel{conninfo = NConnInfo, conn_state = connected}.
+    schedule_connection_expire(Channel#channel{conninfo = NConnInfo, conn_state = connected}).
 
 %%--------------------------------------------------------------------
 %% Ensure disconnected

apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway_coap, [
     {description, "CoAP Gateway"},
-    {vsn, "0.1.7"},
+    {vsn, "0.1.8"},
     {registered, []},
     {applications, [kernel, stdlib, emqx, emqx_gateway]},
     {env, []},

apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl

@@ -29,6 +29,7 @@
 
 -include_lib("er_coap_client/include/coap.hrl").
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/asserts.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
@@ -83,6 +84,17 @@ init_per_testcase(t_connection_with_authn_failed, Config) ->
         fun(_) -> {error, bad_username_or_password} end
     ),
     Config;
+init_per_testcase(t_connection_with_expire, Config) ->
+    ok = meck:new(emqx_access_control, [passthrough, no_history]),
+    ok = meck:expect(
+        emqx_access_control,
+        authenticate,
+        fun(_) ->
+            {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 100}}
+        end
+    ),
+    snabbkaffe:start_trace(),
+    Config;
 init_per_testcase(t_heartbeat, Config) ->
     NewHeartbeat = 800,
     OldConf = emqx:get_raw_config([gateway, coap]),
@@ -103,6 +115,10 @@ end_per_testcase(t_heartbeat, Config) ->
     OldConf = ?config(old_conf, Config),
     {ok, _} = emqx_gateway_conf:update_gateway(coap, OldConf),
     ok;
+end_per_testcase(t_connection_with_expire, Config) ->
+    snabbkaffe:stop(),
+    meck:unload(emqx_access_control),
+    Config;
 end_per_testcase(_, Config) ->
     ok = meck:unload(emqx_access_control),
     Config.
@@ -270,6 +286,26 @@ t_connection_with_authn_failed(_) ->
     ),
     ok.
 
+t_connection_with_expire(_) ->
+    ChId = {{127, 0, 0, 1}, 5683},
+    {ok, Sock} = er_coap_udp_socket:start_link(),
+    {ok, Channel} = er_coap_udp_socket:get_channel(Sock, ChId),
+
+    URI = ?MQTT_PREFIX ++ "/connection?clientid=client1",
+
+    ?assertWaitEvent(
+        begin
+            Req = make_req(post),
+            {ok, created, _Data} = do_request(Channel, URI, Req)
+        end,
+        #{
+            ?snk_kind := conn_process_terminated,
+            clientid := <<"client1">>,
+            reason := {shutdown, expired}
+        },
+        5000
+    ).
+
 t_publish(_) ->
     %% can publish to a normal topic
     Topics = [

apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl

@@ -302,6 +302,9 @@ handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) -
     {shutdown, Reason, Channel};
 handle_timeout(_TRef, force_close_idle, Channel) ->
     {shutdown, idle_timeout, Channel};
+handle_timeout(_TRef, connection_expire, Channel) ->
+    NChannel = remove_timer_ref(connection_expire, Channel),
+    {ok, [{event, disconnected}, {close, expired}], NChannel};
 handle_timeout(_TRef, Msg, Channel) ->
     ?SLOG(warning, #{
         msg => "unexpected_timeout_signal",
@@ -666,10 +669,18 @@ ensure_connected(
 ) ->
     NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
     ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
-    Channel#channel{
+    schedule_connection_expire(Channel#channel{
         conninfo = NConnInfo,
         conn_state = connected
-    }.
+    }).
+
+schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
+    case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
+        undefined ->
+            Channel;
+        Interval ->
+            ensure_timer(connection_expire, Interval, Channel)
+    end.
 
 ensure_disconnected(
     Reason,

apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway_exproto, [
     {description, "ExProto Gateway"},
-    {vsn, "0.1.9"},
+    {vsn, "0.1.10"},
     {registered, []},
     {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]},
     {env, []},

apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl

@@ -21,6 +21,7 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/asserts.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
@@ -81,6 +82,7 @@ groups() ->
         t_raw_publish,
         t_auth_deny,
         t_acl_deny,
+        t_auth_expire,
         t_hook_connected_disconnected,
         t_hook_session_subscribed_unsubscribed,
         t_hook_message_delivered
@@ -157,14 +159,17 @@ end_per_group(_, Cfg) ->
 init_per_testcase(TestCase, Cfg) when
     TestCase == t_enter_passive_mode
 ->
+    snabbkaffe:start_trace(),
     case proplists:get_value(listener_type, Cfg) of
         udp -> {skip, ignore};
         _ -> Cfg
     end;
 init_per_testcase(_TestCase, Cfg) ->
+    snabbkaffe:start_trace(),
     Cfg.
 
 end_per_testcase(_TestCase, _Cfg) ->
+    snabbkaffe:stop(),
     ok.
 
 listener_confs(Type) ->
@@ -290,6 +295,42 @@ t_auth_deny(Cfg) ->
         end,
     meck:unload([emqx_gateway_ctx]).
 
+t_auth_expire(Cfg) ->
+    SockType = proplists:get_value(listener_type, Cfg),
+    Sock = open(SockType),
+
+    Client = #{
+        proto_name => <<"demo">>,
+        proto_ver => <<"v0.1">>,
+        clientid => <<"test_client_1">>
+    },
+    Password = <<"123456">>,
+
+    ok = meck:new(emqx_access_control, [passthrough, no_history]),
+    ok = meck:expect(
+        emqx_access_control,
+        authenticate,
+        fun(_) ->
+            {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
+        end
+    ),
+
+    ConnBin = frame_connect(Client, Password),
+    ConnAckBin = frame_connack(0),
+
+    ?assertWaitEvent(
+        begin
+            send(Sock, ConnBin),
+            {ok, ConnAckBin} = recv(Sock, 5000)
+        end,
+        #{
+            ?snk_kind := conn_process_terminated,
+            clientid := <<"test_client_1">>,
+            reason := {shutdown, expired}
+        },
+        5000
+    ).
+
 t_acl_deny(Cfg) ->
     SockType = proplists:get_value(listener_type, Cfg),
     Sock = open(SockType),
@@ -332,7 +373,6 @@ t_acl_deny(Cfg) ->
     close(Sock).
 
 t_keepalive_timeout(Cfg) ->
-    ok = snabbkaffe:start_trace(),
     SockType = proplists:get_value(listener_type, Cfg),
     Sock = open(SockType),
 
@@ -383,8 +423,7 @@ t_keepalive_timeout(Cfg) ->
             ?assertEqual(1, length(?of_kind(conn_process_terminated, Trace))),
             %% socket port should be closed
             ?assertEqual({error, closed}, recv(Sock, 5000))
-    end,
+    end.
-    snabbkaffe:stop().
 
 t_hook_connected_disconnected(Cfg) ->
     SockType = proplists:get_value(listener_type, Cfg),
@@ -513,7 +552,6 @@ t_hook_message_delivered(Cfg) ->
     emqx_hooks:del('message.delivered', {?MODULE, hook_fun5}).
 
 t_idle_timeout(Cfg) ->
-    ok = snabbkaffe:start_trace(),
     SockType = proplists:get_value(listener_type, Cfg),
     Sock = open(SockType),
 
@@ -551,8 +589,7 @@ t_idle_timeout(Cfg) ->
                 {ok, #{reason := {shutdown, idle_timeout}}},
                 ?block_until(#{?snk_kind := conn_process_terminated}, 10000)
             )
-    end,
+    end.
-    snabbkaffe:stop().
 
 %%--------------------------------------------------------------------
 %% Utils

apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway_gbt32960, [
     {description, "GBT32960 Gateway"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {applications, [kernel, stdlib, emqx, emqx_gateway]},
     {env, []},

apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl

@@ -72,7 +72,8 @@
 
 -define(TIMER_TABLE, #{
     alive_timer => keepalive,
-    retry_timer => retry_delivery
+    retry_timer => retry_delivery,
+    connection_expire_timer => connection_expire
 }).
 
 -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]).
@@ -468,6 +469,13 @@ handle_timeout(
             {Outgoings2, NChannel} = dispatch_frame(Channel#channel{inflight = NInflight}),
             {ok, [{outgoing, Outgoings ++ Outgoings2}], reset_timer(retry_timer, NChannel)}
     end;
+handle_timeout(
+    _TRef,
+    connection_expire,
+    Channel
+) ->
+    NChannel = clean_timer(connection_expire_timer, Channel),
+    {ok, [{event, disconnected}, {close, expired}], NChannel};
 handle_timeout(_TRef, Msg, Channel) ->
     log(error, #{msg => "unexpected_timeout", content => Msg}, Channel),
     {ok, Channel}.
@@ -591,10 +599,18 @@ ensure_connected(
 ) ->
     NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
     ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
-    Channel#channel{
+    schedule_connection_expire(Channel#channel{
         conninfo = NConnInfo,
         conn_state = connected
-    }.
+    }).
+
+schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
+    case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
+        undefined ->
+            Channel;
+        Interval ->
+            ensure_timer(connection_expire_timer, Interval, Channel)
+    end.
 
 process_connect(
     Frame,

apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl

@@ -11,6 +11,7 @@
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("emqx/include/asserts.hrl").
 
 -define(BYTE, 8 / big - integer).
 -define(WORD, 16 / big - integer).
@@ -52,6 +53,14 @@ end_per_suite(Config) ->
     emqx_cth_suite:stop(?config(suite_apps, Config)),
     ok.
 
+init_per_testcase(_, Config) ->
+    snabbkaffe:start_trace(),
+    Config.
+
+end_per_testcase(_, _Config) ->
+    snabbkaffe:stop(),
+    ok.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% helper functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 encode(Cmd, Vin, Data) ->
@@ -171,6 +180,28 @@ t_case01_login_channel_info(_Config) ->
 
     ok = gen_tcp:close(Socket).
 
+t_case01_auth_expire(_Config) ->
+    ok = meck:new(emqx_access_control, [passthrough, no_history]),
+    ok = meck:expect(
+        emqx_access_control,
+        authenticate,
+        fun(_) ->
+            {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
+        end
+    ),
+
+    ?assertWaitEvent(
+        begin
+            {ok, _Socket} = login_first()
+        end,
+        #{
+            ?snk_kind := conn_process_terminated,
+            clientid := <<"1G1BL52P7TR115520">>,
+            reason := {shutdown, expired}
+        },
+        5000
+    ).
+
 t_case02_reportinfo_0x01(_Config) ->
     % send VEHICLE LOGIN
     {ok, Socket} = login_first(),

apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway_lwm2m, [
     {description, "LwM2M Gateway"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap, xmerl]},
     {env, []},

apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl

@@ -202,6 +202,8 @@ handle_timeout(_, {transport, _} = Msg, Channel) ->
     call_session(timeout, Msg, Channel);
 handle_timeout(_, disconnect, Channel) ->
     {shutdown, normal, Channel};
+handle_timeout(_, connection_expire, Channel) ->
+    {shutdown, expired, Channel};
 handle_timeout(_, _, Channel) ->
     {ok, Channel}.
 
@@ -353,10 +355,18 @@ ensure_connected(
 
     NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
     ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
-    Channel#channel{
+    schedule_connection_expire(Channel#channel{
         conninfo = NConnInfo,
         conn_state = connected
-    }.
+    }).
+
+schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
+    case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
+        undefined ->
+            Channel;
+        Interval ->
+            make_timer(connection_expire, Interval, connection_expire, Channel)
+    end.
 
 %%--------------------------------------------------------------------
 %% Ensure disconnected

apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl

@@ -36,6 +36,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("emqx/include/asserts.hrl").
 
 -record(coap_content, {content_format, payload = <<>>}).
 
@@ -66,6 +67,7 @@ groups() ->
     [
         {test_grp_0_register, [RepeatOpt], [
             case01_register,
+            case01_auth_expire,
             case01_register_additional_opts,
             %% TODO now we can't handle partial decode packet
             %% case01_register_incorrect_opts,
@@ -145,6 +147,7 @@ end_per_suite(Config) ->
     Config.
 
 init_per_testcase(TestCase, Config) ->
+    snabbkaffe:start_trace(),
     GatewayConfig =
         case TestCase of
             case09_auto_observe ->
@@ -171,6 +174,7 @@ end_per_testcase(_AllTestCase, Config) ->
     timer:sleep(300),
     gen_udp:close(?config(sock, Config)),
     emqtt:disconnect(?config(emqx_c, Config)),
+    snabbkaffe:stop(),
     ok = application:stop(emqx_gateway).
 
 default_config() ->
@@ -280,6 +284,43 @@ case01_register(Config) ->
     timer:sleep(50),
     false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()).
 
+case01_auth_expire(Config) ->
+    ok = meck:new(emqx_access_control, [passthrough, no_history]),
+    ok = meck:expect(
+        emqx_access_control,
+        authenticate,
+        fun(_) ->
+            {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
+        end
+    ),
+
+    %%----------------------------------------
+    %% REGISTER command
+    %%----------------------------------------
+    UdpSock = ?config(sock, Config),
+    Epn = "urn:oma:lwm2m:oma:3",
+    MsgId = 12,
+
+    ?assertWaitEvent(
+        test_send_coap_request(
+            UdpSock,
+            post,
+            sprintf("coap://127.0.0.1:~b/rd?ep=~ts&lt=345&lwm2m=1", [?PORT, Epn]),
+            #coap_content{
+                content_format = <<"text/plain">>,
+                payload = <<"</1>, </2>, </3>, </4>, </5>">>
+            },
+            [],
+            MsgId
+        ),
+        #{
+            ?snk_kind := conn_process_terminated,
+            clientid := <<"urn:oma:lwm2m:oma:3">>,
+            reason := {shutdown, expired}
+        },
+        5000
+    ).
+
 case01_register_additional_opts(Config) ->
     %%----------------------------------------
     %% REGISTER command

apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl

@@ -364,10 +364,18 @@ ensure_connected(
 ) ->
     NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
     ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
-    Channel#channel{
+    schedule_connection_expire(Channel#channel{
         conninfo = NConnInfo,
         conn_state = connected
-    }.
+    }).
+
+schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
+    case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
+        undefined ->
+            Channel;
+        Interval ->
+            ensure_timer(connection_expire, Interval, Channel)
+    end.
 
 process_connect(
     Channel = #channel{
@@ -2122,6 +2130,9 @@ handle_timeout(_TRef, expire_session, Channel) ->
     shutdown(expired, Channel);
 handle_timeout(_TRef, expire_asleep, Channel) ->
     shutdown(asleep_timeout, Channel);
+handle_timeout(_TRef, connection_expire, Channel) ->
+    NChannel = clean_timer(connection_expire, Channel),
+    handle_out(disconnect, expired, NChannel);
 handle_timeout(_TRef, Msg, Channel) ->
     %% NOTE
     %% We do not expect `emqx_mqttsn_session` to set up any custom timers (i.e with

apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl

@@ -33,6 +33,7 @@
 -include_lib("common_test/include/ct.hrl").
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/asserts.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -141,6 +142,14 @@ end_per_suite(Config) ->
     emqx_common_test_http:delete_default_app(),
     emqx_cth_suite:stop(?config(suite_apps, Config)).
 
+init_per_testcase(_TestCase, Config) ->
+    snabbkaffe:start_trace(),
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    snabbkaffe:stop(),
+    ok.
+
 restart_mqttsn_with_subs_resume_on() ->
     Conf = emqx:get_raw_config([gateway, mqttsn]),
     emqx_gateway_conf:update_gateway(
@@ -206,6 +215,36 @@ t_connect(_) ->
     ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
     gen_udp:close(Socket).
 
+t_auth_expire(_) ->
+    SockName = {'mqttsn:udp:default', 1884},
+    ?assertEqual(true, lists:keymember(SockName, 1, esockd:listeners())),
+
+    ok = meck:new(emqx_access_control, [passthrough, no_history]),
+    ok = meck:expect(
+        emqx_access_control,
+        authenticate,
+        fun(_) ->
+            {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
+        end
+    ),
+
+    ?assertWaitEvent(
+        begin
+            {ok, Socket} = gen_udp:open(0, [binary]),
+            send_connect_msg(Socket, <<"client_id_test1">>),
+            ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
+
+            ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
+            gen_udp:close(Socket)
+        end,
+        #{
+            ?snk_kind := conn_process_terminated,
+            clientid := <<"client_id_test1">>,
+            reason := {shutdown, expired}
+        },
+        5000
+    ).
+
 t_first_disconnect(_) ->
     SockName = {'mqttsn:udp:default', 1884},
     ?assertEqual(true, lists:keymember(SockName, 1, esockd:listeners())),

apps/emqx_gateway_ocpp/src/emqx_ocpp_channel.erl

@@ -89,7 +89,8 @@
 -type replies() :: reply() | [reply()].
 
 -define(TIMER_TABLE, #{
-    alive_timer => keepalive
+    alive_timer => keepalive,
+    connection_expire_timer => connection_expire
 }).
 
 -define(INFO_KEYS, [
@@ -315,20 +316,13 @@ enrich_client(
         expiry_interval => 0,
         receive_maximum => 1
     },
-    NClientInfo = fix_mountpoint(
+    NClientInfo =
         ClientInfo#{
             clientid => ClientId,
             username => Username
-        }
+        },
-    ),
     {ok, Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo}}.
 
-fix_mountpoint(ClientInfo = #{mountpoint := undefined}) ->
-    ClientInfo;
-fix_mountpoint(ClientInfo = #{mountpoint := Mountpoint}) ->
-    Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo),
-    ClientInfo#{mountpoint := Mountpoint1}.
-
 set_log_meta(#channel{
     clientinfo = #{clientid := ClientId},
     conninfo = #{peername := Peername}
@@ -350,15 +344,14 @@ check_banned(_UserInfo, #channel{clientinfo = ClientInfo}) ->
 
 auth_connect(
     #{password := Password},
-    #channel{clientinfo = ClientInfo} = Channel
+    #channel{ctx = Ctx, clientinfo = ClientInfo} = Channel
 ) ->
     #{
         clientid := ClientId,
         username := Username
     } = ClientInfo,
-    case emqx_access_control:authenticate(ClientInfo#{password => Password}) of
+    case emqx_gateway_ctx:authenticate(Ctx, ClientInfo#{password => Password}) of
-        {ok, AuthResult} ->
+        {ok, NClientInfo} ->
-            NClientInfo = maps:merge(ClientInfo, AuthResult),
             {ok, Channel#channel{clientinfo = NClientInfo}};
         {error, Reason} ->
             ?SLOG(warning, #{
@@ -659,6 +652,9 @@ handle_timeout(
         {error, timeout} ->
             handle_out(disconnect, keepalive_timeout, Channel)
     end;
+handle_timeout(_TRef, connection_expire, Channel) ->
+    %% No take over implemented, so just shutdown
+    shutdown(expired, Channel);
 handle_timeout(_TRef, Msg, Channel) ->
     ?SLOG(error, #{msg => "unexpected_timeout", timeout_msg => Msg}),
     {ok, Channel}.
@@ -796,10 +792,18 @@ ensure_connected(
 ) ->
     NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
     ok = run_hooks('client.connected', [ClientInfo, NConnInfo]),
-    Channel#channel{
+    schedule_connection_expire(Channel#channel{
         conninfo = NConnInfo,
         conn_state = connected
-    }.
+    }).
+
+schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
+    case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
+        undefined ->
+            Channel;
+        Interval ->
+            ensure_timer(connection_expire_timer, Interval, Channel)
+    end.
 
 ensure_disconnected(
     Reason,

apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl

@@ -20,6 +20,7 @@
 -include("emqx_ocpp.hrl").
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx/include/types.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -logger_header("[OCPP/WS]").
 
@@ -513,7 +514,8 @@ websocket_close(Reason, State) ->
     handle_info({sock_closed, Reason}, State).
 
 terminate(Reason, _Req, #state{channel = Channel}) ->
-    ?SLOG(debug, #{msg => "terminated", reason => Reason}),
+    ClientId = emqx_ocpp_channel:info(clientid, Channel),
+    ?tp(debug, conn_process_terminated, #{reason => Reason, clientid => ClientId}),
     emqx_ocpp_channel:terminate(Reason, Channel);
 terminate(_Reason, _Req, _UnExpectedState) ->
     ok.

apps/emqx_gateway_ocpp/test/emqx_ocpp_SUITE.erl

@@ -19,6 +19,7 @@
 -include("emqx_ocpp.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("emqx/include/asserts.hrl").
 
 -compile(export_all).
 -compile(nowarn_export_all).
@@ -32,8 +33,6 @@
     ]
 ).
 
--define(HEARTBEAT, <<$\n>>).
-
 %% erlfmt-ignore
 -define(CONF_DEFAULT, <<"
     gateway.ocpp {
@@ -82,6 +81,14 @@ end_per_suite(Config) ->
     emqx_cth_suite:stop(?config(suite_apps, Config)),
     ok.
 
+init_per_testcase(_TestCase, Config) ->
+    snabbkaffe:start_trace(),
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    snabbkaffe:stop(),
+    ok.
+
 default_config() ->
     ?CONF_DEFAULT.
 
@@ -188,6 +195,26 @@ t_adjust_keepalive_timer(_Config) ->
     ?assertEqual(undefined, emqx_gateway_cm:get_chan_info(ocpp, <<"client1">>)),
     ok.
 
+t_auth_expire(_Config) ->
+    ok = meck:new(emqx_access_control, [passthrough, no_history]),
+    ok = meck:expect(
+        emqx_access_control,
+        authenticate,
+        fun(_) ->
+            {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
+        end
+    ),
+
+    ?assertWaitEvent(
+        {ok, _ClientPid} = connect("127.0.0.1", 33033, <<"client1">>),
+        #{
+            ?snk_kind := conn_process_terminated,
+            clientid := <<"client1">>,
+            reason := {shutdown, expired}
+        },
+        5000
+    ).
+
 t_listeners_status(_Config) ->
     {200, [Listener]} = request(get, "/gateways/ocpp/listeners"),
     ?assertMatch(

apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl

@@ -93,7 +93,8 @@
 -define(TIMER_TABLE, #{
     incoming_timer => keepalive,
     outgoing_timer => keepalive_send,
-    clean_trans_timer => clean_trans
+    clean_trans_timer => clean_trans,
+    connection_expire_timer => connection_expire
 }).
 
 -define(TRANS_TIMEOUT, 60000).
@@ -356,10 +357,18 @@ ensure_connected(
 ) ->
     NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)},
     ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]),
-    Channel#channel{
+    schedule_connection_expire(Channel#channel{
         conninfo = NConnInfo,
         conn_state = connected
-    }.
+    }).
+
+schedule_connection_expire(Channel = #channel{ctx = Ctx, clientinfo = ClientInfo}) ->
+    case emqx_gateway_ctx:connection_expire_interval(Ctx, ClientInfo) of
+        undefined ->
+            Channel;
+        Interval ->
+            ensure_timer(connection_expire_timer, Interval, Channel)
+    end.
 
 process_connect(
     Channel = #channel{
@@ -1137,7 +1146,10 @@ handle_timeout(_TRef, clean_trans, Channel = #channel{transaction = Trans}) ->
         end,
         Trans
     ),
-    {ok, ensure_clean_trans_timer(Channel#channel{transaction = NTrans})}.
+    {ok, ensure_clean_trans_timer(Channel#channel{transaction = NTrans})};
+handle_timeout(_TRef, connection_expire, Channel) ->
+    %% No session take over implemented, just shut down
+    shutdown(expired, Channel).
 
 %%--------------------------------------------------------------------
 %% Terminate

apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl

@@ -18,6 +18,7 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("emqx/include/asserts.hrl").
 -include("emqx_stomp.hrl").
 
 -compile(export_all).
@@ -78,6 +79,14 @@ end_per_suite(Config) ->
     emqx_cth_suite:stop(?config(suite_apps, Config)),
     ok.
 
+init_per_testcase(_TestCase, Config) ->
+    snabbkaffe:start_trace(),
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    snabbkaffe:stop(),
+    ok.
+
 default_config() ->
     ?CONF_DEFAULT.
 
@@ -141,6 +150,34 @@ t_connect(_) ->
     end,
     with_connection(ProtocolError).
 
+t_auth_expire(_) ->
+    ok = meck:new(emqx_access_control, [passthrough, no_history]),
+    ok = meck:expect(
+        emqx_access_control,
+        authenticate,
+        fun(_) ->
+            {ok, #{is_superuser => false, expire_at => erlang:system_time(millisecond) + 500}}
+        end
+    ),
+
+    ConnectWithExpire = fun(Sock) ->
+        ?assertWaitEvent(
+            begin
+                ok = send_connection_frame(Sock, <<"guest">>, <<"guest">>, <<"1000,2000">>),
+                {ok, Frame} = recv_a_frame(Sock),
+                ?assertMatch(<<"CONNECTED">>, Frame#stomp_frame.command)
+            end,
+            #{
+                ?snk_kind := conn_process_terminated,
+                clientid := _,
+                reason := {shutdown, expired}
+            },
+            5000
+        )
+    end,
+    with_connection(ConnectWithExpire),
+    meck:unload(emqx_access_control).
+
 t_heartbeat(_) ->
     %% Test heart beat
     with_connection(fun(Sock) ->

apps/emqx_license/src/emqx_license.erl

@@ -117,7 +117,9 @@ import_config(#{<<"license">> := Config}) ->
             {ok, #{root_key => license, changed => Changed1}};
         Error ->
             {error, #{root_key => license, reason => Error}}
-    end.
+    end;
+import_config(_RawConf) ->
+    {ok, #{root_key => license, changed => []}}.
 
 %%------------------------------------------------------------------------------
 %% emqx_config_handler callbacks

apps/emqx_machine/priv/reboot_lists.eterm

@@ -89,6 +89,7 @@
             emqx_license,
             emqx_enterprise,
             emqx_message_validation,
+            emqx_connector_aggregator,
             emqx_bridge_kafka,
             emqx_bridge_pulsar,
             emqx_bridge_gcp_pubsub,

apps/emqx_management/src/emqx_mgmt_api_clients.erl

@@ -1745,6 +1745,18 @@ format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}, Opts) ->
 format_channel_info(undefined, {ClientId, PSInfo0 = #{}}, _Opts) ->
     format_persistent_session_info(ClientId, PSInfo0).
 
+format_persistent_session_info(
+    _ClientId, #{metadata := #{offline_info := #{chan_info := ChanInfo, stats := Stats}}} = PSInfo
+) ->
+    Info0 = format_channel_info(_Node = undefined, {_Key = undefined, ChanInfo, Stats}, #{
+        fields => all
+    }),
+    Info0#{
+        connected => false,
+        durable => true,
+        is_persistent => true,
+        subscriptions_cnt => maps:size(maps:get(subscriptions, PSInfo, #{}))
+    };
 format_persistent_session_info(ClientId, PSInfo0) ->
     Metadata = maps:get(metadata, PSInfo0, #{}),
     {ProtoName, ProtoVer} = maps:get(protocol, Metadata),
@@ -1762,6 +1774,7 @@ format_persistent_session_info(ClientId, PSInfo0) ->
         clientid => ClientId,
         connected => false,
         connected_at => CreatedAt,
+        durable => true,
         ip_address => IpAddress,
         is_persistent => true,
         port => Port,

apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl

@@ -171,21 +171,32 @@ subscriptions(get, #{query_string := QString}) ->
             {200, Result}
     end.
 
-format(WhichNode, {{Topic, _Subscriber}, SubOpts}) ->
+format(WhichNode, {{Topic, Subscriber}, SubOpts}) ->
+    FallbackClientId =
+        case is_binary(Subscriber) of
+            true ->
+                Subscriber;
+            false ->
+                %% e.g.: could be a pid...
+                null
+        end,
     maps:merge(
         #{
             topic => emqx_topic:maybe_format_share(Topic),
-            clientid => maps:get(subid, SubOpts, null),
+            clientid => maps:get(subid, SubOpts, FallbackClientId),
-            node => WhichNode,
+            node => convert_null(WhichNode),
             durable => false
         },
-        maps:with([qos, nl, rap, rh], SubOpts)
+        maps:with([qos, nl, rap, rh, durable], SubOpts)
     ).
 
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
 
+convert_null(undefined) -> null;
+convert_null(Val) -> Val.
+
 check_match_topic(#{<<"match_topic">> := MatchTopic}) ->
     try emqx_topic:parse(MatchTopic) of
         {#share{}, _} -> {error, invalid_match_topic};

apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl

@@ -49,6 +49,7 @@ persistent_session_testcases() ->
         t_persistent_sessions3,
         t_persistent_sessions4,
         t_persistent_sessions5,
+        t_persistent_sessions_subscriptions1,
         t_list_clients_v2
     ].
 client_msgs_testcases() ->
@@ -333,7 +334,7 @@ t_persistent_sessions2(Config) ->
             %% 2) Client connects to the same node and takes over, listed only once.
             C2 = connect_client(#{port => Port1, clientid => ClientId}),
             assert_single_client(O#{node => N1, clientid => ClientId, status => connected}),
-            ok = emqtt:disconnect(C2, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}),
+            disconnect_and_destroy_session(C2),
             ?retry(
                 100,
                 20,
@@ -377,7 +378,7 @@ t_persistent_sessions3(Config) ->
                     list_request(APIPort, "node=" ++ atom_to_list(N1))
                 )
             ),
-            ok = emqtt:disconnect(C2, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0})
+            disconnect_and_destroy_session(C2)
         end,
         []
     ),
@@ -417,7 +418,7 @@ t_persistent_sessions4(Config) ->
                     list_request(APIPort, "node=" ++ atom_to_list(N1))
                 )
             ),
-            ok = emqtt:disconnect(C2, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0})
+            disconnect_and_destroy_session(C2)
         end,
         []
     ),
@@ -552,6 +553,63 @@ t_persistent_sessions5(Config) ->
     ),
     ok.
 
+%% Check that the output of `/clients/:clientid/subscriptions' has the expected keys.
+t_persistent_sessions_subscriptions1(Config) ->
+    [N1, _N2] = ?config(nodes, Config),
+    APIPort = 18084,
+    Port1 = get_mqtt_port(N1, tcp),
+
+    ?assertMatch({ok, {{_, 200, _}, _, #{<<"data">> := []}}}, list_request(APIPort)),
+
+    ?check_trace(
+        begin
+            ClientId = <<"c1">>,
+            C1 = connect_client(#{port => Port1, clientid => ClientId}),
+            {ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(C1, <<"topic/1">>, 1),
+            ?assertMatch(
+                {ok,
+                    {{_, 200, _}, _, [
+                        #{
+                            <<"durable">> := true,
+                            <<"node">> := <<_/binary>>,
+                            <<"clientid">> := ClientId,
+                            <<"qos">> := 1,
+                            <<"rap">> := 0,
+                            <<"rh">> := 0,
+                            <<"nl">> := 0,
+                            <<"topic">> := <<"topic/1">>
+                        }
+                    ]}},
+                get_subscriptions_request(APIPort, ClientId)
+            ),
+
+            %% Just disconnect
+            ok = emqtt:disconnect(C1),
+            ?assertMatch(
+                {ok,
+                    {{_, 200, _}, _, [
+                        #{
+                            <<"durable">> := true,
+                            <<"node">> := null,
+                            <<"clientid">> := ClientId,
+                            <<"qos">> := 1,
+                            <<"rap">> := 0,
+                            <<"rh">> := 0,
+                            <<"nl">> := 0,
+                            <<"topic">> := <<"topic/1">>
+                        }
+                    ]}},
+                get_subscriptions_request(APIPort, ClientId)
+            ),
+
+            C2 = connect_client(#{port => Port1, clientid => ClientId}),
+            disconnect_and_destroy_session(C2),
+            ok
+        end,
+        []
+    ),
+    ok.
+
 t_clients_bad_value_type(_) ->
     %% get /clients
     AuthHeader = [emqx_common_test_http:default_auth_header()],
@@ -1800,6 +1858,16 @@ maybe_json_decode(X) ->
         {error, _} -> X
     end.
 
+get_subscriptions_request(APIPort, ClientId) ->
+    Host = "http://127.0.0.1:" ++ integer_to_list(APIPort),
+    Path = emqx_mgmt_api_test_util:api_path(Host, ["clients", ClientId, "subscriptions"]),
+    request(get, Path, []).
+
+get_client_request(Port, ClientId) ->
+    Host = "http://127.0.0.1:" ++ integer_to_list(Port),
+    Path = emqx_mgmt_api_test_util:api_path(Host, ["clients", ClientId]),
+    request(get, Path, []).
+
 list_request(Port) ->
     list_request(Port, _QueryParams = "").
 
@@ -1874,6 +1942,19 @@ assert_single_client(Opts) ->
         {ok, {{_, 200, _}, _, #{<<"connected">> := IsConnected}}},
         lookup_request(ClientId, APIPort)
     ),
+    ?assertMatch(
+        {ok,
+            {{_, 200, _}, _, #{
+                <<"connected">> := IsConnected,
+                <<"is_persistent">> := true,
+                %% contains statistics from disconnect time
+                <<"recv_pkt">> := _,
+                %% contains channel info from disconnect time
+                <<"listener">> := _,
+                <<"clean_start">> := _
+            }}},
+        get_client_request(APIPort, ClientId)
+    ),
     ok.
 
 connect_client(Opts) ->
@@ -1937,3 +2018,6 @@ do_traverse_in_reverse_v2(APIPort, QueryParams0, [Cursor | Rest], DirectOrderCli
     {ok, {{_, 200, _}, _, #{<<"data">> := Rows}}} = Res0,
     ClientIds = [ClientId || #{<<"clientid">> := ClientId} <- Rows],
     do_traverse_in_reverse_v2(APIPort, QueryParams0, Rest, DirectOrderClientIds, ClientIds ++ Acc).
+
+disconnect_and_destroy_session(Client) ->
+    ok = emqtt:disconnect(Client, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}).

apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl

@@ -290,25 +290,89 @@ t_http_test_json_formatter(_Config) ->
         end
      || JSONEntry <- LogEntries
     ],
+    ListIterFun =
+        fun
+            ListIterFunRec([]) ->
+                ok;
+            ListIterFunRec([Item | Rest]) ->
+                receive
+                    From ->
+                        From ! {list_iter_item, Item}
+                end,
+                ListIterFunRec(Rest)
+        end,
+    ListIter = spawn_link(fun() -> ListIterFun(DecodedLogEntries) end),
+    NextFun =
+        fun() ->
+            ListIter ! self(),
+            receive
+                {list_iter_item, Item} ->
+                    Item
+            end
+        end,
     ?assertMatch(
-        [
         #{<<"meta">> := #{<<"payload">> := <<"log_this_message">>}},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := #{<<"payload">> := <<"\nlog\nthis\nmessage">>}},
+        NextFun()
+    ),
+    ?assertMatch(
         #{
             <<"meta">> := #{<<"payload">> := <<"\\\nlog\n_\\n_this\nmessage\\">>}
         },
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := #{<<"payload">> := <<"\"log_this_message\"">>}},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := #{<<"str">> := <<"str">>}},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := #{<<"term">> := <<"{notjson}">>}},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := <<_/binary>>},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := #{<<"integer">> := 42}},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := #{<<"float">> := 1.2}},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := <<_/binary>>},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := <<_/binary>>},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := <<_/binary>>},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := #{<<"sub">> := #{}}},
+        NextFun()
+    ),
+    ?assertMatch(
         #{<<"meta">> := #{<<"sub">> := #{<<"key">> := <<"value">>}}},
-            #{<<"meta">> := #{<<"true">> := <<"true">>, <<"false">> := <<"false">>}},
+        NextFun()
+    ),
+    ?assertMatch(
+        #{<<"meta">> := #{<<"true">> := true, <<"false">> := false}},
+        NextFun()
+    ),
+    ?assertMatch(
         #{
             <<"meta">> := #{
                 <<"list">> := #{
@@ -317,16 +381,25 @@ t_http_test_json_formatter(_Config) ->
                 }
             }
         },
+        NextFun()
+    ),
+    ?assertMatch(
         #{
             <<"meta">> := #{
                 <<"client_ids">> := [<<"a">>, <<"b">>, <<"c">>]
             }
         },
+        NextFun()
+    ),
+    ?assertMatch(
         #{
             <<"meta">> := #{
                 <<"rule_ids">> := [<<"a">>, <<"b">>, <<"c">>]
             }
         },
+        NextFun()
+    ),
+    ?assertMatch(
         #{
             <<"meta">> := #{
                 <<"action_info">> := #{
@@ -334,10 +407,8 @@ t_http_test_json_formatter(_Config) ->
                     <<"name">> := <<"emqx_bridge_http_test_lib">>
                 }
             }
-            }
+        },
-            | _
+        NextFun()
-        ],
-        DecodedLogEntries
     ),
     {ok, Delete} = request_api(delete, api_path("trace/" ++ binary_to_list(Name))),
     ?assertEqual(<<>>, Delete),
@@ -495,7 +566,7 @@ create_trace(Name, Type, TypeValue, Start) ->
             ?block_until(#{?snk_kind := update_trace_done})
         end,
         fun(Trace) ->
-            ?assertMatch([#{}], ?of_kind(update_trace_done, Trace))
+            ?assertMatch([#{} | _], ?of_kind(update_trace_done, Trace))
         end
     ).
 

apps/emqx_message_validation/src/emqx_message_validation.erl

@@ -69,10 +69,22 @@ remove_handler() ->
     ok.
 
 load() ->
-    lists:foreach(fun insert/1, emqx:get_config(?VALIDATIONS_CONF_PATH, [])).
+    Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
+    lists:foreach(
+        fun({Pos, Validation}) ->
+            ok = emqx_message_validation_registry:insert(Pos, Validation)
+        end,
+        lists:enumerate(Validations)
+    ).
 
 unload() ->
-    lists:foreach(fun delete/1, emqx:get_config(?VALIDATIONS_CONF_PATH, [])).
+    Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
+    lists:foreach(
+        fun(Validation) ->
+            ok = emqx_message_validation_registry:delete(Validation)
+        end,
+        Validations
+    ).
 
 -spec list() -> [validation()].
 list() ->
@@ -81,7 +93,7 @@ list() ->
 -spec reorder([validation_name()]) ->
     {ok, _} | {error, _}.
 reorder(Order) ->
-    emqx:update_config(
+    emqx_conf:update(
         ?VALIDATIONS_CONF_PATH,
         {reorder, Order},
         #{override_to => cluster}
@@ -95,7 +107,7 @@ lookup(Name) ->
 -spec insert(validation()) ->
     {ok, _} | {error, _}.
 insert(Validation) ->
-    emqx:update_config(
+    emqx_conf:update(
         ?VALIDATIONS_CONF_PATH,
         {append, Validation},
         #{override_to => cluster}
@@ -104,7 +116,7 @@ insert(Validation) ->
 -spec update(validation()) ->
     {ok, _} | {error, _}.
 update(Validation) ->
-    emqx:update_config(
+    emqx_conf:update(
         ?VALIDATIONS_CONF_PATH,
         {update, Validation},
         #{override_to => cluster}
@@ -113,7 +125,7 @@ update(Validation) ->
 -spec delete(validation_name()) ->
     {ok, _} | {error, _}.
 delete(Name) ->
-    emqx:update_config(
+    emqx_conf:update(
         ?VALIDATIONS_CONF_PATH,
         {delete, Name},
         #{override_to => cluster}
@@ -247,8 +259,8 @@ evaluate_schema_check(Check, Validation, #message{payload = Data}) ->
     #{name := Name} = Validation,
     ExtraArgs =
         case Check of
-            #{type := protobuf, message_name := MessageName} ->
+            #{type := protobuf, message_type := MessageType} ->
-                [MessageName];
+                [MessageType];
             _ ->
                 []
         end,

apps/emqx_message_validation/src/emqx_message_validation_schema.erl

@@ -18,8 +18,6 @@
     api_schema/1
 ]).
 
--export([validate_name/1]).
-
 %%------------------------------------------------------------------------------
 %% Type declarations
 %%------------------------------------------------------------------------------
@@ -55,7 +53,7 @@ fields(validation) ->
                 binary(),
                 #{
                     required => true,
-                    validator => fun validate_name/1,
+                    validator => fun emqx_resource:validate_name/1,
                     desc => ?DESC("name")
                 }
             )},
@@ -123,8 +121,8 @@ fields(check_protobuf) ->
     [
         {type, mk(protobuf, #{default => protobuf, desc => ?DESC("check_protobuf_type")})},
         {schema, mk(binary(), #{required => true, desc => ?DESC("check_protobuf_schema")})},
-        {message_name,
+        {message_type,
-            mk(binary(), #{required => true, desc => ?DESC("check_protobuf_message_name")})}
+            mk(binary(), #{required => true, desc => ?DESC("check_protobuf_message_type")})}
     ];
 fields(check_avro) ->
     [
@@ -200,16 +198,6 @@ ensure_array(undefined, _) -> undefined;
 ensure_array(L, _) when is_list(L) -> L;
 ensure_array(B, _) -> [B].
 
-validate_name(Name) ->
-    %% see `MAP_KEY_RE' in hocon_tconf
-    RE = <<"^[A-Za-z0-9]+[A-Za-z0-9-_]*$">>,
-    case re:run(Name, RE, [{capture, none}]) of
-        match ->
-            ok;
-        nomatch ->
-            {error, <<"must conform to regex: ", RE/binary>>}
-    end.
-
 validate_sql(SQL) ->
     case emqx_message_validation:parse_sql_check(SQL) of
         {ok, _Parsed} ->

apps/emqx_message_validation/test/emqx_message_validation_http_api_SUITE.erl

@@ -317,9 +317,9 @@ avro_create_serde(SerdeName) ->
     on_exit(fun() -> ok = emqx_schema_registry:delete_schema(SerdeName) end),
     ok.
 
-protobuf_valid_payloads(SerdeName, MessageName) ->
+protobuf_valid_payloads(SerdeName, MessageType) ->
     lists:map(
-        fun(Payload) -> emqx_schema_registry_serde:encode(SerdeName, Payload, [MessageName]) end,
+        fun(Payload) -> emqx_schema_registry_serde:encode(SerdeName, Payload, [MessageType]) end,
         [
             #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>},
             #{<<"name">> => <<"some name">>, <<"id">> => 10}
@@ -1176,11 +1176,11 @@ t_schema_check_avro(_Config) ->
 
 t_schema_check_protobuf(_Config) ->
     SerdeName = <<"myserde">>,
-    MessageName = <<"Person">>,
+    MessageType = <<"Person">>,
     protobuf_create_serde(SerdeName),
 
     Name1 = <<"foo">>,
-    Check1 = schema_check(protobuf, SerdeName, #{<<"message_name">> => MessageName}),
+    Check1 = schema_check(protobuf, SerdeName, #{<<"message_type">> => MessageType}),
     Validation1 = validation(Name1, [Check1]),
     {201, _} = insert(Validation1),
 
@@ -1192,7 +1192,7 @@ t_schema_check_protobuf(_Config) ->
             ok = publish(C, <<"t/1">>, {raw, Payload}),
             ?assertReceive({publish, _})
         end,
-        protobuf_valid_payloads(SerdeName, MessageName)
+        protobuf_valid_payloads(SerdeName, MessageType)
     ),
     lists:foreach(
         fun(Payload) ->
@@ -1203,7 +1203,7 @@ t_schema_check_protobuf(_Config) ->
     ),
 
     %% Bad config: unknown message name
-    Check2 = schema_check(protobuf, SerdeName, #{<<"message_name">> => <<"idontexist">>}),
+    Check2 = schema_check(protobuf, SerdeName, #{<<"message_type">> => <<"idontexist">>}),
     Validation2 = validation(Name1, [Check2]),
     {200, _} = update(Validation2),
 
@@ -1212,7 +1212,7 @@ t_schema_check_protobuf(_Config) ->
             ok = publish(C, <<"t/1">>, {raw, Payload}),
             ?assertNotReceive({publish, _})
         end,
-        protobuf_valid_payloads(SerdeName, MessageName)
+        protobuf_valid_payloads(SerdeName, MessageType)
     ),
 
     ok.