Merge pull request #13392 from thalesmg/20240702-m-sync-r57-mix-umbrella

sync release-57 to master

apps/emqx/include/emqx.hrl

@@ -95,4 +95,10 @@
     until :: integer()
 }).
 
+%%--------------------------------------------------------------------
+%% Configurations
+%%--------------------------------------------------------------------
+-define(KIND_REPLICATE, replicate).
+-define(KIND_INITIATE, initiate).
+
 -endif.

apps/emqx/lib/mix/tasks/compile.copy_srcs.ex

@@ -0,0 +1,37 @@
+defmodule Mix.Tasks.Compile.CopySrcs do
+  use Mix.Task.Compiler
+
+  @recursive true
+
+  @impl true
+  def run(_args) do
+    Mix.Project.get!()
+    config = Mix.Project.config()
+    extra_dirs = config[:extra_dirs]
+
+    unless extra_dirs && is_list(extra_dirs) do
+      Mix.raise("application option :extra_dirs in #{Mix.Project.project_file()} must be a list of directories under the application")
+    end
+
+    app_root = File.cwd!()
+    app_build_path = Mix.Project.app_path(config)
+
+    for extra_dir <- extra_dirs do
+      src = Path.join([app_root, extra_dir])
+      dest = Path.join([app_build_path, extra_dir])
+      File.rm(dest)
+      case File.ln_s(src, dest) do
+        :ok ->
+          :ok
+
+        {:error, :eexist} ->
+          Mix.shell().info(IO.ANSI.format([:yellow, "#{dest} still exists after attempted removal"]))
+          :ok
+
+        {:error, error} ->
+          Mix.raise("error trying to link #{src} to #{dest}: #{error}")
+      end
+    end
+    {:noop, []}
+  end
+end

apps/emqx/mix.exs

@@ -0,0 +1,59 @@
+defmodule EMQX.MixProject do
+  use Mix.Project
+
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_paths: UMP.erlc_paths(),
+      erlc_options: [
+        {:i, "src"}
+        | UMP.erlc_options()
+      ],
+      compilers: Mix.compilers() ++ [:copy_srcs],
+      # used by our `Mix.Tasks.Compile.CopySrcs` compiler
+      extra_dirs: extra_dirs(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [
+      ## FIXME!!! go though emqx.app.src and add missing stuff...
+      extra_applications: [:public_key, :ssl, :os_mon, :logger, :mnesia] ++ UMP.extra_applications(),
+      mod: {:emqx_app, []}
+    ]
+  end
+
+  def deps() do
+    ## FIXME!!! go though emqx.app.src and add missing stuff...
+    [
+      {:emqx_utils, in_umbrella: true},
+      {:emqx_ds_backends, in_umbrella: true},
+
+      {:ekka, github: "emqx/ekka", tag: "0.19.3", override: true},
+      {:esockd, github: "emqx/esockd", tag: "5.11.2"},
+      {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.42.2", override: true},
+      {:lc, github: "emqx/lc", tag: "0.3.2", override: true},
+      {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true},
+    ] ++ UMP.quicer_dep()
+  end
+
+  defp extra_dirs() do
+    dirs = ["src", "etc"]
+    if UMP.test_env?() do
+      ["test", "integration_test" | dirs]
+    else
+      dirs
+    end
+  end
+end

apps/emqx/src/config/emqx_config_logger.erl

@@ -239,8 +239,9 @@ log_formatter(HandlerName, Conf) ->
         end,
     TsFormat = timestamp_format(Conf),
     WithMfa = conf_get("with_mfa", Conf),
+    PayloadEncode = conf_get("payload_encode", Conf, text),
     do_formatter(
-        Format, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa
+        Format, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa, PayloadEncode
     ).
 
 %% auto | epoch | rfc3339
@@ -248,16 +249,17 @@ timestamp_format(Conf) ->
     conf_get("timestamp_format", Conf).
 
 %% helpers
-do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa) ->
+do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa, PayloadEncode) ->
     {emqx_logger_jsonfmt, #{
         chars_limit => CharsLimit,
         single_line => SingleLine,
         time_offset => TimeOffSet,
         depth => Depth,
         timestamp_format => TsFormat,
-        with_mfa => WithMfa
+        with_mfa => WithMfa,
+        payload_encode => PayloadEncode
     }};
-do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa) ->
+do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa, PayloadEncode) ->
     {emqx_logger_textfmt, #{
         template => ["[", level, "] ", msg, "\n"],
         chars_limit => CharsLimit,
@@ -265,7 +267,8 @@ do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa)
         time_offset => TimeOffSet,
         depth => Depth,
         timestamp_format => TsFormat,
-        with_mfa => WithMfa
+        with_mfa => WithMfa,
+        payload_encode => PayloadEncode
     }}.
 
 %% Don't record all logger message

apps/emqx/src/emqx.erl

@@ -61,9 +61,12 @@
     get_raw_config/2,
     update_config/2,
     update_config/3,
+    update_config/4,
     remove_config/1,
     remove_config/2,
+    remove_config/3,
     reset_config/2,
+    reset_config/3,
     data_dir/0,
     etc_file/1,
     cert_file/1,
@@ -195,7 +198,7 @@ get_raw_config(KeyPath, Default) ->
 -spec update_config(emqx_utils_maps:config_key_path(), emqx_config:update_request()) ->
     {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
 update_config(KeyPath, UpdateReq) ->
-    update_config(KeyPath, UpdateReq, #{}).
+    update_config(KeyPath, UpdateReq, #{}, #{}).
 
 -spec update_config(
     emqx_utils_maps:config_key_path(),
@@ -203,30 +206,56 @@ update_config(KeyPath, UpdateReq) ->
     emqx_config:update_opts()
 ) ->
     {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
-update_config([RootName | _] = KeyPath, UpdateReq, Opts) ->
+update_config(KeyPath, UpdateReq, Opts) ->
+    update_config(KeyPath, UpdateReq, Opts, #{}).
+
+-spec update_config(
+    emqx_utils_maps:config_key_path(),
+    emqx_config:update_request(),
+    emqx_config:update_opts(),
+    emqx_config:cluster_rpc_opts()
+) ->
+    {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
+update_config([RootName | _] = KeyPath, UpdateReq, Opts, ClusterRpcOpts) ->
     emqx_config_handler:update_config(
         emqx_config:get_schema_mod(RootName),
         KeyPath,
-        {{update, UpdateReq}, Opts}
+        {{update, UpdateReq}, Opts},
+        ClusterRpcOpts
     ).
 
 -spec remove_config(emqx_utils_maps:config_key_path()) ->
     {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
 remove_config(KeyPath) ->
-    remove_config(KeyPath, #{}).
+    remove_config(KeyPath, #{}, #{}).
 
 -spec remove_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) ->
     {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
-remove_config([RootName | _] = KeyPath, Opts) ->
+remove_config([_RootName | _] = KeyPath, Opts) ->
+    remove_config(KeyPath, Opts, #{}).
+
+-spec remove_config(
+    emqx_utils_maps:config_key_path(), emqx_config:update_opts(), emqx_config:cluster_rpc_opts()
+) ->
+    {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
+remove_config([RootName | _] = KeyPath, Opts, ClusterRpcOpts) ->
     emqx_config_handler:update_config(
         emqx_config:get_schema_mod(RootName),
         KeyPath,
-        {remove, Opts}
+        {remove, Opts},
+        ClusterRpcOpts
     ).
 
 -spec reset_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) ->
     {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
 reset_config([RootName | SubKeys] = KeyPath, Opts) ->
+    reset_config([RootName | SubKeys] = KeyPath, Opts, #{}).
+
+-spec reset_config(
+    emqx_utils_maps:config_key_path(), emqx_config:update_opts(), emqx_config:cluster_rpc_opts()
+) ->
+    {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
+reset_config([RootName | SubKeys] = KeyPath, Opts, ClusterRpcOpts) ->
     case emqx_config:get_default_value(KeyPath) of
         {ok, Default} ->
             Mod = emqx_config:get_schema_mod(RootName),
@@ -235,7 +264,8 @@ reset_config([RootName | SubKeys] = KeyPath, Opts) ->
                     emqx_config_handler:update_config(
                         Mod,
                         KeyPath,
-                        {{update, Default}, Opts}
+                        {{update, Default}, Opts},
+                        ClusterRpcOpts
                     );
                 false ->
                     NewConf =
@@ -247,7 +277,8 @@ reset_config([RootName | SubKeys] = KeyPath, Opts) ->
                     emqx_config_handler:update_config(
                         Mod,
                         [RootName],
-                        {{update, NewConf}, Opts}
+                        {{update, NewConf}, Opts},
+                        ClusterRpcOpts
                     )
             end;
         {error, _} = Error ->

apps/emqx/src/emqx_channel.erl

@@ -270,7 +270,7 @@ init(
         },
         Zone
     ),
-    {NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo),
+    {NClientInfo, NConnInfo} = take_conn_info_fields([ws_cookie, peersni], ClientInfo, ConnInfo),
     #channel{
         conninfo = NConnInfo,
         clientinfo = NClientInfo,
@@ -310,13 +310,19 @@ set_peercert_infos(Peercert, ClientInfo, Zone) ->
     ClientId = PeercetAs(peer_cert_as_clientid),
     ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}.
 
-take_ws_cookie(ClientInfo, ConnInfo) ->
+take_conn_info_fields(Fields, ClientInfo, ConnInfo) ->
-    case maps:take(ws_cookie, ConnInfo) of
+    lists:foldl(
-        {WsCookie, NConnInfo} ->
+        fun(Field, {ClientInfo0, ConnInfo0}) ->
-            {ClientInfo#{ws_cookie => WsCookie}, NConnInfo};
+            case maps:take(Field, ConnInfo0) of
-        _ ->
+                {Value, NConnInfo} ->
-            {ClientInfo, ConnInfo}
+                    {ClientInfo0#{Field => Value}, NConnInfo};
-    end.
+                _ ->
+                    {ClientInfo0, ConnInfo0}
+            end
+        end,
+        {ClientInfo, ConnInfo},
+        Fields
+    ).
 
 %%--------------------------------------------------------------------
 %% Handle incoming packet

apps/emqx/src/emqx_config.erl

@@ -118,6 +118,7 @@
     config/0,
     app_envs/0,
     update_opts/0,
+    cluster_rpc_opts/0,
     update_cmd/0,
     update_args/0,
     update_error/0,
@@ -147,6 +148,7 @@
     raw_config => emqx_config:raw_config(),
     post_config_update => #{module() => any()}
 }.
+-type cluster_rpc_opts() :: #{kind => ?KIND_INITIATE | ?KIND_REPLICATE}.
 
 %% raw_config() is the config that is NOT parsed and translated by hocon schema
 -type raw_config() :: #{binary() => term()} | list() | undefined.

apps/emqx/src/emqx_config_handler.erl

@@ -18,6 +18,7 @@
 -module(emqx_config_handler).
 
 -include("logger.hrl").
+-include("emqx.hrl").
 -include("emqx_schema.hrl").
 -include_lib("hocon/include/hocon_types.hrl").
 
@@ -30,6 +31,7 @@
     add_handler/2,
     remove_handler/1,
     update_config/3,
+    update_config/4,
     get_raw_cluster_override_conf/0,
     info/0
 ]).
@@ -53,9 +55,13 @@
 
 -optional_callbacks([
     pre_config_update/3,
+    pre_config_update/4,
     propagated_pre_config_update/3,
+    propagated_pre_config_update/4,
     post_config_update/5,
-    propagated_post_config_update/5
+    post_config_update/6,
+    propagated_post_config_update/5,
+    propagated_post_config_update/6
 ]).
 
 -callback pre_config_update([atom()], emqx_config:update_request(), emqx_config:raw_config()) ->
@@ -83,6 +89,38 @@
 ) ->
     ok | {ok, Result :: any()} | {error, Reason :: term()}.
 
+-callback pre_config_update(
+    [atom()], emqx_config:update_request(), emqx_config:raw_config(), emqx_config:cluster_rpc_opts()
+) ->
+    ok | {ok, emqx_config:update_request()} | {error, term()}.
+-callback propagated_pre_config_update(
+    [binary()],
+    emqx_config:update_request(),
+    emqx_config:raw_config(),
+    emqx_config:cluster_rpc_opts()
+) ->
+    ok | {ok, emqx_config:update_request()} | {error, term()}.
+
+-callback post_config_update(
+    [atom()],
+    emqx_config:update_request(),
+    emqx_config:config(),
+    emqx_config:config(),
+    emqx_config:app_envs(),
+    emqx_config:cluster_rpc_opts()
+) ->
+    ok | {ok, Result :: any()} | {error, Reason :: term()}.
+
+-callback propagated_post_config_update(
+    [atom()],
+    emqx_config:update_request(),
+    emqx_config:config(),
+    emqx_config:config(),
+    emqx_config:app_envs(),
+    emqx_config:cluster_rpc_opts()
+) ->
+    ok | {ok, Result :: any()} | {error, Reason :: term()}.
+
 -type state() :: #{handlers := any()}.
 -type config_key_path() :: emqx_utils_maps:config_key_path().
 
@@ -92,12 +130,17 @@ start_link() ->
 stop() ->
     gen_server:stop(?MODULE).
 
--spec update_config(module(), config_key_path(), emqx_config:update_args()) ->
-    {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
 update_config(SchemaModule, ConfKeyPath, UpdateArgs) ->
+    update_config(SchemaModule, ConfKeyPath, UpdateArgs, #{}).
+
+-spec update_config(module(), config_key_path(), emqx_config:update_args(), map()) ->
+    {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
+update_config(SchemaModule, ConfKeyPath, UpdateArgs, ClusterOpts) ->
     %% force convert the path to a list of atoms, as there maybe some wildcard names/ids in the path
     AtomKeyPath = [atom(Key) || Key <- ConfKeyPath],
-    gen_server:call(?MODULE, {change_config, SchemaModule, AtomKeyPath, UpdateArgs}, infinity).
+    gen_server:call(
+        ?MODULE, {change_config, SchemaModule, AtomKeyPath, UpdateArgs, ClusterOpts}, infinity
+    ).
 
 -spec add_handler(config_key_path(), handler_name()) ->
     ok | {error, {conflict, list()}}.
@@ -130,11 +173,11 @@ handle_call({add_handler, ConfKeyPath, HandlerName}, _From, State = #{handlers :
         {error, _Reason} = Error -> {reply, Error, State}
     end;
 handle_call(
-    {change_config, SchemaModule, ConfKeyPath, UpdateArgs},
+    {change_config, SchemaModule, ConfKeyPath, UpdateArgs, ClusterRpcOpts},
     _From,
     #{handlers := Handlers} = State
 ) ->
-    Result = handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs),
+    Result = handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs, ClusterRpcOpts),
     {reply, Result, State};
 handle_call(get_raw_cluster_override_conf, _From, State) ->
     Reply = emqx_config:read_override_conf(#{override_to => cluster}),
@@ -203,9 +246,9 @@ filter_top_level_handlers(Handlers) ->
         Handlers
     ).
 
-handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs) ->
+handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs, ClusterRpcOpts) ->
     try
-        do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs)
+        do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs, ClusterRpcOpts)
     catch
         throw:Reason ->
             {error, Reason};
@@ -217,13 +260,14 @@ handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs) ->
                 update_req => UpdateArgs,
                 module => SchemaModule,
                 key_path => ConfKeyPath,
+                cluster_rpc_opts => ClusterRpcOpts,
                 stacktrace => ST
             }),
             {error, {config_update_crashed, Reason}}
     end.
 
-do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs) ->
+do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs, ClusterOpts) ->
-    case process_update_request(ConfKeyPath, Handlers, UpdateArgs) of
+    case process_update_request(ConfKeyPath, Handlers, UpdateArgs, ClusterOpts) of
         {ok, NewRawConf, OverrideConf, Opts} ->
             check_and_save_configs(
                 SchemaModule,
@@ -232,23 +276,24 @@ do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs) ->
                 NewRawConf,
                 OverrideConf,
                 UpdateArgs,
-                Opts
+                Opts,
+                ClusterOpts
             );
         {error, Result} ->
             {error, Result}
     end.
 
-process_update_request([_], _Handlers, {remove, _Opts}) ->
+process_update_request([_], _Handlers, {remove, _Opts}, _ClusterRpcOpts) ->
     {error, "remove_root_is_forbidden"};
-process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) ->
+process_update_request(ConfKeyPath, _Handlers, {remove, Opts}, _ClusterRpcOpts) ->
     OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
     BinKeyPath = bin_path(ConfKeyPath),
     NewRawConf = emqx_utils_maps:deep_remove(BinKeyPath, OldRawConf),
     OverrideConf = remove_from_override_config(BinKeyPath, Opts),
     {ok, NewRawConf, OverrideConf, Opts};
-process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) ->
+process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}, ClusterRpcOpts) ->
     OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
-    case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of
+    case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq, ClusterRpcOpts) of
         {ok, NewRawConf} ->
             OverrideConf = merge_to_override_config(NewRawConf, Opts),
             {ok, NewRawConf, OverrideConf, Opts};
@@ -256,15 +301,16 @@ process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) ->
             Error
     end.
 
-do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) ->
+do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq, ClusterRpcOpts) ->
-    do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq, []).
+    do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq, ClusterRpcOpts, []).
 
-do_update_config([], Handlers, OldRawConf, UpdateReq, ConfKeyPath) ->
+do_update_config([], Handlers, OldRawConf, UpdateReq, ClusterRpcOpts, ConfKeyPath) ->
     call_pre_config_update(#{
         handlers => Handlers,
         old_raw_conf => OldRawConf,
         update_req => UpdateReq,
         conf_key_path => ConfKeyPath,
+        cluster_rpc_opts => ClusterRpcOpts,
         callback => pre_config_update,
         is_propagated => false
     });
@@ -273,13 +319,18 @@ do_update_config(
     Handlers,
     OldRawConf,
     UpdateReq,
+    ClusterRpcOpts,
     ConfKeyPath0
 ) ->
     ConfKeyPath = ConfKeyPath0 ++ [ConfKey],
     ConfKeyBin = bin(ConfKey),
     SubOldRawConf = get_sub_config(ConfKeyBin, OldRawConf),
     SubHandlers = get_sub_handlers(ConfKey, Handlers),
-    case do_update_config(SubConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq, ConfKeyPath) of
+    case
+        do_update_config(
+            SubConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq, ClusterRpcOpts, ConfKeyPath
+        )
+    of
         {ok, NewUpdateReq} ->
             merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf);
         Error ->
@@ -293,12 +344,18 @@ check_and_save_configs(
     NewRawConf,
     OverrideConf,
     UpdateArgs,
-    Opts
+    Opts,
+    ClusterOpts
 ) ->
     Schema = schema(SchemaModule, ConfKeyPath),
+    Kind = maps:get(kind, ClusterOpts, ?KIND_INITIATE),
     {AppEnvs, NewConf} = emqx_config:check_config(Schema, NewRawConf),
     OldConf = emqx_config:get_root(ConfKeyPath),
-    case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of
+    case
+        do_post_config_update(
+            ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, ClusterOpts, #{}
+        )
+    of
         {ok, Result0} ->
             post_update_ok(
                 AppEnvs,
@@ -310,21 +367,24 @@ check_and_save_configs(
                 UpdateArgs,
                 Result0
             );
-        {error, {post_config_update, HandlerName, Reason}} ->
+        {error, {post_config_update, HandlerName, Reason}} when Kind =/= ?KIND_INITIATE ->
-            HandlePostFailureFun =
+            ?SLOG(critical, #{
-                fun() ->
+                msg => "post_config_update_failed_but_save_the_config_anyway",
-                    post_update_ok(
+                handler => HandlerName,
-                        AppEnvs,
+                reason => Reason
-                        NewConf,
+            }),
-                        NewRawConf,
+            post_update_ok(
-                        OverrideConf,
+                AppEnvs,
-                        Opts,
+                NewConf,
-                        ConfKeyPath,
+                NewRawConf,
-                        UpdateArgs,
+                OverrideConf,
-                        #{}
+                Opts,
-                    )
+                ConfKeyPath,
-                end,
+                UpdateArgs,
-            {error, {post_config_update, HandlerName, {Reason, HandlePostFailureFun}}}
+                #{}
+            );
+        {error, _} = Error ->
+            Error
     end.
 
 post_update_ok(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts, ConfKeyPath, UpdateArgs, Result0) ->
@@ -332,7 +392,9 @@ post_update_ok(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts, ConfKeyPath, Up
     Result1 = return_change_result(ConfKeyPath, UpdateArgs),
     {ok, Result1#{post_config_update => Result0}}.
 
-do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, Result) ->
+do_post_config_update(
+    ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, ClusterOpts, Result
+) ->
     do_post_config_update(
         ConfKeyPath,
         Handlers,
@@ -340,6 +402,7 @@ do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateAr
         NewConf,
         AppEnvs,
         UpdateArgs,
+        ClusterOpts,
         Result,
         []
     ).
@@ -352,6 +415,7 @@ do_post_config_update(
     AppEnvs,
     UpdateArgs,
     Result,
+    ClusterOpts,
     ConfKeyPath
 ) ->
     call_post_config_update(#{
@@ -362,6 +426,7 @@ do_post_config_update(
         update_req => up_req(UpdateArgs),
         result => Result,
         conf_key_path => ConfKeyPath,
+        cluster_rpc_opts => ClusterOpts,
         callback => post_config_update
     });
 do_post_config_update(
@@ -371,6 +436,7 @@ do_post_config_update(
     NewConf,
     AppEnvs,
     UpdateArgs,
+    ClusterOpts,
     Result,
     ConfKeyPath0
 ) ->
@@ -385,6 +451,7 @@ do_post_config_update(
         SubNewConf,
         AppEnvs,
         UpdateArgs,
+        ClusterOpts,
         Result,
         ConfKeyPath
     ).
@@ -428,37 +495,41 @@ call_proper_pre_config_update(
     #{
         handlers := #{?MOD := Module},
         callback := Callback,
-        update_req := UpdateReq,
+        update_req := UpdateReq
-        old_raw_conf := OldRawConf
     } = Ctx
 ) ->
-    case erlang:function_exported(Module, Callback, 3) of
+    Arity = get_function_arity(Module, Callback, [3, 4]),
-        true ->
+    case apply_pre_config_update(Module, Callback, Arity, Ctx) of
-            case apply_pre_config_update(Module, Ctx) of
+        {ok, NewUpdateReq} ->
-                {ok, NewUpdateReq} ->
+            {ok, NewUpdateReq};
-                    {ok, NewUpdateReq};
+        ok ->
-                ok ->
+            {ok, UpdateReq};
-                    {ok, UpdateReq};
+        {error, Reason} ->
-                {error, Reason} ->
+            {error, {pre_config_update, Module, Reason}}
-                    {error, {pre_config_update, Module, Reason}}
-            end;
-        false ->
-            merge_to_old_config(UpdateReq, OldRawConf)
     end;
 call_proper_pre_config_update(
     #{update_req := UpdateReq}
 ) ->
     {ok, UpdateReq}.
 
-apply_pre_config_update(Module, #{
+apply_pre_config_update(Module, Callback, 3, #{
+    conf_key_path := ConfKeyPath,
+    update_req := UpdateReq,
+    old_raw_conf := OldRawConf
+}) ->
+    Module:Callback(ConfKeyPath, UpdateReq, OldRawConf);
+apply_pre_config_update(Module, Callback, 4, #{
     conf_key_path := ConfKeyPath,
     update_req := UpdateReq,
     old_raw_conf := OldRawConf,
-    callback := Callback
+    cluster_rpc_opts := ClusterRpcOpts
 }) ->
-    Module:Callback(
+    Module:Callback(ConfKeyPath, UpdateReq, OldRawConf, ClusterRpcOpts);
-        ConfKeyPath, UpdateReq, OldRawConf
+apply_pre_config_update(_Module, _Callback, false, #{
-    ).
+    update_req := UpdateReq,
+    old_raw_conf := OldRawConf
+}) ->
+    merge_to_old_config(UpdateReq, OldRawConf).
 
 propagate_pre_config_updates_to_subconf(
     #{handlers := #{?WKEY := _}} = Ctx
@@ -560,28 +631,23 @@ call_proper_post_config_update(
         result := Result
     } = Ctx
 ) ->
-    case erlang:function_exported(Module, Callback, 5) of
+    Arity = get_function_arity(Module, Callback, [5, 6]),
-        true ->
+    case apply_post_config_update(Module, Callback, Arity, Ctx) of
-            case apply_post_config_update(Module, Ctx) of
+        ok -> {ok, Result};
-                ok -> {ok, Result};
+        {ok, Result1} -> {ok, Result#{Module => Result1}};
-                {ok, Result1} -> {ok, Result#{Module => Result1}};
+        {error, Reason} -> {error, {post_config_update, Module, Reason}}
-                {error, Reason} -> {error, {post_config_update, Module, Reason}}
-            end;
-        false ->
-            {ok, Result}
     end;
 call_proper_post_config_update(
     #{result := Result} = _Ctx
 ) ->
     {ok, Result}.
 
-apply_post_config_update(Module, #{
+apply_post_config_update(Module, Callback, 5, #{
     conf_key_path := ConfKeyPath,
     update_req := UpdateReq,
     new_conf := NewConf,
     old_conf := OldConf,
-    app_envs := AppEnvs,
+    app_envs := AppEnvs
-    callback := Callback
 }) ->
     Module:Callback(
         ConfKeyPath,
@@ -589,7 +655,25 @@ apply_post_config_update(Module, #{
         NewConf,
         OldConf,
         AppEnvs
-    ).
+    );
+apply_post_config_update(Module, Callback, 6, #{
+    conf_key_path := ConfKeyPath,
+    update_req := UpdateReq,
+    cluster_rpc_opts := ClusterRpcOpts,
+    new_conf := NewConf,
+    old_conf := OldConf,
+    app_envs := AppEnvs
+}) ->
+    Module:Callback(
+        ConfKeyPath,
+        UpdateReq,
+        NewConf,
+        OldConf,
+        AppEnvs,
+        ClusterRpcOpts
+    );
+apply_post_config_update(_Module, _Callback, false, _Ctx) ->
+    ok.
 
 propagate_post_config_updates_to_subconf(
     #{handlers := #{?WKEY := _}} = Ctx
@@ -768,7 +852,9 @@ assert_callback_function(Mod) ->
     _ = apply(Mod, module_info, []),
     case
         erlang:function_exported(Mod, pre_config_update, 3) orelse
-            erlang:function_exported(Mod, post_config_update, 5)
+            erlang:function_exported(Mod, post_config_update, 5) orelse
+            erlang:function_exported(Mod, pre_config_update, 4) orelse
+            erlang:function_exported(Mod, post_config_update, 6)
     of
         true -> ok;
         false -> error(#{msg => "bad_emqx_config_handler_callback", module => Mod})
@@ -811,3 +897,13 @@ load_prev_handlers() ->
 
 save_handlers(Handlers) ->
     application:set_env(emqx, ?MODULE, Handlers).
+
+get_function_arity(_Module, _Callback, []) ->
+    false;
+get_function_arity(Module, Callback, [Arity | Opts]) ->
+    %% ensure module is loaded
+    Module = Module:module_info(module),
+    case erlang:function_exported(Module, Callback, Arity) of
+        true -> Arity;
+        false -> get_function_arity(Module, Callback, Opts)
+    end.

apps/emqx/src/emqx_connection.erl

@@ -305,11 +305,13 @@ init_state(
     {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]),
     {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]),
     Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]),
+    PeerSNI = Transport:ensure_ok_or_exit(peersni, [Socket]),
     ConnInfo = #{
         socktype => Transport:type(Socket),
         peername => Peername,
         sockname => Sockname,
         peercert => Peercert,
+        peersni => PeerSNI,
         conn_mod => ?MODULE
     },
 

apps/emqx/src/emqx_logger_jsonfmt.erl

@@ -55,7 +55,8 @@
     depth => pos_integer() | unlimited,
     report_cb => logger:report_cb(),
     single_line => boolean(),
-    chars_limit => unlimited | pos_integer()
+    chars_limit => unlimited | pos_integer(),
+    payload_encode => text | hidden | hex
 }.
 
 -define(IS_STRING(String), (is_list(String) orelse is_binary(String))).
@@ -103,7 +104,8 @@ format(Msg, Meta, Config) ->
 
 maybe_format_msg(undefined, _Meta, _Config) ->
     #{};
-maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) ->
+maybe_format_msg({report, Report0} = Msg, #{report_cb := Cb} = Meta, Config) ->
+    Report = emqx_logger_textfmt:try_encode_payload(Report0, Config),
     case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of
         true ->
             %% reporting a map without a customised format function

apps/emqx/src/emqx_logger_textfmt.erl

@@ -20,11 +20,12 @@
 
 -export([format/2]).
 -export([check_config/1]).
--export([try_format_unicode/1]).
+-export([try_format_unicode/1, try_encode_payload/2]).
 %% Used in the other log formatters
 -export([evaluate_lazy_values_if_dbg_level/1, evaluate_lazy_values/1]).
 
-check_config(X) -> logger_formatter:check_config(maps:without([timestamp_format, with_mfa], X)).
+check_config(X) ->
+    logger_formatter:check_config(maps:without([timestamp_format, with_mfa, payload_encode], X)).
 
 %% Principle here is to delegate the formatting to logger_formatter:format/2
 %% as much as possible, and only enrich the report with clientid, peername, topic, username
@@ -107,9 +108,10 @@ is_list_report_acceptable(#{report_cb := Cb}) ->
 is_list_report_acceptable(_) ->
     false.
 
-enrich_report(ReportRaw, Meta, Config) ->
+enrich_report(ReportRaw0, Meta, Config) ->
     %% clientid and peername always in emqx_conn's process metadata.
     %% topic and username can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2
+    ReportRaw = try_encode_payload(ReportRaw0, Config),
     Topic =
         case maps:get(topic, Meta, undefined) of
             undefined -> maps:get(topic, ReportRaw, undefined);
@@ -177,3 +179,15 @@ enrich_topic({Fmt, Args}, #{topic := Topic}) when is_list(Fmt) ->
     {" topic: ~ts" ++ Fmt, [Topic | Args]};
 enrich_topic(Msg, _) ->
     Msg.
+
+try_encode_payload(#{payload := Payload} = Report, #{payload_encode := Encode}) ->
+    Report#{payload := encode_payload(Payload, Encode)};
+try_encode_payload(Report, _Config) ->
+    Report.
+
+encode_payload(Payload, text) ->
+    Payload;
+encode_payload(_Payload, hidden) ->
+    "******";
+encode_payload(Payload, hex) ->
+    binary:encode_hex(Payload).

apps/emqx/src/emqx_quic_stream.erl

@@ -39,7 +39,8 @@
     getopts/2,
     peername/1,
     sockname/1,
-    peercert/1
+    peercert/1,
+    peersni/1
 ]).
 -include_lib("quicer/include/quicer.hrl").
 -include_lib("emqx/include/emqx_quic.hrl").
@@ -106,6 +107,10 @@ peercert(_S) ->
     %% @todo but unsupported by msquic
     nossl.
 
+peersni(_S) ->
+    %% @todo
+    undefined.
+
 getstat({quic, Conn, _Stream, _Info}, Stats) ->
     case quicer:getstat(Conn, Stats) of
         {error, _} -> {error, closed};

apps/emqx/src/emqx_ws_connection.erl

@@ -280,7 +280,7 @@ websocket_init([Req, Opts]) ->
     #{zone := Zone, limiter := LimiterCfg, listener := {Type, Listener} = ListenerCfg} = Opts,
     case check_max_connection(Type, Listener) of
         allow ->
-            {Peername, PeerCert} = get_peer_info(Type, Listener, Req, Opts),
+            {Peername, PeerCert, PeerSNI} = get_peer_info(Type, Listener, Req, Opts),
             Sockname = cowboy_req:sock(Req),
             WsCookie = get_ws_cookie(Req),
             ConnInfo = #{
@@ -288,6 +288,7 @@ websocket_init([Req, Opts]) ->
                 peername => Peername,
                 sockname => Sockname,
                 peercert => PeerCert,
+                peersni => PeerSNI,
                 ws_cookie => WsCookie,
                 conn_mod => ?MODULE
             },
@@ -376,11 +377,12 @@ get_ws_cookie(Req) ->
     end.
 
 get_peer_info(Type, Listener, Req, Opts) ->
+    Host = maps:get(host, Req, undefined),
     case
         emqx_config:get_listener_conf(Type, Listener, [proxy_protocol]) andalso
             maps:get(proxy_header, Req)
     of
-        #{src_address := SrcAddr, src_port := SrcPort, ssl := SSL} ->
+        #{src_address := SrcAddr, src_port := SrcPort, ssl := SSL} = ProxyInfo ->
             SourceName = {SrcAddr, SrcPort},
             %% Notice: CN is only available in Proxy Protocol V2 additional info.
             %% `CN` is unsupported in Proxy Protocol V1
@@ -392,12 +394,14 @@ get_peer_info(Type, Listener, Req, Opts) ->
                     undefined -> undefined;
                     CN -> [{pp2_ssl_cn, CN}]
                 end,
-            {SourceName, SourceSSL};
+            PeerSNI = maps:get(authority, ProxyInfo, Host),
-        #{src_address := SrcAddr, src_port := SrcPort} ->
+            {SourceName, SourceSSL, PeerSNI};
+        #{src_address := SrcAddr, src_port := SrcPort} = ProxyInfo ->
+            PeerSNI = maps:get(authority, ProxyInfo, Host),
             SourceName = {SrcAddr, SrcPort},
-            {SourceName, nossl};
+            {SourceName, nossl, PeerSNI};
         _ ->
-            {get_peer(Req, Opts), cowboy_req:cert(Req)}
+            {get_peer(Req, Opts), cowboy_req:cert(Req), Host}
     end.
 
 websocket_handle({binary, Data}, State) when is_list(Data) ->

apps/emqx/test/emqx_authz_cache_SUITE.erl

@@ -37,7 +37,7 @@ end_per_suite(Config) ->
 %%--------------------------------------------------------------------
 
 t_cache_exclude(_) ->
-    ClientId = <<"test-id1">>,
+    ClientId = atom_to_binary(?FUNCTION_NAME),
     {ok, Client} = emqtt:start_link([{clientid, ClientId}]),
     {ok, _} = emqtt:connect(Client),
     {ok, _, _} = emqtt:subscribe(Client, <<"nocache/+/#">>, 0),
@@ -47,11 +47,12 @@ t_cache_exclude(_) ->
     emqtt:stop(Client).
 
 t_clean_authz_cache(_) ->
-    {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}]),
+    ClientId = atom_to_binary(?FUNCTION_NAME),
+    {ok, Client} = emqtt:start_link([{clientid, ClientId}]),
     {ok, _} = emqtt:connect(Client),
     {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
     emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0),
-    ClientPid = find_client_pid(<<"emqx_c">>),
+    ClientPid = find_client_pid(ClientId),
     Caches = list_cache(ClientPid),
     ct:log("authz caches: ~p", [Caches]),
     ?assert(length(Caches) > 0),
@@ -60,11 +61,12 @@ t_clean_authz_cache(_) ->
     emqtt:stop(Client).
 
 t_drain_authz_cache(_) ->
-    {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}]),
+    ClientId = atom_to_binary(?FUNCTION_NAME),
+    {ok, Client} = emqtt:start_link([{clientid, ClientId}]),
     {ok, _} = emqtt:connect(Client),
     {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
     emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0),
-    ClientPid = find_client_pid(<<"emqx_c">>),
+    ClientPid = find_client_pid(ClientId),
     Caches = list_cache(ClientPid),
     ct:log("authz caches: ~p", [Caches]),
     ?assert(length(Caches) > 0),

apps/emqx/test/emqx_bpapi_SUITE.erl

@@ -21,7 +21,7 @@
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("stdlib/include/assert.hrl").
--include_lib("emqx/src/bpapi/emqx_bpapi.hrl").
+-include("../src/bpapi/emqx_bpapi.hrl").
 
 all() -> emqx_common_test_helpers:all(?MODULE).
 

apps/emqx/test/emqx_common_test_helpers.erl

@@ -248,6 +248,7 @@ render_and_load_app_config(App, Opts) ->
             %% turn throw into error
             error({Conf, E, St})
     end.
+
 do_render_app_config(App, Schema, ConfigFile, Opts) ->
     %% copy acl_conf must run before read_schema_configs
     copy_acl_conf(),

apps/emqx/test/emqx_config_handler_SUITE.erl

@@ -415,14 +415,7 @@ assert_update_result(FailedPath, Update, Expect) ->
 
 assert_update_result(Paths, UpdatePath, Update, Expect) ->
     with_update_result(Paths, UpdatePath, Update, fun(Old, Result) ->
-        case Expect of
+        ?assertEqual(Expect, Result),
-            {error, {post_config_update, ?MODULE, post_config_update_error}} ->
-                ?assertMatch(
-                    {error, {post_config_update, ?MODULE, {post_config_update_error, _}}}, Result
-                );
-            _ ->
-                ?assertEqual(Expect, Result)
-        end,
         New = emqx:get_raw_config(UpdatePath, undefined),
         ?assertEqual(Old, New)
     end).

apps/emqx/test/emqx_connection_SUITE.erl

@@ -84,7 +84,8 @@ init_per_testcase(TestCase, Config) when
         fun
             (peername, [sock]) -> {ok, {{127, 0, 0, 1}, 3456}};
             (sockname, [sock]) -> {ok, {{127, 0, 0, 1}, 1883}};
-            (peercert, [sock]) -> undefined
+            (peercert, [sock]) -> undefined;
+            (peersni, [sock]) -> undefined
         end
     ),
     ok = meck:expect(emqx_transport, setopts, fun(_Sock, _Opts) -> ok end),

apps/emqx/test/emqx_cth_cluster.erl

@@ -391,7 +391,14 @@ node_init(#{name := Node, work_dir := WorkDir}) ->
     _ = share_load_module(Node, cthr),
     %% Enable snabbkaffe trace forwarding
     ok = snabbkaffe:forward_trace(Node),
-    when_cover_enabled(fun() -> {ok, _} = cover:start([Node]) end),
+    when_cover_enabled(fun() ->
+        case cover:start([Node]) of
+            {ok, _} ->
+                ok;
+            {error, {already_started, _}} ->
+                ok
+        end
+    end),
     ok.
 
 %% Returns 'true' if this node should appear in running nodes list.
@@ -456,7 +463,7 @@ stop(Nodes) ->
 
 stop_node(Name) ->
     Node = node_name(Name),
-    when_cover_enabled(fun() -> cover:flush([Node]) end),
+    when_cover_enabled(fun() -> ok = cover:flush([Node]) end),
     ok = emqx_cth_peer:stop(Node).
 
 %% Ports

apps/emqx/test/emqx_cth_listener.erl

@@ -0,0 +1,145 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_cth_listener).
+
+-include_lib("esockd/include/esockd.hrl").
+
+-export([
+    reload_listener_with_ppv2/1,
+    reload_listener_with_ppv2/2,
+    reload_listener_without_ppv2/1
+]).
+
+-export([meck_recv_ppv2/1, clear_meck_recv_ppv2/1]).
+
+-define(DEFAULT_OPTS, #{
+    host => "127.0.0.1",
+    proto_ver => v5,
+    connect_timeout => 5,
+    ssl => false
+}).
+
+%%--------------------------------------------------------------------
+%% APIs
+%%--------------------------------------------------------------------
+
+reload_listener_with_ppv2(Path = [listeners, _Type, _Name]) ->
+    reload_listener_with_ppv2(Path, <<>>).
+
+reload_listener_with_ppv2(Path = [listeners, Type, Name], DefaultSni) when
+    Type == tcp; Type == ws
+->
+    Cfg = emqx_config:get(Path),
+    ok = emqx_config:put(Path, Cfg#{proxy_protocol => true}),
+    ok = emqx_listeners:restart_listener(
+        emqx_listeners:listener_id(Type, Name)
+    ),
+    ok = meck_recv_ppv2(Type),
+    client_conn_fn(Type, maps:get(bind, Cfg), DefaultSni).
+
+client_conn_fn(tcp, Bind, Sni) ->
+    client_conn_fn_gen(connect, ?DEFAULT_OPTS#{port => bind2port(Bind), sni => Sni});
+client_conn_fn(ws, Bind, Sni) ->
+    client_conn_fn_gen(ws_connect, ?DEFAULT_OPTS#{port => bind2port(Bind), sni => Sni}).
+
+bind2port({_, Port}) -> Port;
+bind2port(Port) when is_integer(Port) -> Port.
+
+client_conn_fn_gen(Connect, Opts0) ->
+    fun(ClientId, Opts1) ->
+        Opts2 = maps:merge(Opts0, Opts1#{clientid => ClientId}),
+        Sni = maps:get(sni, Opts2, undefined),
+        NOpts = prepare_sni_for_meck(Sni, Opts2),
+        {ok, C} = emqtt:start_link(NOpts),
+        case emqtt:Connect(C) of
+            {ok, _} -> {ok, C};
+            {error, _} = Err -> Err
+        end
+    end.
+
+prepare_sni_for_meck(ClientSni, Opts) when is_binary(ClientSni) ->
+    ServerSni =
+        case ClientSni of
+            disable -> undefined;
+            _ -> ClientSni
+        end,
+    persistent_term:put(current_client_sni, ServerSni),
+    case maps:get(ssl, Opts, false) of
+        false ->
+            Opts;
+        true ->
+            SslOpts = maps:get(ssl_opts, Opts, #{}),
+            Opts#{ssl_opts => [{server_name_indication, ClientSni} | SslOpts]}
+    end.
+
+reload_listener_without_ppv2(Path = [listeners, Type, Name]) when
+    Type == tcp; Type == ws
+->
+    Cfg = emqx_config:get(Path),
+    ok = emqx_config:put(Path, Cfg#{proxy_protocol => false}),
+    ok = emqx_listeners:restart_listener(
+        emqx_listeners:listener_id(Type, Name)
+    ),
+    ok = clear_meck_recv_ppv2(Type).
+
+meck_recv_ppv2(tcp) ->
+    ok = meck:new(esockd_proxy_protocol, [passthrough, no_history, no_link]),
+    ok = meck:expect(
+        esockd_proxy_protocol,
+        recv,
+        fun(_Transport, Socket, _Timeout) ->
+            SNI = persistent_term:get(current_client_sni, undefined),
+            {ok, {SrcAddr, SrcPort}} = esockd_transport:peername(Socket),
+            {ok, {DstAddr, DstPort}} = esockd_transport:sockname(Socket),
+            {ok, #proxy_socket{
+                inet = inet4,
+                socket = Socket,
+                src_addr = SrcAddr,
+                dst_addr = DstAddr,
+                src_port = SrcPort,
+                dst_port = DstPort,
+                pp2_additional_info = [{pp2_authority, SNI}]
+            }}
+        end
+    );
+meck_recv_ppv2(ws) ->
+    ok = meck:new(ranch_tcp, [passthrough, no_history, no_link]),
+    ok = meck:expect(
+        ranch_tcp,
+        recv_proxy_header,
+        fun(Socket, _Timeout) ->
+            SNI = persistent_term:get(current_client_sni, undefined),
+            {ok, {SrcAddr, SrcPort}} = esockd_transport:peername(Socket),
+            {ok, {DstAddr, DstPort}} = esockd_transport:sockname(Socket),
+            {ok, #{
+                authority => SNI,
+                command => proxy,
+                dest_address => DstAddr,
+                dest_port => DstPort,
+                src_address => SrcAddr,
+                src_port => SrcPort,
+                transport_family => ipv4,
+                transport_protocol => stream,
+                version => 2
+            }}
+        end
+    ).
+
+clear_meck_recv_ppv2(tcp) ->
+    ok = meck:unload(esockd_proxy_protocol);
+clear_meck_recv_ppv2(ws) ->
+    ok = meck:unload(ranch_tcp).

apps/emqx/test/emqx_cth_suite.erl

@@ -71,6 +71,7 @@
 -export([start_app/3]).
 -export([stop_apps/1]).
 
+-export([default_config/2]).
 -export([merge_appspec/2]).
 -export([merge_config/2]).
 
@@ -243,6 +244,7 @@ log_appspec(App, #{}) ->
 
 spec_fmt(fc, config) -> "~n~ts";
 spec_fmt(fc, _) -> "~p";
+spec_fmt(ffun, {config, false}) -> "false (don't inhibit config loader)";
 spec_fmt(ffun, {config, C}) -> render_config(C);
 spec_fmt(ffun, {_, X}) -> X.
 
@@ -349,6 +351,7 @@ default_appspec(emqx_conf, SuiteOpts) ->
             data_dir => unicode:characters_to_binary(maps:get(work_dir, SuiteOpts, "data"))
         }
     },
+    SharedApps = maps:get(emqx_conf_shared_apps, SuiteOpts, [emqx, emqx_auth]),
     % NOTE
     % Since `emqx_conf_schema` manages config for a lot of applications, it's good to include
     % their defaults as well.
@@ -357,10 +360,7 @@ default_appspec(emqx_conf, SuiteOpts) ->
             emqx_utils_maps:deep_merge(Acc, default_config(App, SuiteOpts))
         end,
         Config,
-        [
+        SharedApps
-            emqx,
-            emqx_auth
-        ]
     ),
     #{
         config => SharedConfig,

apps/emqx/test/emqx_listeners_SUITE.erl

@@ -593,7 +593,7 @@ t_quic_update_opts_fail(Config) ->
 
         %% THEN: Reload failed but old listener is rollbacked.
         ?assertMatch(
-            {error, {post_config_update, emqx_listeners, {{rollbacked, {error, tls_error}}, _}}},
+            {error, {post_config_update, emqx_listeners, {rollbacked, {error, tls_error}}}},
             UpdateResult1
         ),
 

apps/emqx/test/emqx_logger_fmt_SUITE.erl

@@ -36,12 +36,18 @@ t_text_fmt_lazy_values(_) ->
 t_text_fmt_lazy_values_only_in_debug_level_events(_) ->
     check_fmt_lazy_values_only_in_debug_level_events(emqx_logger_textfmt).
 
+t_text_payload(_) ->
+    check_fmt_payload(emqx_logger_textfmt).
+
 t_json_fmt_lazy_values(_) ->
     check_fmt_lazy_values(emqx_logger_jsonfmt).
 
 t_json_fmt_lazy_values_only_in_debug_level_events(_) ->
     check_fmt_lazy_values_only_in_debug_level_events(emqx_logger_jsonfmt).
 
+t_json_payload(_) ->
+    check_fmt_payload(emqx_logger_jsonfmt).
+
 check_fmt_lazy_values(FormatModule) ->
     LogEntryIOData = FormatModule:format(event_with_lazy_value(), conf()),
     LogEntryBin = unicode:characters_to_binary(LogEntryIOData),
@@ -62,6 +68,18 @@ check_fmt_lazy_values_only_in_debug_level_events(FormatModule) ->
     ?assertNotEqual(nomatch, binary:match(LogEntryBin, [<<"emqx_trace_format_func_data">>])),
     ok.
 
+check_fmt_payload(FormatModule) ->
+    %% For performace reason we only search for lazy values to evaluate if log level is debug
+    WarningEvent = (event_with_lazy_value())#{level => info},
+    Conf = conf(),
+    LogEntryIOData = FormatModule:format(WarningEvent, Conf#{payload_encode => hidden}),
+    LogEntryBin = unicode:characters_to_binary(LogEntryIOData),
+    %% The input data for the formatting should exist
+    ?assertEqual(nomatch, binary:match(LogEntryBin, [<<"content">>])),
+    %% The lazy value should not have been evaluated
+    ?assertNotEqual(nomatch, binary:match(LogEntryBin, [<<"******">>])),
+    ok.
+
 conf() ->
     #{
         time_offset => [],
@@ -84,7 +102,8 @@ event_with_lazy_value() ->
             {report, #{
                 reason =>
                     #emqx_trace_format_func_data{function = fun(Data) -> Data end, data = hej},
-                msg => hej
+                msg => hej,
+                payload => <<"content">>
             }},
         level => debug
     }.

apps/emqx/test/emqx_peersni_SUITE.erl

@@ -0,0 +1,169 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_peersni_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-include_lib("esockd/include/esockd.hrl").
+
+-define(SERVER_NAME, <<"localhost">>).
+
+%%--------------------------------------------------------------------
+%% setups
+%%--------------------------------------------------------------------
+
+all() ->
+    [
+        {group, tcp_ppv2},
+        {group, ws_ppv2},
+        {group, ssl},
+        {group, wss}
+    ].
+
+groups() ->
+    TCs = emqx_common_test_helpers:all(?MODULE),
+    [
+        {tcp_ppv2, [], TCs},
+        {ws_ppv2, [], TCs},
+        {ssl, [], TCs},
+        {wss, [], TCs}
+    ].
+
+init_per_suite(Config) ->
+    Apps = emqx_cth_suite:start(
+        [{emqx, #{}}],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{apps, Apps} | Config].
+
+end_per_suite(Config) ->
+    emqx_cth_suite:stop(proplists:get_value(apps, Config)).
+
+init_per_group(tcp_ppv2, Config) ->
+    ClientFn = emqx_cth_listener:reload_listener_with_ppv2(
+        [listeners, tcp, default],
+        ?SERVER_NAME
+    ),
+    [{client_fn, ClientFn} | Config];
+init_per_group(ws_ppv2, Config) ->
+    ClientFn = emqx_cth_listener:reload_listener_with_ppv2(
+        [listeners, ws, default],
+        ?SERVER_NAME
+    ),
+    [{client_fn, ClientFn} | Config];
+init_per_group(ssl, Config) ->
+    ClientFn = fun(ClientId, Opts) ->
+        Opts1 = Opts#{
+            host => ?SERVER_NAME,
+            port => 8883,
+            ssl => true,
+            ssl_opts => [
+                {verify, verify_none},
+                {server_name_indication, binary_to_list(?SERVER_NAME)}
+            ]
+        },
+        {ok, C} = emqtt:start_link(Opts1#{clientid => ClientId}),
+        case emqtt:connect(C) of
+            {ok, _} -> {ok, C};
+            {error, _} = Err -> Err
+        end
+    end,
+    [{client_fn, ClientFn} | Config];
+init_per_group(wss, Config) ->
+    ClientFn = fun(ClientId, Opts) ->
+        Opts1 = Opts#{
+            host => ?SERVER_NAME,
+            port => 8084,
+            ws_transport_options => [
+                {transport, tls},
+                {protocols, [http]},
+                {transport_opts, [
+                    {verify, verify_none},
+                    {server_name_indication, binary_to_list(?SERVER_NAME)},
+                    {customize_hostname_check, []}
+                ]}
+            ]
+        },
+        {ok, C} = emqtt:start_link(Opts1#{clientid => ClientId}),
+        case emqtt:ws_connect(C) of
+            {ok, _} -> {ok, C};
+            {error, _} = Err -> Err
+        end
+    end,
+    [{client_fn, ClientFn} | Config];
+init_per_group(_, Config) ->
+    Config.
+
+end_per_group(tcp_ppv2, _Config) ->
+    emqx_cth_listener:reload_listener_without_ppv2([listeners, tcp, default]);
+end_per_group(ws_ppv2, _Config) ->
+    emqx_cth_listener:reload_listener_without_ppv2([listeners, ws, default]);
+end_per_group(_, _Config) ->
+    ok.
+
+init_per_testcase(TestCase, Config) ->
+    case erlang:function_exported(?MODULE, TestCase, 2) of
+        true -> ?MODULE:TestCase(init, Config);
+        _ -> Config
+    end.
+
+end_per_testcase(TestCase, Config) ->
+    case erlang:function_exported(?MODULE, TestCase, 2) of
+        true -> ?MODULE:TestCase('end', Config);
+        false -> ok
+    end,
+    Config.
+
+%%--------------------------------------------------------------------
+%% cases
+%%--------------------------------------------------------------------
+
+t_peersni_saved_into_conninfo(Config) ->
+    process_flag(trap_exit, true),
+
+    ClientId = <<"test-clientid1">>,
+    ClientFn = proplists:get_value(client_fn, Config),
+
+    {ok, Client} = ClientFn(ClientId, _Opts = #{}),
+    ?assertMatch(#{clientinfo := #{peersni := ?SERVER_NAME}}, emqx_cm:get_chan_info(ClientId)),
+
+    ok = emqtt:disconnect(Client).
+
+t_parse_peersni_to_client_attr(Config) ->
+    process_flag(trap_exit, true),
+
+    %% set the peersni to the client attribute
+    {ok, Variform} = emqx_variform:compile("nth(1, tokens(peersni, 'h'))"),
+    emqx_config:put([mqtt, client_attrs_init], [
+        #{expression => Variform, set_as_attr => mnts}
+    ]),
+
+    ClientId = <<"test-clientid2">>,
+    ClientFn = proplists:get_value(client_fn, Config),
+    {ok, Client} = ClientFn(ClientId, _Opts = #{}),
+
+    ?assertMatch(
+        #{clientinfo := #{client_attrs := #{mnts := <<"local">>}}}, emqx_cm:get_chan_info(ClientId)
+    ),
+
+    ok = emqtt:disconnect(Client).

apps/emqx/test/emqx_persistent_session_ds_router_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(export_all).
 -compile(nowarn_export_all).
 
--include_lib("emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl").
+-include("../src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 

apps/emqx/test/emqx_ws_connection_SUITE.erl

@@ -169,7 +169,7 @@ t_header(_) ->
     set_ws_opts(proxy_address_header, <<"x-forwarded-for">>),
     set_ws_opts(proxy_port_header, <<"x-forwarded-port">>),
     {ok, St, _} = ?ws_conn:websocket_init([
-        req,
+        #{},
         #{
             zone => default,
             limiter => limiter_cfg(),
@@ -573,7 +573,7 @@ t_shutdown(_) ->
 st() -> st(#{}).
 st(InitFields) when is_map(InitFields) ->
     {ok, St, _} = ?ws_conn:websocket_init([
-        req,
+        #{},
         #{
             zone => default,
             listener => {ws, default},

apps/emqx_audit/mix.exs

@@ -0,0 +1,27 @@
+defmodule EMQXAudit.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_audit,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_audit_app, []}]
+  end
+
+  def deps() do
+    [{:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}]
+  end
+end

apps/emqx_audit/test/emqx_audit_api_SUITE.erl

@@ -18,6 +18,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
 
 all() ->
     [
@@ -54,18 +55,27 @@ common_tests() ->
 }).
 
 init_per_suite(Config) ->
-    _ = application:load(emqx_conf),
+    Apps = emqx_cth_suite:start(
-    emqx_config:erase_all(),
+        [
-    emqx_mgmt_api_test_util:init_suite([emqx_ctl, emqx_conf, emqx_audit]),
+            emqx_ctl,
-    ok = emqx_common_test_helpers:load_config(emqx_enterprise_schema, ?CONF_DEFAULT),
+            emqx,
-    emqx_config:save_schema_mod_and_names(emqx_enterprise_schema),
+            {emqx_conf, #{
-    ok = emqx_config_logger:refresh_config(),
+                config => ?CONF_DEFAULT,
-    application:set_env(emqx, boot_modules, []),
+                schema_mod => emqx_enterprise_schema
-    emqx_conf_cli:load(),
+            }},
-    Config.
+            emqx_modules,
+            emqx_audit,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{apps, Apps} | Config].
 
-end_per_suite(_) ->
+end_per_suite(Config) ->
-    emqx_mgmt_api_test_util:end_suite([emqx_audit, emqx_conf, emqx_ctl]).
+    Apps = ?config(apps, Config),
+    ok = emqx_cth_suite:stop(Apps),
+    ok.
 
 t_http_api(_) ->
     process_flag(trap_exit, true),
@@ -164,6 +174,7 @@ t_cli(_Config) ->
         ],
         Data
     ),
+    [ShowLogEntry] = Data,
     %% check create at is valid
     [#{<<"created_at">> := CreateAtRaw}] = Data,
     CreateAt = calendar:rfc3339_to_system_time(binary_to_list(CreateAtRaw), [{unit, microsecond}]),
@@ -172,7 +183,10 @@ t_cli(_Config) ->
     %% check cli filter
     {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader),
     #{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]),
-    ?assertEqual(Data, Data1),
+    ?assertMatch(
+        [ShowLogEntry, #{<<"operation_type">> := <<"emqx">>, <<"args">> := [<<"start">>]}],
+        Data1
+    ),
     {ok, Res2} = emqx_mgmt_api_test_util:request_api(
         get, AuditPath, "from=erlang_console", AuthHeader
     ),

apps/emqx_auth/etc/auth-built-in-db-bootstrap.csv

@@ -0,0 +1 @@
+user_id,password,is_superuser

apps/emqx_auth/mix.exs

@@ -0,0 +1,44 @@
+defmodule EMQXAuth.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth,
+      version: "0.1.0",
+      build_path: "../../_build",
+      compilers: Mix.compilers() ++ [:copy_srcs],
+      # used by our `Mix.Tasks.Compile.CopySrcs` compiler
+      extra_dirs: extra_dirs(),
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_auth_app, []}]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_utils, in_umbrella: true}
+    ]
+  end
+
+  defp extra_dirs() do
+    dirs = ["etc"]
+    if UMP.test_env?() do
+      ["test" | dirs]
+    else
+      dirs
+    end
+  end
+end

apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl

@@ -142,7 +142,7 @@ salt_position(desc) -> "Salt position for PLAIN, MD5, SHA, SHA256 and SHA512 alg
 salt_position(_) -> undefined.
 
 dk_length(type) ->
-    integer();
+    pos_integer();
 dk_length(required) ->
     false;
 dk_length(desc) ->

apps/emqx_auth/src/emqx_authz/emqx_authz.erl

@@ -51,7 +51,7 @@
     set_feature_available/2
 ]).
 
--export([post_config_update/5, pre_config_update/3]).
+-export([pre_config_update/4, post_config_update/5]).
 
 -export([
     maybe_read_source_files/1,
@@ -194,8 +194,8 @@ update({?CMD_DELETE, Type}, Sources) ->
 update(Cmd, Sources) ->
     emqx_authz_utils:update_config(?CONF_KEY_PATH, {Cmd, Sources}).
 
-pre_config_update(Path, Cmd, Sources) ->
+pre_config_update(Path, Cmd, Sources, ClusterRpcOpts) ->
-    try do_pre_config_update(Path, Cmd, Sources) of
+    try do_pre_config_update(Path, Cmd, Sources, ClusterRpcOpts) of
         {error, Reason} -> {error, Reason};
         NSources -> {ok, NSources}
     catch
@@ -215,58 +215,64 @@ pre_config_update(Path, Cmd, Sources) ->
             {error, Reason}
     end.
 
-do_pre_config_update(?CONF_KEY_PATH, Cmd, Sources) ->
+do_pre_config_update(?CONF_KEY_PATH, Cmd, Sources, Opts) ->
-    do_pre_config_update(Cmd, Sources);
+    do_pre_config_update(Cmd, Sources, Opts);
-do_pre_config_update(?ROOT_KEY, {?CMD_MERGE, NewConf}, OldConf) ->
+do_pre_config_update(?ROOT_KEY, {?CMD_MERGE, NewConf}, OldConf, Opts) ->
-    do_pre_config_merge(NewConf, OldConf);
+    do_pre_config_merge(NewConf, OldConf, Opts);
-do_pre_config_update(?ROOT_KEY, NewConf, OldConf) ->
+do_pre_config_update(?ROOT_KEY, NewConf, OldConf, Opts) ->
-    do_pre_config_replace(NewConf, OldConf).
+    do_pre_config_replace(NewConf, OldConf, Opts).
 
-do_pre_config_merge(NewConf, OldConf) ->
+do_pre_config_merge(NewConf, OldConf, Opts) ->
     MergeConf = emqx_utils_maps:deep_merge(OldConf, NewConf),
     NewSources = merge_sources(OldConf, NewConf),
-    do_pre_config_replace(MergeConf#{<<"sources">> => NewSources}, OldConf).
+    do_pre_config_replace(MergeConf#{<<"sources">> => NewSources}, OldConf, Opts).
 
 %% override the entire config when updating the root key
 %% emqx_conf:update(?ROOT_KEY, Conf);
-do_pre_config_replace(Conf, Conf) ->
+do_pre_config_replace(Conf, Conf, _Opts) ->
     Conf;
-do_pre_config_replace(NewConf, OldConf) ->
+do_pre_config_replace(NewConf, OldConf, Opts) ->
     NewSources = get_sources(NewConf),
     OldSources = get_sources(OldConf),
-    ReplaceSources = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources),
+    ReplaceSources = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources, Opts),
     NewConf#{<<"sources">> => ReplaceSources}.
 
-do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) ->
+do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources, _Opts) ->
     do_move(Cmd, Sources);
-do_pre_config_update({?CMD_PREPEND, Source}, Sources) ->
+do_pre_config_update({?CMD_PREPEND, Source}, Sources, _Opts) ->
     NSource = maybe_write_source_files(Source),
     NSources = [NSource] ++ Sources,
     ok = check_dup_types(NSources),
     NSources;
-do_pre_config_update({?CMD_APPEND, Source}, Sources) ->
+do_pre_config_update({?CMD_APPEND, Source}, Sources, _Opts) ->
     NSource = maybe_write_source_files(Source),
     NSources = Sources ++ [NSource],
     ok = check_dup_types(NSources),
     NSources;
-do_pre_config_update({{?CMD_REPLACE, Type}, Source}, Sources) ->
+do_pre_config_update({{?CMD_REPLACE, Type}, Source}, Sources, _Opts) ->
     NSource = maybe_write_source_files(Source),
     {_Old, Front, Rear} = take(Type, Sources),
     NSources = Front ++ [NSource | Rear],
     ok = check_dup_types(NSources),
     NSources;
-do_pre_config_update({{?CMD_DELETE, Type}, _Source}, Sources) ->
+do_pre_config_update({{?CMD_DELETE, Type}, _Source}, Sources, Opts) ->
-    {_Old, Front, Rear} = take(Type, Sources),
+    case type_take(Type, Sources) of
-    NSources = Front ++ Rear,
+        not_found ->
-    NSources;
+            case emqx_cluster_rpc:is_initiator(Opts) of
-do_pre_config_update({?CMD_REPLACE, Sources}, _OldSources) ->
+                true -> throw({not_found_source, Type});
+                false -> Sources
+            end;
+        {_Found, NSources} ->
+            NSources
+    end;
+do_pre_config_update({?CMD_REPLACE, Sources}, _OldSources, _Opts) ->
     %% overwrite the entire config!
     NSources = lists:map(fun maybe_write_source_files/1, Sources),
     ok = check_dup_types(NSources),
     NSources;
-do_pre_config_update({?CMD_REORDER, NewSourcesOrder}, OldSources) ->
+do_pre_config_update({?CMD_REORDER, NewSourcesOrder}, OldSources, _Opts) ->
     reorder_sources(NewSourcesOrder, OldSources);
-do_pre_config_update({Op, Source}, Sources) ->
+do_pre_config_update({Op, Source}, Sources, _Opts) ->
     throw({bad_request, #{op => Op, source => Source, sources => Sources}}).
 
 post_config_update(_, _, undefined, _OldSource, _AppEnvs) ->
@@ -293,9 +299,13 @@ do_post_config_update(?CONF_KEY_PATH, {{?CMD_REPLACE, Type}, RawNewSource}, Sour
     Front ++ [InitedSources] ++ Rear;
 do_post_config_update(?CONF_KEY_PATH, {{?CMD_DELETE, Type}, _RawNewSource}, _Sources) ->
     OldInitedSources = lookup(),
-    {OldSource, Front, Rear} = take(Type, OldInitedSources),
+    case type_take(Type, OldInitedSources) of
-    ok = ensure_deleted(OldSource, #{clear_metric => true}),
+        not_found ->
-    Front ++ Rear;
+            OldInitedSources;
+        {Found, NSources} ->
+            ok = ensure_deleted(Found, #{clear_metric => true}),
+            NSources
+    end;
 do_post_config_update(?CONF_KEY_PATH, {?CMD_REPLACE, _RawNewSources}, Sources) ->
     overwrite_entire_sources(Sources);
 do_post_config_update(?CONF_KEY_PATH, {?CMD_REORDER, NewSourcesOrder}, _Sources) ->
@@ -317,18 +327,32 @@ do_post_config_update(?ROOT_KEY, _Conf, NewConf) ->
 
 overwrite_entire_sources(Sources) ->
     PrevSources = lookup(),
-    NewSourcesTypes = lists:map(fun type/1, Sources),
+    #{
-    EnsureDelete = fun(S) ->
+        removed := Removed,
-        TypeName = type(S),
+        added := Added,
-        Opts =
+        identical := Identical,
-            case lists:member(TypeName, NewSourcesTypes) of
+        changed := Changed
-                true -> #{clear_metric => false};
+    } = emqx_utils:diff_lists(Sources, PrevSources, fun type/1),
-                false -> #{clear_metric => true}
+    lists:foreach(
-            end,
+        fun(S) -> ensure_deleted(S, #{clear_metric => true}) end,
-        ensure_deleted(S, Opts)
+        Removed
-    end,
+    ),
-    lists:foreach(EnsureDelete, PrevSources),
+    AddedSources = create_sources(Added),
-    create_sources(Sources).
+    ChangedSources = lists:map(
+        fun({Old, New}) ->
+            update_source(type(New), Old, New)
+        end,
+        Changed
+    ),
+    New = Identical ++ AddedSources ++ ChangedSources,
+    lists:map(
+        fun(Type) ->
+            SearchFun = fun(S) -> type(S) =:= type(Type) end,
+            {value, Val} = lists:search(SearchFun, New),
+            Val
+        end,
+        Sources
+    ).
 
 %% @doc do source move
 do_move({?CMD_MOVE, Type, ?CMD_MOVE_FRONT}, Sources) ->

apps/emqx_auth/src/emqx_authz/emqx_authz_app.erl

@@ -19,6 +19,8 @@
 %% @end
 %%%-------------------------------------------------------------------
 
+%% TODO: delete this module
+
 -module(emqx_authz_app).
 
 -behaviour(application).

apps/emqx_auth/src/emqx_authz/emqx_authz_rule_raw.erl

@@ -122,7 +122,7 @@ validate_rule_topics(RuleRaw) ->
     throw({missing_topic_or_topics, RuleRaw}).
 
 validate_rule_topic(<<"eq ", TopicRaw/binary>>) ->
-    {eq, validate_rule_topic(TopicRaw)};
+    {eq, TopicRaw};
 validate_rule_topic(TopicRaw) when is_binary(TopicRaw) -> TopicRaw.
 
 validate_rule_permission(<<"allow">>) -> allow;

apps/emqx_auth/test/data/user-credentials-plain_v2.csv

@@ -0,0 +1,3 @@
+user_id,password,is_superuser
+myuser3,Password4,true
+myuser4,Password3,false

apps/emqx_auth/test/emqx_authz/emqx_authz_api_cache_SUITE.erl

@@ -32,32 +32,30 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = emqx_mgmt_api_test_util:init_suite(
+    Apps = emqx_cth_suite:start(
-        [emqx_conf, emqx_auth],
+        [
-        fun set_special_configs/1
+            emqx_conf,
+            {emqx_auth, #{
+                config =>
+                    #{
+                        authorization =>
+                            #{
+                                cache => #{enabled => true},
+                                no_match => deny,
+                                sources => []
+                            }
+                    }
+            }},
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
-    Config.
+    [{apps, Apps} | Config].
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    {ok, _} = emqx:update_config(
+    Apps = ?config(apps, Config),
-        [authorization],
+    emqx_cth_suite:stop(Apps),
-        #{
-            <<"no_match">> => <<"allow">>,
-            <<"cache">> => #{<<"enable">> => <<"true">>},
-            <<"sources">> => []
-        }
-    ),
-    emqx_mgmt_api_test_util:end_suite([emqx_auth, emqx_conf]),
-    ok.
-
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(emqx_auth) ->
-    {ok, _} = emqx:update_config([authorization, cache, enable], true),
-    {ok, _} = emqx:update_config([authorization, no_match], deny),
-    {ok, _} = emqx:update_config([authorization, sources], []),
-    ok;
-set_special_configs(_App) ->
     ok.
 
 t_clean_cache(_) ->

apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl

@@ -0,0 +1,128 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authz_api_cluster_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-import(emqx_mgmt_api_test_util, [uri/1]).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("emqx/include/emqx_placeholder.hrl").
+
+-define(SOURCE_HTTP, #{
+    <<"type">> => <<"http">>,
+    <<"enable">> => true,
+    <<"url">> => <<"https://fake.com:443/acl?username=", ?PH_USERNAME/binary>>,
+    <<"ssl">> => #{<<"enable">> => true},
+    <<"headers">> => #{},
+    <<"method">> => <<"get">>,
+    <<"request_timeout">> => <<"5s">>
+}).
+-define(ON(NODE, BODY), erpc:call(NODE, fun() -> BODY end)).
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    WorkDir = ?config(priv_dir, Config),
+    Cluster = mk_cluster_spec(#{}),
+    Nodes = [NodePrimary | _] = emqx_cth_cluster:start(Cluster, #{work_dir => WorkDir}),
+    lists:foreach(fun(N) -> ?ON(N, emqx_authz_test_lib:register_fake_sources([http])) end, Nodes),
+    {ok, App} = ?ON(NodePrimary, emqx_common_test_http:create_default_app()),
+    [{api, App}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config].
+
+end_per_suite(Config) ->
+    ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)).
+
+t_api(Config) ->
+    APINode = ?config(node, Config),
+    {ok, 200, Result1} = request(get, uri(["authorization", "sources"]), [], Config),
+    ?assertEqual([], get_sources(Result1)),
+    {ok, 204, _} = request(post, uri(["authorization", "sources"]), ?SOURCE_HTTP, Config),
+    AllNodes = ?config(cluster_nodes, Config),
+    [OtherNode] = AllNodes -- [APINode],
+    lists:foreach(
+        fun(N) ->
+            ?ON(
+                N,
+                ?assertMatch(
+                    [#{<<"type">> := <<"http">>}],
+                    emqx:get_raw_config([authorization, sources])
+                )
+            )
+        end,
+        AllNodes
+    ),
+    %% delete the source from the second node.
+    ?ON(OtherNode, begin
+        {ok, _} = emqx:update_config([authorization], #{<<"sources">> => []}),
+        ?assertMatch([], emqx:get_raw_config([authorization, sources]))
+    end),
+    ?assertMatch(
+        {ok, 204, _},
+        request(
+            delete,
+            uri(["authorization", "sources", "http"]),
+            [],
+            Config
+        )
+    ),
+    lists:foreach(
+        fun(N) ->
+            ?ON(
+                N,
+                ?assertMatch(
+                    [],
+                    emqx:get_raw_config([authorization, sources])
+                )
+            )
+        end,
+        AllNodes
+    ),
+    ?assertMatch(
+        {ok, 404, _},
+        request(
+            delete,
+            uri(["authorization", "sources", "http"]),
+            [],
+            Config
+        )
+    ),
+    ok.
+
+get_sources(Result) ->
+    maps:get(<<"sources">>, emqx_utils_json:decode(Result, [return_maps])).
+
+mk_cluster_spec(Opts) ->
+    Apps = [
+        emqx,
+        {emqx_conf, "authorization {cache{enable=false},no_match=deny,sources=[]}"},
+        emqx_auth,
+        emqx_management
+    ],
+    Node1Apps = Apps ++ [{emqx_dashboard, "dashboard.listeners.http {enable=true,bind=18083}"}],
+    Node2Apps = Apps,
+    [
+        {emqx_authz_api_cluster_SUITE1, Opts#{role => core, apps => Node1Apps}},
+        {emqx_authz_api_cluster_SUITE2, Opts#{role => core, apps => Node2Apps}}
+    ].
+
+request(Method, URL, Body, Config) ->
+    AuthHeader = emqx_common_test_http:auth_header(?config(api, Config)),
+    Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]},
+    emqx_mgmt_api_test_util:request_api(Method, URL, [], AuthHeader, Body, Opts).

apps/emqx_auth/test/emqx_authz/emqx_authz_api_settings_SUITE.erl

@@ -30,33 +30,30 @@ groups() ->
     [].
 
 init_per_suite(Config) ->
-    ok = emqx_mgmt_api_test_util:init_suite(
+    Apps = emqx_cth_suite:start(
-        [emqx_conf, emqx_auth, emqx_dashboard],
+        [
-        fun set_special_configs/1
+            emqx_conf,
+            {emqx_auth, #{
+                config =>
+                    #{
+                        authorization =>
+                            #{
+                                cache => #{enabled => true},
+                                no_match => allow,
+                                sources => []
+                            }
+                    }
+            }},
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
-    Config.
+    [{apps, Apps} | Config].
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    {ok, _} = emqx:update_config(
+    Apps = ?config(apps, Config),
-        [authorization],
+    emqx_cth_suite:stop(Apps),
-        #{
-            <<"no_match">> => <<"allow">>,
-            <<"cache">> => #{<<"enable">> => <<"true">>},
-            <<"sources">> => []
-        }
-    ),
-    ok = stop_apps([emqx_resource]),
-    emqx_mgmt_api_test_util:end_suite([emqx_auth, emqx_conf]),
-    ok.
-
-set_special_configs(emqx_dashboard) ->
-    emqx_dashboard_api_test_helpers:set_default_config();
-set_special_configs(emqx_auth) ->
-    {ok, _} = emqx:update_config([authorization, cache, enable], false),
-    {ok, _} = emqx:update_config([authorization, no_match], deny),
-    {ok, _} = emqx:update_config([authorization, sources], []),
-    ok;
-set_special_configs(_App) ->
     ok.
 
 %%------------------------------------------------------------------------------

apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl

@@ -34,22 +34,30 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    ok = emqx_common_test_helpers:start_apps(
+    Apps = emqx_cth_suite:start(
-        [emqx_conf, emqx_auth],
+        [
-        fun set_special_configs/1
+            emqx_conf,
+            {emqx_auth, #{
+                config =>
+                    #{
+                        authorization =>
+                            #{
+                                cache => #{enabled => false},
+                                no_match => deny,
+                                sources => []
+                            }
+                    }
+            }},
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
-    Config.
+    [{apps, Apps} | Config].
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    {ok, _} = emqx:update_config(
+    Apps = ?config(apps, Config),
-        [authorization],
+    emqx_cth_suite:stop(Apps),
-        #{
-            <<"no_match">> => <<"allow">>,
-            <<"cache">> => #{<<"enable">> => <<"true">>},
-            <<"sources">> => []
-        }
-    ),
-    emqx_common_test_helpers:stop_apps([emqx_auth, emqx_conf]),
     ok.
 
 init_per_testcase(_TestCase, Config) ->
@@ -58,14 +66,6 @@ end_per_testcase(_TestCase, _Config) ->
     _ = emqx_authz:set_feature_available(rich_actions, true),
     ok.
 
-set_special_configs(emqx_auth) ->
-    {ok, _} = emqx:update_config([authorization, cache, enable], false),
-    {ok, _} = emqx:update_config([authorization, no_match], deny),
-    {ok, _} = emqx:update_config([authorization, sources], []),
-    ok;
-set_special_configs(_App) ->
-    ok.
-
 t_compile(_) ->
     % NOTE
     % Some of the following testcase are relying on the internal representation of

apps/emqx_auth_ext/mix.exs

@@ -0,0 +1,27 @@
+defmodule EMQXAuthExt.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth_ext,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [{:emqx, in_umbrella: true}]
+  end
+end

apps/emqx_auth_http/mix.exs

@@ -0,0 +1,35 @@
+defmodule EMQXAuthHTTP.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth_http,
+      version: "0.1.0",
+      build_path: "../../_build",
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_auth_http_app, []}]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_auth, in_umbrella: true},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_connector, in_umbrella: true},
+      {:hocon, github: "emqx/hocon", tag: "0.42.2", override: true}
+    ]
+  end
+end

apps/emqx_auth_jwt/mix.exs

@@ -0,0 +1,34 @@
+defmodule EMQXAuthJWT.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth_jwt,
+      version: "0.1.0",
+      build_path: "../../_build",
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_auth_jwt_app, []}]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_auth, in_umbrella: true},
+      {:emqx_resource, in_umbrella: true},
+      {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}
+    ]
+  end
+end

apps/emqx_auth_ldap/mix.exs

@@ -0,0 +1,33 @@
+defmodule EMQXAuthLDAP.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth_ldap,
+      version: "0.1.0",
+      build_path: "../../_build",
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: [:eldap], mod: {:emqx_auth_ldap_app, []}]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_auth, in_umbrella: true},
+      {:emqx_ldap, in_umbrella: true},
+    ]
+  end
+end

apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl

@@ -33,25 +33,27 @@ groups() ->
     emqx_authz_test_lib:table_groups(t_run_case, cases()).
 
 init_per_suite(Config) ->
-    ok = stop_apps([emqx_resource]),
     case emqx_common_test_helpers:is_tcp_server_available(?LDAP_HOST, ?LDAP_DEFAULT_PORT) of
         true ->
-            ok = emqx_common_test_helpers:start_apps(
+            Apps = emqx_cth_suite:start(
-                [emqx_conf, emqx_auth, emqx_auth_ldap],
+                [
-                fun set_special_configs/1
+                    emqx,
+                    emqx_conf,
+                    emqx_auth,
+                    emqx_auth_ldap
+                ],
+                #{work_dir => emqx_cth_suite:work_dir(Config)}
             ),
-            ok = start_apps([emqx_resource]),
             ok = create_ldap_resource(),
-            Config;
+            [{apps, Apps} | Config];
         false ->
             {skip, no_ldap}
     end.
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    ok = emqx_authz_test_lib:restore_authorizers(),
+    Apps = ?config(apps, Config),
-    ok = emqx_resource:remove_local(?LDAP_RESOURCE),
+    emqx_cth_suite:stop(Apps),
-    ok = stop_apps([emqx_resource]),
+    ok.
-    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_auth, emqx_auth_ldap]).
 
 init_per_group(Group, Config) ->
     [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config].

apps/emqx_auth_mnesia/mix.exs

@@ -0,0 +1,29 @@
+defmodule EMQXAuthMnesia.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth_mnesia,
+      version: "0.1.0",
+      build_path: "../../_build",
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_auth_mnesia_app, []}]
+  end
+
+  def deps() do
+    [{:emqx, in_umbrella: true}, {:emqx_auth, in_umbrella: true}]
+  end
+end

apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl

@@ -171,67 +171,61 @@ do_destroy(UserGroup) ->
         mnesia:select(?TAB, group_match_spec(UserGroup), write)
     ).
 
-import_users({PasswordType, Filename, FileData}, State) ->
+import_users(ImportSource, State) ->
+    import_users(ImportSource, State, #{override => true}).
+
+import_users({PasswordType, Filename, FileData}, State, Opts) ->
     Convertor = convertor(PasswordType, State),
-    try
+    try parse_import_users(Filename, FileData, Convertor) of
-        {_NewUsersCnt, Users} = parse_import_users(Filename, FileData, Convertor),
+        {_NewUsersCnt, Users} ->
-        case length(Users) > 0 andalso do_import_users(Users) of
+            case do_import_users(Users, Opts#{filename => Filename}) of
-            false ->
+                ok ->
-                error(empty_users);
+                    ok;
-            ok ->
+                %% Do not log empty user entries.
-                ok;
+                %% The default etc/auth-built-in-db.csv file contains an empty user entry.
-            {error, Reason} ->
+                {error, empty_users} ->
-                _ = do_clean_imported_users(Users),
+                    {error, empty_users};
-                error(Reason)
+                {error, Reason} ->
-        end
+                    ?SLOG(
+                        warning,
+                        #{
+                            msg => "import_authn_users_failed",
+                            reason => Reason,
+                            type => PasswordType,
+                            filename => Filename
+                        }
+                    ),
+                    {error, Reason}
+            end
     catch
-        error:Reason1:Stk ->
+        error:Reason:Stk ->
             ?SLOG(
                 warning,
                 #{
-                    msg => "import_users_failed",
+                    msg => "parse_authn_users_failed",
-                    reason => Reason1,
+                    reason => Reason,
                     type => PasswordType,
                     filename => Filename,
                     stacktrace => Stk
                 }
             ),
-            {error, Reason1}
+            {error, Reason}
     end.
 
-do_import_users(Users) ->
+do_import_users([], _Opts) ->
+    {error, empty_users};
+do_import_users(Users, Opts) ->
     trans(
         fun() ->
             lists:foreach(
-                fun(
+                fun(User) ->
-                    #{
+                    insert_user(User, Opts)
-                        <<"user_group">> := UserGroup,
-                        <<"user_id">> := UserID,
-                        <<"password_hash">> := PasswordHash,
-                        <<"salt">> := Salt,
-                        <<"is_superuser">> := IsSuperuser
-                    }
-                ) ->
-                    insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser)
                 end,
                 Users
             )
         end
     ).
 
-do_clean_imported_users(Users) ->
-    lists:foreach(
-        fun(
-            #{
-                <<"user_group">> := UserGroup,
-                <<"user_id">> := UserID
-            }
-        ) ->
-            mria:dirty_delete(?TAB, {UserGroup, UserID})
-        end,
-        Users
-    ).
-
 add_user(
     UserInfo,
     State
@@ -338,7 +332,14 @@ run_fuzzy_filter(
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) ->
+insert_user(User, Opts) ->
+    #{
+        <<"user_group">> := UserGroup,
+        <<"user_id">> := UserID,
+        <<"password_hash">> := PasswordHash,
+        <<"salt">> := Salt,
+        <<"is_superuser">> := IsSuperuser
+    } = User,
     UserInfoRecord =
         #user_info{user_id = DBUserID} =
         user_info_record(UserGroup, UserID, PasswordHash, Salt, IsSuperuser),
@@ -348,14 +349,22 @@ insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) ->
         [UserInfoRecord] ->
             ok;
         [_] ->
+            Msg =
+                case maps:get(override, Opts, false) of
+                    true ->
+                        insert_user(UserInfoRecord),
+                        "override_an_exists_userid_into_authentication_database_ok";
+                    false ->
+                        "import_an_exists_userid_into_authentication_database_failed"
+                end,
             ?SLOG(warning, #{
-                msg => "bootstrap_authentication_overridden_in_the_built_in_database",
+                msg => Msg,
                 user_id => UserID,
                 group_id => UserGroup,
+                bootstrap_file => maps:get(filename, Opts),
                 suggestion =>
-                    "If you have made changes in other way, remove the user_id from the bootstrap file."
+                    "If you've altered it differently, delete the user_id from the bootstrap file."
-            }),
+            })
-            insert_user(UserInfoRecord)
     end.
 
 insert_user(#user_info{} = UserInfoRecord) ->
@@ -505,7 +514,7 @@ reader_fn(Filename0, Data) ->
                     error(Reason)
             end;
         <<".csv">> ->
-            %% Example: data/user-credentials.csv
+            %% Example: etc/auth-built-in-db-bootstrap.csv
             emqx_utils_stream:csv(Data);
         <<>> ->
             error(unknown_file_format);
@@ -550,16 +559,15 @@ is_superuser(#{<<"is_superuser">> := true}) -> true;
 is_superuser(_) -> false.
 
 boostrap_user_from_file(Config, State) ->
-    case maps:get(boostrap_file, Config, <<>>) of
+    case maps:get(bootstrap_file, Config, <<>>) of
         <<>> ->
             ok;
         FileName0 ->
-            #{boostrap_type := Type} = Config,
+            #{bootstrap_type := Type} = Config,
             FileName = emqx_schema:naive_env_interpolation(FileName0),
             case file:read_file(FileName) of
                 {ok, FileData} ->
-                    %% if there is a key conflict, override with the key which from the bootstrap file
+                    _ = import_users({Type, FileName, FileData}, State, #{override => false}),
-                    _ = import_users({Type, FileName, FileData}, State),
                     ok;
                 {error, Reason} ->
                     ?SLOG(warning, #{

apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl

@@ -46,7 +46,7 @@ select_union_member(_Kind, _Value) ->
 fields(builtin_db) ->
     [
         {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1}
-    ] ++ common_fields() ++ bootstrap_fields();
+    ] ++ common_fields();
 fields(builtin_db_api) ->
     [
         {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw_api/1}
@@ -68,7 +68,8 @@ common_fields() ->
         {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SIMPLE)},
         {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)},
         {user_id_type, fun user_id_type/1}
-    ] ++ emqx_authn_schema:common_fields().
+    ] ++ bootstrap_fields() ++
+        emqx_authn_schema:common_fields().
 
 bootstrap_fields() ->
     [
@@ -78,7 +79,7 @@ bootstrap_fields() ->
                 #{
                     desc => ?DESC(bootstrap_file),
                     required => false,
-                    default => <<>>
+                    default => <<"${EMQX_ETC_DIR}/auth-built-in-db-bootstrap.csv">>
                 }
             )},
         {bootstrap_type,

apps/emqx_auth_mnesia/test/emqx_authn_api_mnesia_SUITE.erl

@@ -51,6 +51,7 @@ end_per_testcase(_, Config) ->
 init_per_suite(Config) ->
     Apps = emqx_cth_suite:start(
         [
+            emqx_ctl,
             emqx,
             emqx_conf,
             emqx_auth,

apps/emqx_auth_mnesia/test/emqx_authn_mnesia_SUITE.erl

@@ -56,6 +56,7 @@ t_create(_) ->
     Config1 = Config0#{password_hash_algorithm => #{name => sha256}},
     {ok, _} = emqx_authn_mnesia:create(?AUTHN_ID, Config1),
     ok.
+
 t_bootstrap_file(_) ->
     Config = config(),
     %% hash to hash
@@ -102,25 +103,39 @@ t_bootstrap_file(_) ->
         ],
         test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.json">>)
     ),
+    Opts = #{clean => false},
+    Result = test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.csv">>, Opts),
     ?assertMatch(
         [
             {user_info, {_, <<"myuser3">>}, _, _, true},
             {user_info, {_, <<"myuser4">>}, _, _, false}
         ],
-        test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.csv">>)
+        Result
+    ),
+    %% Don't override the exist user id.
+    ?assertMatch(
+        Result, test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain_v2.csv">>)
     ),
     ok.
 
 test_bootstrap_file(Config0, Type, File) ->
+    test_bootstrap_file(Config0, Type, File, #{clean => true}).
+
+test_bootstrap_file(Config0, Type, File, Opts) ->
     {Type, Filename, _FileData} = sample_filename_and_data(Type, File),
     Config2 = Config0#{
-        boostrap_file => Filename,
+        bootstrap_file => Filename,
-        boostrap_type => Type
+        bootstrap_type => Type
     },
     {ok, State0} = emqx_authn_mnesia:create(?AUTHN_ID, Config2),
     Result = ets:tab2list(emqx_authn_mnesia),
-    ok = emqx_authn_mnesia:destroy(State0),
+    case maps:get(clean, Opts) of
-    ?assertMatch([], ets:tab2list(emqx_authn_mnesia)),
+        true ->
+            ok = emqx_authn_mnesia:destroy(State0),
+            ?assertMatch([], ets:tab2list(emqx_authn_mnesia));
+        _ ->
+            ok
+    end,
     Result.
 
 t_update(_) ->

apps/emqx_auth_mnesia/test/emqx_authz_mnesia_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include_lib("emqx_authz.hrl").
+-include_lib("emqx_auth/include/emqx_authz.hrl").
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -256,6 +256,30 @@ t_destroy(_Config) ->
         emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
     ).
 
+t_conf_cli_load(_Config) ->
+    ClientInfo = emqx_authz_test_lib:base_client_info(),
+
+    ok = emqx_authz_mnesia:store_rules(
+        {username, <<"username">>},
+        [#{<<"permission">> => <<"allow">>, <<"action">> => <<"publish">>, <<"topic">> => <<"t">>}]
+    ),
+
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
+    ),
+    PrevRules = ets:tab2list(emqx_acl),
+    Hocon = emqx_conf_cli:get_config("authorization"),
+    Bin = iolist_to_binary(hocon_pp:do(Hocon, #{})),
+    ok = emqx_conf_cli:load_config(Bin, #{mode => merge}),
+    %% ensure emqx_acl table not clear
+    ?assertEqual(PrevRules, ets:tab2list(emqx_acl)),
+    %% still working
+    ?assertEqual(
+        allow,
+        emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
+    ).
+
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------

apps/emqx_auth_mongodb/mix.exs

@@ -0,0 +1,33 @@
+defmodule EMQXAuthMongoDB.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth_mongodb,
+      version: "0.1.0",
+      build_path: "../../_build",
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_auth_mongodb_app, []}]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_auth, in_umbrella: true},
+      {:emqx_mongodb, in_umbrella: true},
+    ]
+  end
+end

apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include_lib("emqx_connector/include/emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").

apps/emqx_auth_mongodb/test/emqx_authn_mongodb_tls_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include_lib("emqx_connector/include/emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").

apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(export_all).
 
 -include_lib("emqx_auth/include/emqx_authz.hrl").
--include_lib("emqx_connector/include/emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("emqx/include/emqx_placeholder.hrl").

apps/emqx_auth_mysql/mix.exs

@@ -0,0 +1,33 @@
+defmodule EMQXAuthMySQL.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth_mysql,
+      version: "0.1.0",
+      build_path: "../../_build",
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_auth_mysql_app, []}]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_auth, in_umbrella: true},
+      {:emqx_mysql, in_umbrella: true}
+    ]
+  end
+end

apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include_lib("emqx_connector/include/emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").

apps/emqx_auth_mysql/test/emqx_authn_mysql_tls_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include_lib("emqx_connector/include/emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").

apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include("emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("emqx_auth/include/emqx_authz.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").

apps/emqx_auth_postgresql/mix.exs

@@ -0,0 +1,33 @@
+defmodule EMQXAuthPostgreSQL.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth_postgresql,
+      version: "0.1.0",
+      build_path: "../../_build",
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_auth_postgresql_app, []}]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_auth, in_umbrella: true},
+      {:emqx_postgresql, in_umbrella: true},
+    ]
+  end
+end

apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include_lib("emqx_postgresql/include/emqx_postgresql.hrl").
+-include_lib("../../emqx_postgresql/include/emqx_postgresql.hrl").
 -include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -196,7 +196,7 @@ test_user_auth(#{
         ?GLOBAL
     ).
 
-t_authenticate_disabled_prepared_statements(Config) ->
+t_authenticate_disabled_prepared_statements(_Config) ->
     ResConfig = maps:merge(pgsql_config(), #{disable_prepared_statements => true}),
     {ok, _} = emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, ResConfig),
     on_exit(fun() ->

apps/emqx_auth_postgresql/test/emqx_authn_postgresql_tls_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include_lib("emqx_postgresql/include/emqx_postgresql.hrl").
+-include_lib("../../emqx_postgresql/include/emqx_postgresql.hrl").
 -include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").

apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl

@@ -18,7 +18,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include_lib("emqx_postgresql/include/emqx_postgresql.hrl").
+-include_lib("../../emqx_postgresql/include/emqx_postgresql.hrl").
 -include_lib("emqx_auth/include/emqx_authz.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").

apps/emqx_auth_redis/mix.exs

@@ -0,0 +1,33 @@
+defmodule EMQXAuthRedis.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auth_redis,
+      version: "0.1.0",
+      build_path: "../../_build",
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_auth_redis_app, []}]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_auth, in_umbrella: true},
+      {:emqx_redis, in_umbrella: true}
+    ]
+  end
+end

apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include_lib("emqx_connector/include/emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").

apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl

@@ -19,7 +19,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include("emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("emqx_auth/include/emqx_authz.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").

apps/emqx_auto_subscribe/mix.exs

@@ -0,0 +1,27 @@
+defmodule EMQXAutoSubscribe.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_auto_subscribe,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_auto_subscribe_app, []}]
+  end
+
+  def deps() do
+    [{:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}]
+  end
+end

apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl

@@ -19,8 +19,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
-
+-include_lib("common_test/include/ct.hrl").
--define(APP, emqx_auto_subscribe).
 
 -define(TOPIC_C, <<"/c/${clientid}">>).
 -define(TOPIC_U, <<"/u/${username}">>).
@@ -44,8 +43,6 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    mria:start(),
-    application:stop(?APP),
     meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
     meck:expect(emqx_schema, fields, fun
         ("auto_subscribe") ->
@@ -60,43 +57,45 @@ init_per_suite(Config) ->
     meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end),
     meck:expect(emqx_resource, remove, fun(_) -> ok end),
 
-    application:load(emqx_dashboard),
+    ASCfg = <<
-    application:load(?APP),
+        "auto_subscribe {\n"
-    ok = emqx_common_test_helpers:load_config(
+        "            topics = [\n"
-        emqx_auto_subscribe_schema,
+        "                {\n"
-        <<
+        "                    topic = \"/c/${clientid}\"\n"
-            "auto_subscribe {\n"
+        "                },\n"
-            "            topics = [\n"
+        "                {\n"
-            "                {\n"
+        "                    topic = \"/u/${username}\"\n"
-            "                    topic = \"/c/${clientid}\"\n"
+        "                },\n"
-            "                },\n"
+        "                {\n"
-            "                {\n"
+        "                    topic = \"/h/${host}\"\n"
-            "                    topic = \"/u/${username}\"\n"
+        "                },\n"
-            "                },\n"
+        "                {\n"
-            "                {\n"
+        "                    topic = \"/p/${port}\"\n"
-            "                    topic = \"/h/${host}\"\n"
+        "                },\n"
-            "                },\n"
+        "                {\n"
-            "                {\n"
+        "                    topic = \"/client/${clientid}/username/${username}/host/${host}/port/${port}\"\n"
-            "                    topic = \"/p/${port}\"\n"
+        "                },\n"
-            "                },\n"
+        "                {\n"
-            "                {\n"
+        "                    topic = \"/topic/simple\"\n"
-            "                    topic = \"/client/${clientid}/username/${username}/host/${host}/port/${port}\"\n"
+        "                    qos   = 1\n"
-            "                },\n"
+        "                    rh    = 0\n"
-            "                {\n"
+        "                    rap   = 0\n"
-            "                    topic = \"/topic/simple\"\n"
+        "                    nl    = 0\n"
-            "                    qos   = 1\n"
+        "                }\n"
-            "                    rh    = 0\n"
+        "            ]\n"
-            "                    rap   = 0\n"
+        "        }"
-            "                    nl    = 0\n"
+    >>,
-            "                }\n"
+    Apps = emqx_cth_suite:start(
-            "            ]\n"
+        [
-            "        }"
+            emqx,
-        >>
+            emqx_conf,
+            {emqx_auto_subscribe, ASCfg},
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
-    emqx_mgmt_api_test_util:init_suite(
+    [{apps, Apps} | Config].
-        [emqx_conf, ?APP]
-    ),
-    Config.
 
 init_per_testcase(t_get_basic_usage_info, Config) ->
     {ok, _} = emqx_auto_subscribe:update([]),
@@ -119,13 +118,10 @@ topic_config(T) ->
         nl => 0
     }.
 
-end_per_suite(_) ->
+end_per_suite(Config) ->
-    application:unload(emqx_management),
+    Apps = ?config(apps, Config),
-    application:unload(emqx_conf),
+    emqx_cth_suite:stop(Apps),
-    application:unload(?APP),
+    ok.
-    meck:unload(emqx_resource),
-    meck:unload(emqx_schema),
-    emqx_mgmt_api_test_util:end_suite([emqx_conf, ?APP]).
 
 t_auto_subscribe(_) ->
     emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]),

apps/emqx_bridge/mix.exs

@@ -0,0 +1,45 @@
+defmodule EMQXBridge.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge,
+      version: "0.1.0",
+      build_path: "../../_build",
+      compilers: Mix.compilers() ++ [:copy_srcs],
+      # used by our `Mix.Tasks.Compile.CopySrcs` compiler
+      extra_dirs: extra_dirs(),
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications(), mod: {:emqx_bridge_app, []}]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_connector, in_umbrella: true},
+    ]
+  end
+
+  defp extra_dirs() do
+    dirs = []
+    if UMP.test_env?() do
+      ["test" | dirs]
+    else
+      dirs
+    end
+  end
+end

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -503,7 +503,18 @@ schema("/bridges_probe") ->
     end.
 
 '/bridges/:id'(get, #{bindings := #{id := Id}}) ->
-    ?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200));
+    ?TRY_PARSE_ID(
+        Id,
+        begin
+            CompatErrorMsg = non_compat_bridge_msg(),
+            case lookup_from_all_nodes(BridgeType, BridgeName, 200) of
+                {400, #{code := 'BAD_REQUEST', message := CompatErrorMsg}} ->
+                    ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
+                Res ->
+                    Res
+            end
+        end
+    );
 '/bridges/:id'(put, #{bindings := #{id := Id}, body := Conf0}) ->
     Conf1 = filter_out_request_body(Conf0),
     ?TRY_PARSE_ID(
@@ -636,7 +647,7 @@ lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) ->
         {ok, [{error, not_found} | _]} ->
             ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
         {ok, [{error, not_bridge_v1_compatible} | _]} ->
-            ?NOT_FOUND(non_compat_bridge_msg());
+            ?BAD_REQUEST(non_compat_bridge_msg());
         {error, Reason} ->
             ?INTERNAL_ERROR(Reason)
     end.

apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl

@@ -22,25 +22,22 @@ init_per_suite(Config, Apps) ->
     [{start_apps, Apps} | Config].
 
 end_per_suite(Config) ->
-    delete_all_bridges_and_connectors(),
+    Apps = ?config(apps, Config),
-    emqx_mgmt_api_test_util:end_suite(),
+    emqx_cth_suite:stop(Apps),
-    ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
-    ok = emqx_connector_test_helpers:stop_apps(lists:reverse(?config(start_apps, Config))),
-    _ = application:stop(emqx_connector),
     ok.
 
 init_per_group(TestGroup, BridgeType, Config) ->
     ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
     ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
     emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
-    application:load(emqx_bridge),
+    Apps = emqx_cth_suite:start(
-    ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+        ?config(start_apps, Config),
-    ok = emqx_connector_test_helpers:start_apps(?config(start_apps, Config)),
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
-    {ok, _} = application:ensure_all_started(emqx_connector),
+    ),
-    emqx_mgmt_api_test_util:init_suite(),
     UniqueNum = integer_to_binary(erlang:unique_integer([positive])),
     MQTTTopic = <<"mqtt/topic/abc", UniqueNum/binary>>,
     [
+        {apps, Apps},
         {proxy_host, ProxyHost},
         {proxy_port, ProxyPort},
         {mqtt_topic, MQTTTopic},
@@ -50,10 +47,11 @@ init_per_group(TestGroup, BridgeType, Config) ->
     ].
 
 end_per_group(Config) ->
+    Apps = ?config(apps, Config),
     ProxyHost = ?config(proxy_host, Config),
     ProxyPort = ?config(proxy_port, Config),
     emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
-    %    delete_all_bridges(),
+    emqx_cth_suite:stop(Apps),
     ok.
 
 init_per_testcase(TestCase, Config0, BridgeConfigCb) ->

apps/emqx_bridge_azure_event_hub/mix.exs

@@ -0,0 +1,38 @@
+defmodule EMQXBridgeAzureEventHub.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_azure_event_hub,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"},
+      {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},
+      {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"},
+      {:brod, github: "kafka4beam/brod", tag: "3.18.0"},
+      ## TODO: remove `mix.exs` from `wolff` and remove this override
+      ## TODO: remove `mix.exs` from `pulsar` and remove this override
+      {:snappyer, "1.2.9", override: true},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false}
+    ]
+  end
+end

apps/emqx_bridge_cassandra/mix.exs

@@ -0,0 +1,32 @@
+defmodule EMQXBridgeCassandra.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_cassandra,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:ecql, github: "emqx/ecql", tag: "v0.7.0"},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false}
+    ]
+  end
+end

apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl

@@ -148,8 +148,6 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    emqx_mgmt_api_test_util:end_suite(),
-    ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
     ok.
 
 init_per_testcase(_Testcase, Config) ->
@@ -191,11 +189,10 @@ common_init(Config0) ->
                     emqx_bridge,
                     emqx_rule_engine,
                     emqx_management,
-                    {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}
+                    emqx_mgmt_api_test_util:emqx_dashboard()
                 ],
                 #{work_dir => emqx_cth_suite:work_dir(Config0)}
             ),
-            {ok, _Api} = emqx_common_test_http:create_default_app(),
             % Connect to cassnadra directly and create the table
             catch connect_and_drop_table(Config0),
             connect_and_create_table(Config0),

apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl

@@ -9,7 +9,7 @@
 
 -include_lib("common_test/include/ct.hrl").
 -include("emqx_bridge_cassandra.hrl").
--include("emqx_connector/include/emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("stdlib/include/assert.hrl").
@@ -53,10 +53,24 @@ cassandra_servers(CassandraHost, CassandraPort) ->
     ).
 
 init_per_suite(Config) ->
-    ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+    Apps = emqx_cth_suite:start(
-    ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
+        [
-    {ok, _} = application:ensure_all_started(emqx_connector),
+            emqx,
-    Config.
+            emqx_conf,
+            emqx_bridge_cassandra,
+            emqx_bridge,
+            emqx_rule_engine,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{apps, Apps} | Config].
+
+end_per_suite(Config) ->
+    Apps = ?config(apps, Config),
+    emqx_cth_suite:stop(Apps),
+    ok.
 
 init_per_group(Group, Config) ->
     {CassandraHost, CassandraPort, AuthOpts} =
@@ -98,11 +112,6 @@ init_per_group(Group, Config) ->
 end_per_group(_Group, _Config) ->
     ok.
 
-end_per_suite(_Config) ->
-    ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
-    ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
-    _ = application:stop(emqx_connector).
-
 init_per_testcase(_, Config) ->
     Config.
 

apps/emqx_bridge_clickhouse/mix.exs

@@ -0,0 +1,32 @@
+defmodule EMQXBridgeClickhouse.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_clickhouse,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:clickhouse, github: "emqx/clickhouse-client-erl", tag: "0.3.1"},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false}
+    ]
+  end
+end

apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl

@@ -7,9 +7,9 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--define(APP, emqx_bridge_clickhouse).
 -define(CLICKHOUSE_HOST, "clickhouse").
 -define(CLICKHOUSE_PORT, "8123").
+-include_lib("common_test/include/ct.hrl").
 -include_lib("emqx_connector/include/emqx_connector.hrl").
 
 %% See comment in
@@ -25,17 +25,26 @@ init_per_suite(Config) ->
     Port = list_to_integer(clickhouse_port()),
     case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of
         true ->
-            emqx_common_test_helpers:render_and_load_app_config(emqx_conf),
+            Apps = emqx_cth_suite:start(
-            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+                [
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource, ?APP]),
+                    emqx,
-            snabbkaffe:fix_ct_logging(),
+                    emqx_conf,
+                    emqx_bridge_clickhouse,
+                    emqx_connector,
+                    emqx_bridge,
+                    emqx_rule_engine,
+                    emqx_management,
+                    emqx_mgmt_api_test_util:emqx_dashboard()
+                ],
+                #{work_dir => emqx_cth_suite:work_dir(Config)}
+            ),
             %% Create the db table
             Conn = start_clickhouse_connection(),
             % erlang:monitor,sb
             {ok, _, _} = clickhouse:query(Conn, sql_create_database(), #{}),
             {ok, _, _} = clickhouse:query(Conn, sql_create_table(), []),
             clickhouse:query(Conn, sql_find_key(42), []),
-            [{clickhouse_connection, Conn} | Config];
+            [{apps, Apps}, {clickhouse_connection, Conn} | Config];
         false ->
             case os:getenv("IS_CI") of
                 "yes" ->
@@ -74,8 +83,9 @@ start_clickhouse_connection() ->
 end_per_suite(Config) ->
     ClickhouseConnection = proplists:get_value(clickhouse_connection, Config),
     clickhouse:stop(ClickhouseConnection),
-    ok = emqx_connector_test_helpers:stop_apps([?APP, emqx_resource]),
+    Apps = ?config(apps, Config),
-    ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]).
+    emqx_cth_suite:stop(Apps),
+    ok.
 
 init_per_testcase(_, Config) ->
     reset_table(Config),

apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl

@@ -7,7 +7,7 @@
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include("emqx_connector.hrl").
+-include("../../emqx_connector/include/emqx_connector.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("stdlib/include/assert.hrl").
 -include_lib("common_test/include/ct.hrl").
@@ -43,8 +43,19 @@ init_per_suite(Config) ->
     Port = list_to_integer(emqx_bridge_clickhouse_SUITE:clickhouse_port()),
     case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of
         true ->
-            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+            Apps = emqx_cth_suite:start(
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource, ?APP]),
+                [
+                    emqx,
+                    emqx_conf,
+                    emqx_bridge_clickhouse,
+                    emqx_connector,
+                    emqx_bridge,
+                    emqx_rule_engine,
+                    emqx_management,
+                    emqx_mgmt_api_test_util:emqx_dashboard()
+                ],
+                #{work_dir => emqx_cth_suite:work_dir(Config)}
+            ),
             %% Create the db table
             {ok, Conn} =
                 clickhouse:start_link([
@@ -55,7 +66,7 @@ init_per_suite(Config) ->
                 ]),
             {ok, _, _} = clickhouse:query(Conn, <<"CREATE DATABASE IF NOT EXISTS mqtt">>, #{}),
             clickhouse:stop(Conn),
-            Config;
+            [{apps, Apps} | Config];
         false ->
             case os:getenv("IS_CI") of
                 "yes" ->
@@ -65,9 +76,10 @@ init_per_suite(Config) ->
             end
     end.
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
+    Apps = ?config(apps, Config),
-    ok = emqx_connector_test_helpers:stop_apps([?APP, emqx_resource]).
+    emqx_cth_suite:stop(Apps),
+    ok.
 
 init_per_testcase(_, Config) ->
     Config.

apps/emqx_bridge_confluent/mix.exs

@@ -0,0 +1,38 @@
+defmodule EMQXBridgeConfluent.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_confluent,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"},
+      {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},
+      {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"},
+      {:brod, github: "kafka4beam/brod", tag: "3.18.0"},
+      ## TODO: remove `mix.exs` from `wolff` and remove this override
+      ## TODO: remove `mix.exs` from `pulsar` and remove this override
+      {:snappyer, "1.2.9", override: true},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false}
+    ]
+  end
+end

apps/emqx_bridge_dynamo/mix.exs

@@ -0,0 +1,32 @@
+defmodule EMQXBridgeDynamo.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_dynamo,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:erlcloud, github: "emqx/erlcloud", tag: "3.7.0.3"},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false}
+    ]
+  end
+end

apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl

@@ -69,14 +69,18 @@ init_per_group(_Group, Config) ->
     Config.
 
 end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch ->
+    Apps = ?config(apps, Config),
     ProxyHost = ?config(proxy_host, Config),
     ProxyPort = ?config(proxy_port, Config),
     emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    emqx_cth_suite:stop(Apps),
     ok;
 end_per_group(Group, Config) when Group =:= flaky ->
+    Apps = ?config(apps, Config),
     ProxyHost = ?config(proxy_host, Config),
     ProxyPort = ?config(proxy_port, Config),
     emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    emqx_cth_suite:stop(Apps),
     timer:sleep(1000),
     ok;
 end_per_group(_Group, _Config) ->
@@ -135,18 +139,23 @@ common_init(ConfigT) ->
             ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
             ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
             emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
-            % Ensure enterprise bridge module is loaded
+            Apps = emqx_cth_suite:start(
-            ok = emqx_common_test_helpers:start_apps([
+                [
-                emqx_conf, emqx_resource, emqx_bridge, emqx_rule_engine
+                    emqx_conf,
-            ]),
+                    emqx_bridge_dynamo,
-            _ = application:ensure_all_started(erlcloud),
+                    emqx_bridge,
-            _ = emqx_bridge_enterprise:module_info(),
+                    emqx_rule_engine,
-            emqx_mgmt_api_test_util:init_suite(),
+                    emqx_management,
+                    emqx_mgmt_api_test_util:emqx_dashboard()
+                ],
+                #{work_dir => emqx_cth_suite:work_dir(Config0)}
+            ),
             % setup dynamo
             setup_dynamo(Config0),
             {Name, TDConf} = dynamo_config(BridgeType, Config0),
             Config =
                 [
+                    {apps, Apps},
                     {dynamo_config, TDConf},
                     {dynamo_bridge_type, BridgeType},
                     {dynamo_name, Name},

apps/emqx_bridge_es/mix.exs

@@ -0,0 +1,33 @@
+defmodule EMQXBridgeEs.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_es,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false},
+      {:emqx_bridge_http, in_umbrella: true}
+    ]
+  end
+end

apps/emqx_bridge_gcp_pubsub/mix.exs

@@ -0,0 +1,32 @@
+defmodule EMQXBridgeGcpPubsub.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_gcp_pubsub,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false},
+      {:emqx_bridge_http, in_umbrella: true}
+    ]
+  end
+end

apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl

@@ -519,6 +519,7 @@ wait_acked(Opts) ->
     %% no need to check return value; we check the property in
     %% the check phase.  this is just to give it a chance to do
     %% so and avoid flakiness.  should be fast.
+    ct:pal("waiting ~b ms until acked...", [Timeout]),
     Res = snabbkaffe:block_until(
         ?match_n_events(N, #{?snk_kind := gcp_pubsub_consumer_worker_acknowledged}),
         Timeout
@@ -1265,11 +1266,12 @@ t_multiple_topic_mappings(Config) ->
 
 %% 2+ pull workers do not duplicate delivered messages
 t_multiple_pull_workers(Config) ->
-    ct:timetrap({seconds, 120}),
+    ct:timetrap({seconds, 121}),
     BridgeName = ?config(consumer_name, Config),
     TopicMapping = ?config(topic_mapping, Config),
     ResourceId = resource_id(Config),
     ?check_trace(
+        #{timetrap => 120_000},
         begin
             NConsumers = 3,
             start_and_subscribe_mqtt(Config),
@@ -1277,7 +1279,7 @@ t_multiple_pull_workers(Config) ->
                 snabbkaffe:subscribe(
                     ?match_event(#{?snk_kind := "gcp_pubsub_consumer_worker_subscription_ready"}),
                     NConsumers,
-                    40_000
+                    infinity
                 ),
             {ok, _} = create_bridge(
                 Config,
@@ -1287,6 +1289,14 @@ t_multiple_pull_workers(Config) ->
                         <<"ack_deadline">> => <<"10m">>,
                         <<"ack_retry_interval">> => <<"1s">>,
                         <<"consumer_workers_per_topic">> => NConsumers
+                    },
+                    <<"resource_opts">> => #{
+                        %% Used by worker when patching subscritpion; we increase it a bit
+                        %% here because (at least) the gcp emulator tends to time out /
+                        %% throttle (?)  workers targeting the same subscription, making
+                        %% the test flakier.
+                        <<"request_ttl">> => <<"5s">>,
+                        <<"resume_interval">> => <<"1s">>
                     }
                 }
             ),
@@ -1294,6 +1304,14 @@ t_multiple_pull_workers(Config) ->
             [#{pubsub_topic := Topic}] = TopicMapping,
             Payload = emqx_guid:to_hexstr(emqx_guid:gen()),
             Messages = [#{<<"data">> => Payload}],
+            ?retry(
+                500,
+                20,
+                ?assertMatch(
+                    {ok, connected},
+                    health_check(Config)
+                )
+            ),
             pubsub_publish(Config, Topic, Messages),
             {ok, Published} = receive_published(),
             ?assertMatch(

apps/emqx_bridge_greptimedb/mix.exs

@@ -0,0 +1,32 @@
+defmodule EMQXBridgeGreptimedb.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_greptimedb,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false},
+      {:greptimedb, github: "GreptimeTeam/greptimedb-ingester-erl", tag: "v0.1.8"}
+    ]
+  end
+end

apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl

@@ -25,11 +25,17 @@ init_per_suite(Config) ->
     Servers = [{GreptimedbTCPHost, GreptimedbTCPPort}],
     case emqx_common_test_helpers:is_all_tcp_servers_available(Servers) of
         true ->
-            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+            Apps = emqx_cth_suite:start(
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
+                [
-            {ok, _} = application:ensure_all_started(emqx_connector),
+                    emqx,
-            {ok, _} = application:ensure_all_started(greptimedb),
+                    emqx_conf,
+                    emqx_bridge_greptimedb,
+                    emqx_bridge
+                ],
+                #{work_dir => emqx_cth_suite:work_dir(Config)}
+            ),
             [
+                {apps, Apps},
                 {greptimedb_tcp_host, GreptimedbTCPHost},
                 {greptimedb_tcp_port, GreptimedbTCPPort}
                 | Config
@@ -43,11 +49,9 @@ init_per_suite(Config) ->
             end
     end.
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
+    Apps = ?config(apps, Config),
-    ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
+    emqx_cth_suite:stop(Apps),
-    _ = application:stop(emqx_connector),
-    _ = application:stop(greptimedb),
     ok.
 
 init_per_testcase(_, Config) ->

apps/emqx_bridge_hstreamdb/mix.exs

@@ -0,0 +1,34 @@
+defmodule EMQXBridgeHstreamdb.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_hstreamdb,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:hstreamdb_erl,
+       github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"},
+      {:emqx, in_umbrella: true},
+      {:emqx_utils, in_umbrella: true},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true}
+    ]
+  end
+end

apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl

@@ -100,10 +100,12 @@ init_per_group(_Group, Config) ->
     Config.
 
 end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch ->
+    Apps = ?config(apps, Config),
     connect_and_delete_stream(Config),
     ProxyHost = ?config(proxy_host, Config),
     ProxyPort = ?config(proxy_port, Config),
     emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    emqx_cth_suite:stop(Apps),
     ok;
 end_per_group(_Group, _Config) ->
     ok.
@@ -408,11 +410,16 @@ common_init(ConfigT) ->
             ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
             ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
             emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
-            % Ensure EE bridge module is loaded
+            Apps = emqx_cth_suite:start(
-            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_resource, emqx_bridge]),
+                [
-            _ = application:ensure_all_started(hstreamdb_erl),
+                    emqx_conf,
-            _ = emqx_bridge_enterprise:module_info(),
+                    emqx_bridge_hstreamdb,
-            emqx_mgmt_api_test_util:init_suite(),
+                    emqx_bridge,
+                    emqx_management,
+                    emqx_mgmt_api_test_util:emqx_dashboard()
+                ],
+                #{work_dir => emqx_cth_suite:work_dir(Config0)}
+            ),
             % Connect to hstreamdb directly
             % drop old stream and then create new one
             connect_and_delete_stream(Config0),
@@ -420,6 +427,7 @@ common_init(ConfigT) ->
             {Name, HStreamDBConf} = hstreamdb_config(BridgeType, Config0),
             Config =
                 [
+                    {apps, Apps},
                     {hstreamdb_config, HStreamDBConf},
                     {hstreamdb_bridge_type, BridgeType},
                     {hstreamdb_name, Name},

apps/emqx_bridge_http/mix.exs

@@ -0,0 +1,33 @@
+defmodule EMQXBridgeHTTP.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_http,
+      version: "0.1.0",
+      build_path: "../../_build",
+      # config_path: "../../config/config.exs",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_resource, in_umbrella: true},
+      {:ehttpc, github: "emqx/ehttpc", tag: "0.4.13"}
+    ]
+  end
+end

apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl

@@ -640,8 +640,14 @@ on_get_channel_status(
     _ChannelId,
     State
 ) ->
-    %% XXX: Reuse the connector status
+    %% N.B.: `on_get_channel_status' expects a different return value than
-    on_get_status(InstId, State).
+    %% `on_get_status'.
+    case on_get_status(InstId, State, fun default_health_checker/2) of
+        {Status, _State, Reason} ->
+            {Status, Reason};
+        Res ->
+            Res
+    end.
 
 on_format_query_result({ok, Status, Headers, Body}) ->
     #{

apps/emqx_bridge_influxdb/mix.exs

@@ -0,0 +1,32 @@
+defmodule EMQXBridgeInfluxdb.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_influxdb,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.13"},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false}
+    ]
+  end
+end

apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl

@@ -53,12 +53,6 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    delete_all_bridges(),
-    emqx_mgmt_api_test_util:end_suite(),
-    ok = emqx_connector_test_helpers:stop_apps([
-        emqx_conf, emqx_bridge, emqx_resource, emqx_rule_engine
-    ]),
-    _ = application:stop(emqx_connector),
     ok.
 
 init_per_group(InfluxDBType, Config0) when
@@ -92,10 +86,18 @@ init_per_group(InfluxDBType, Config0) when
             ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
             ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
             emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
-            ok = start_apps(),
+            Apps = emqx_cth_suite:start(
-            {ok, _} = application:ensure_all_started(emqx_connector),
+                [
-            emqx_mgmt_api_test_util:init_suite(),
+                    emqx_conf,
-            Config = [{use_tls, UseTLS} | Config0],
+                    emqx_bridge_influxdb,
+                    emqx_bridge,
+                    emqx_rule_engine,
+                    emqx_management,
+                    emqx_mgmt_api_test_util:emqx_dashboard()
+                ],
+                #{work_dir => emqx_cth_suite:work_dir(Config0)}
+            ),
+            Config = [{apps, Apps}, {use_tls, UseTLS} | Config0],
             {Name, ConfigString, InfluxDBConfig} = influxdb_config(
                 apiv1, InfluxDBHost, InfluxDBPort, Config
             ),
@@ -164,10 +166,18 @@ init_per_group(InfluxDBType, Config0) when
             ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
             ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
             emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
-            ok = start_apps(),
+            Apps = emqx_cth_suite:start(
-            {ok, _} = application:ensure_all_started(emqx_connector),
+                [
-            emqx_mgmt_api_test_util:init_suite(),
+                    emqx_conf,
-            Config = [{use_tls, UseTLS} | Config0],
+                    emqx_bridge_influxdb,
+                    emqx_bridge,
+                    emqx_rule_engine,
+                    emqx_management,
+                    emqx_mgmt_api_test_util:emqx_dashboard()
+                ],
+                #{work_dir => emqx_cth_suite:work_dir(Config0)}
+            ),
+            Config = [{apps, Apps}, {use_tls, UseTLS} | Config0],
             {Name, ConfigString, InfluxDBConfig} = influxdb_config(
                 apiv2, InfluxDBHost, InfluxDBPort, Config
             ),
@@ -222,12 +232,13 @@ end_per_group(Group, Config) when
     Group =:= apiv2_tcp;
     Group =:= apiv2_tls
 ->
+    Apps = ?config(apps, Config),
     ProxyHost = ?config(proxy_host, Config),
     ProxyPort = ?config(proxy_port, Config),
     EHttpcPoolName = ?config(ehttpc_pool_name, Config),
     emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
     ehttpc_sup:stop_pool(EHttpcPoolName),
-    delete_bridge(Config),
+    emqx_cth_suite:stop(Apps),
     ok;
 end_per_group(_Group, _Config) ->
     ok.
@@ -250,14 +261,6 @@ end_per_testcase(_Testcase, Config) ->
 %% Helper fns
 %%------------------------------------------------------------------------------
 
-start_apps() ->
-    %% some configs in emqx_conf app are mandatory
-    %% we want to make sure they are loaded before
-    %% ekka start in emqx_common_test_helpers:start_apps/1
-    emqx_common_test_helpers:render_and_load_app_config(emqx_conf),
-    ok = emqx_common_test_helpers:start_apps([emqx_conf]),
-    ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]).
-
 example_write_syntax() ->
     %% N.B.: this single space character is relevant
     <<"${topic},clientid=${clientid}", " ", "payload=${payload},",

apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl

@@ -27,10 +27,16 @@ init_per_suite(Config) ->
     Servers = [{InfluxDBTCPHost, InfluxDBTCPPort}, {InfluxDBTLSHost, InfluxDBTLSPort}],
     case emqx_common_test_helpers:is_all_tcp_servers_available(Servers) of
         true ->
-            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+            Apps = emqx_cth_suite:start(
-            ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
+                [
-            {ok, _} = application:ensure_all_started(emqx_connector),
+                    emqx_conf,
+                    emqx_bridge_influxdb,
+                    emqx_bridge
+                ],
+                #{work_dir => emqx_cth_suite:work_dir(Config)}
+            ),
             [
+                {apps, Apps},
                 {influxdb_tcp_host, InfluxDBTCPHost},
                 {influxdb_tcp_port, InfluxDBTCPPort},
                 {influxdb_tls_host, InfluxDBTLSHost},
@@ -46,10 +52,10 @@ init_per_suite(Config) ->
             end
     end.
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
+    Apps = ?config(apps, Config),
-    ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
+    emqx_cth_suite:stop(Apps),
-    _ = application:stop(emqx_connector).
+    ok.
 
 init_per_testcase(_, Config) ->
     Config.

apps/emqx_bridge_iotdb/mix.exs

@@ -0,0 +1,33 @@
+defmodule EMQXBridgeIotdb.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_iotdb,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:emqx, in_umbrella: true},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false},
+      {:emqx_bridge_http, in_umbrella: true}
+    ]
+  end
+end

apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl

@@ -12,7 +12,6 @@
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -define(BRIDGE_TYPE_BIN, <<"iotdb">>).
--define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_bridge_iotdb]).
 
 %%------------------------------------------------------------------------------
 %% CT boilerplate
@@ -34,7 +33,19 @@ groups() ->
     ].
 
 init_per_suite(Config) ->
-    emqx_bridge_v2_testlib:init_per_suite(Config, ?APPS).
+    emqx_bridge_v2_testlib:init_per_suite(
+        Config,
+        [
+            emqx,
+            emqx_conf,
+            emqx_bridge_iotdb,
+            emqx_connector,
+            emqx_bridge,
+            emqx_rule_engine,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ]
+    ).
 
 end_per_suite(Config) ->
     emqx_bridge_v2_testlib:end_per_suite(Config).

apps/emqx_bridge_kafka/mix.exs

@@ -0,0 +1,38 @@
+defmodule EMQXBridgeKafka.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_kafka,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"},
+      {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},
+      {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"},
+      {:brod, github: "kafka4beam/brod", tag: "3.18.0"},
+      ## TODO: remove `mix.exs` from `wolff` and remove this override
+      ## TODO: remove `mix.exs` from `pulsar` and remove this override
+      {:snappyer, "1.2.9", override: true},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false}
+    ]
+  end
+end

apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl

@@ -94,7 +94,7 @@ end_per_testcase(TestCase, Config) when
     TestCase =:= t_ancient_v1_config_migration_without_local_topic
 ->
     Cluster = ?config(cluster, Config),
-    emqx_cth_cluster:stop(Cluster),
+    ok = emqx_cth_cluster:stop(Cluster),
     ok;
 end_per_testcase(_TestCase, Config) ->
     ProxyHost = ?config(proxy_host, Config),

apps/emqx_bridge_kinesis/mix.exs

@@ -0,0 +1,32 @@
+defmodule EMQXBridgeKinesis.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_kinesis,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:erlcloud, github: "emqx/erlcloud", tag: "3.7.0.3"},
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false}
+    ]
+  end
+end

apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl

@@ -42,11 +42,21 @@ init_per_suite(Config) ->
     ProxyName = "kinesis",
     SecretFile = filename:join(?config(priv_dir, Config), "secret"),
     ok = file:write_file(SecretFile, <<?KINESIS_SECRET_KEY>>),
-    ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+    Apps = emqx_cth_suite:start(
-    ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]),
+        [
-    {ok, _} = application:ensure_all_started(emqx_connector),
+            emqx,
-    emqx_mgmt_api_test_util:init_suite(),
+            emqx_conf,
+            emqx_bridge_kinesis,
+            emqx_connector,
+            emqx_bridge,
+            emqx_rule_engine,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
     [
+        {apps, Apps},
         {proxy_host, ProxyHost},
         {proxy_port, ProxyPort},
         {kinesis_port, list_to_integer(os:getenv("KINESIS_PORT", integer_to_list(?KINESIS_PORT)))},
@@ -55,11 +65,9 @@ init_per_suite(Config) ->
         | Config
     ].
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    emqx_mgmt_api_test_util:end_suite(),
+    Apps = ?config(apps, Config),
-    ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
+    emqx_cth_suite:stop(Apps),
-    ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]),
-    _ = application:stop(emqx_connector),
     ok.
 
 init_per_group(with_batch, Config) ->

apps/emqx_bridge_matrix/mix.exs

@@ -0,0 +1,31 @@
+defmodule EMQXBridgeMatrix.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_matrix,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false}
+    ]
+  end
+end

apps/emqx_bridge_mongodb/mix.exs

@@ -0,0 +1,32 @@
+defmodule EMQXBridgeMongodb.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_bridge_mongodb,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    [
+      {:emqx_connector, in_umbrella: true, runtime: false},
+      {:emqx_resource, in_umbrella: true},
+      {:emqx_bridge, in_umbrella: true, runtime: false},
+      {:emqx_mongodb, in_umbrella: true}
+    ]
+  end
+end