From 39d6f612cad142d8d8ffb008b0f8c5150668c069 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 31 May 2023 00:13:15 +0200
Subject: [PATCH 01/29] feat(emqx_release): add new APIs to parse/compare
 release versions

---
 apps/emqx/include/emqx_release.hrl    |  4 +-
 apps/emqx/src/emqx_release.erl        | 53 ++++++++++++++++++++++++-
 apps/emqx/test/emqx_release_tests.erl | 56 +++++++++++++++++++++++++++
 3 files changed, 110 insertions(+), 3 deletions(-)
 create mode 100644 apps/emqx/test/emqx_release_tests.erl

diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index 25b340f50..ecb0baabc 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -31,11 +31,11 @@
 %% NOTE: ALso make sure to follow the instructions in end of
 %% `apps/emqx/src/bpapi/README.md'
 
-%% Community edition
+%% Opensource edition
 -define(EMQX_RELEASE_CE, "5.1.0-alpha.1").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.1.0-alpha.1").
 
-%% the HTTP API version
+%% The HTTP API version
 -define(EMQX_API_VERSION, "5.0").
diff --git a/apps/emqx/src/emqx_release.erl b/apps/emqx/src/emqx_release.erl
index 4ecf8598b..9a930649e 100644
--- a/apps/emqx/src/emqx_release.erl
+++ b/apps/emqx/src/emqx_release.erl
@@ -21,7 +21,10 @@
     edition_vsn_prefix/0,
     edition_longstr/0,
     description/0,
-    version/0
+    version/0,
+    version_with_prefix/0,
+    vsn_compare/1,
+    vsn_compare/2
 ]).
 
 -include("emqx_release.hrl").
@@ -68,6 +71,10 @@ edition_vsn_prefix() ->
 edition_longstr() ->
     maps:get(edition(), ?EMQX_REL_NAME).
 
+%% @doc Return the release version with prefix.
+version_with_prefix() ->
+    edition_vsn_prefix() ++ version().
+
 %% @doc Return the release version.
 version() ->
     case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of
@@ -92,3 +99,47 @@ version() ->
 
 build_vsn() ->
     maps:get(edition(), ?EMQX_REL_VSNS).
+
+%% @doc Compare the given version with the current running version,
+%% return 'newer' 'older' or 'same'.
+vsn_compare("v" ++ Vsn) ->
+    vsn_compare(?EMQX_RELEASE_CE, Vsn);
+vsn_compare("e" ++ Vsn) ->
+    vsn_compare(?EMQX_RELEASE_EE, Vsn).
+
+%% @priv Compare the second argument with the first argument, return
+%% 'newer' 'older' or 'same' semver comparison result.
+vsn_compare(Vsn1, Vsn2) ->
+    ParsedVsn1 = parse_vsn(Vsn1),
+    ParsedVsn2 = parse_vsn(Vsn2),
+    case ParsedVsn1 =:= ParsedVsn2 of
+        true ->
+            same;
+        false when ParsedVsn1 < ParsedVsn2 ->
+            newer;
+        false ->
+            older
+    end.
+
+%% @priv Parse the version string to a tuple.
+%% Return {{Major, Minor, Patch}, Suffix}.
+%% Where Suffix is either an empty string or a tuple like {"rc", 1}.
+%% NOTE: taking the nature ordering of the suffix:
+%% {"alpha", _} < {"beta", _} < {"rc", _} < ""
+parse_vsn(Vsn) ->
+    try
+        [V1, V2, V3 | Suffix0] = string:tokens(Vsn, ".-"),
+        Suffix =
+            case Suffix0 of
+                "" ->
+                    %% For the case like "5.1.0"
+                    "";
+                [ReleaseStage, Number] ->
+                    %% For the case like "5.1.0-rc.1"
+                    {ReleaseStage, list_to_integer(Number)}
+            end,
+        {{list_to_integer(V1), list_to_integer(V2), list_to_integer(V3)}, Suffix}
+    catch
+        _:_ ->
+            erlang:error({invalid_version_string, Vsn})
+    end.
diff --git a/apps/emqx/test/emqx_release_tests.erl b/apps/emqx/test/emqx_release_tests.erl
new file mode 100644
index 000000000..45d9a53c4
--- /dev/null
+++ b/apps/emqx/test/emqx_release_tests.erl
@@ -0,0 +1,56 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_release_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+vsn_compre_test_() ->
+    CurrentVersion = emqx_release:version_with_prefix(),
+    [
+        {"must be 'same' when comparing with current version", fun() ->
+            ?assertEqual(same, emqx_release:vsn_compare(CurrentVersion))
+        end},
+        {"must be 'same' when comparing same version strings", fun() ->
+            ?assertEqual(same, emqx_release:vsn_compare("1.1.1", "1.1.1"))
+        end},
+        {"1.1.1 is older than 1.1.2", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.2", "1.1.1")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.1", "1.1.2"))
+        end},
+        {"1.1.9 is older than 1.1.10", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.10", "1.1.9")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.9", "1.1.10"))
+        end},
+        {"alpha is older than beta", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.1-beta.1", "1.1.1-alpha.2")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.1-alpha.2", "1.1.1-beta.1"))
+        end},
+        {"beta is older than rc", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.1-rc.1", "1.1.1-beta.2")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.1-beta.2", "1.1.1-rc.1"))
+        end},
+        {"rc is older than official cut", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.1", "1.1.1-rc.1")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.1-rc.1", "1.1.1"))
+        end},
+        {"invalid version string will crash", fun() ->
+            ?assertError({invalid_version_string, "1.1.a"}, emqx_release:vsn_compare("v1.1.a")),
+            ?assertError(
+                {invalid_version_string, "1.1.1-alpha"}, emqx_release:vsn_compare("e1.1.1-alpha")
+            )
+        end}
+    ].

From 332daabcc53a7f1f171cc9ae47fcf72328a9523f Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 31 May 2023 14:44:56 +0200
Subject: [PATCH 02/29] fix(config): Do not sync cluster config from nodes
 running new version

---
 apps/emqx_conf/src/emqx_conf_app.erl        | 192 ++++++++++++--------
 apps/emqx_conf/test/emqx_conf_app_SUITE.erl |  39 ++++
 2 files changed, 159 insertions(+), 72 deletions(-)

diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl
index 70234b525..363e367b6 100644
--- a/apps/emqx_conf/src/emqx_conf_app.erl
+++ b/apps/emqx_conf/src/emqx_conf_app.erl
@@ -42,6 +42,8 @@ start(_StartType, _StartArgs) ->
 stop(_State) ->
     ok.
 
+%% Read the cluster config from the local node.
+%% This function is named 'override' due to historical reasons.
 get_override_config_file() ->
     Node = node(),
     case emqx_app:get_init_config_load_done() of
@@ -63,7 +65,7 @@ get_override_config_file() ->
                             tnx_id => TnxId,
                             node => Node,
                             has_deprecated_file => HasDeprecateFile,
-                            release => emqx_app:get_release()
+                            release => emqx_release:version_with_prefix()
                         }
                     end,
                     case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of
@@ -95,7 +97,7 @@ init_conf() ->
     %% Workaround for https://github.com/emqx/mria/issues/94:
     _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000),
     _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]),
-    {ok, TnxId} = copy_override_conf_from_core_node(),
+    {ok, TnxId} = sync_cluster_conf(),
     _ = emqx_app:set_init_tnx_id(TnxId),
     ok = init_load(),
     ok = emqx_app:set_init_config_load_done().
@@ -103,88 +105,134 @@ init_conf() ->
 cluster_nodes() ->
     mria:cluster_nodes(cores) -- [node()].
 
-copy_override_conf_from_core_node() ->
+%% @doc Try to sync the cluster config from other core nodes.
+sync_cluster_conf() ->
     case cluster_nodes() of
-        %% The first core nodes is self.
         [] ->
-            ?SLOG(debug, #{msg => "skip_copy_override_conf_from_core_node"}),
+            %% The first core nodes is self.
+            ?SLOG(debug, #{
+                msg => "skip_sync_cluster_conf", reason => "Running single node"
+            }),
             {ok, ?DEFAULT_INIT_TXN_ID};
         Nodes ->
-            {Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes),
-            {Ready, NotReady0} = lists:partition(fun(Res) -> element(1, Res) =:= ok end, Results),
-            NotReady = lists:filter(fun(Res) -> element(1, Res) =:= error end, NotReady0),
-            case (Failed =/= [] orelse NotReady =/= []) andalso Ready =/= [] of
+            sync_cluster_conf2(Nodes)
+    end.
+
+%% @priv Some core nodes are running, try to sync the cluster config from them.
+sync_cluster_conf2(Nodes) ->
+    {Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes),
+    {Ready, NotReady0} = lists:partition(fun(Res) -> element(1, Res) =:= ok end, Results),
+    NotReady = lists:filter(fun(Res) -> element(1, Res) =:= error end, NotReady0),
+    case (Failed =/= [] orelse NotReady =/= []) of
+        true when Ready =/= [] ->
+            %% Some core nodes failed to reply.
+            Warning = #{
+                nodes => Nodes,
+                failed => Failed,
+                not_ready => NotReady,
+                msg => "ignored_nodes_when_sync_cluster_conf"
+            },
+            ?SLOG(warning, Warning);
+        true ->
+            %% There are core nodes running but no one was able to reply.
+            ?SLOG(error, #{
+                msg => "failed_to_sync_cluster_conf",
+                nodes => Nodes,
+                failed => Failed,
+                not_ready => NotReady
+            });
+        false ->
+            ok
+    end,
+    case Ready of
+        [] ->
+            case should_proceed_with_boot() of
                 true ->
-                    Warning = #{
-                        nodes => Nodes,
-                        failed => Failed,
-                        not_ready => NotReady,
-                        msg => "ignored_bad_nodes_when_copy_init_config"
-                    },
-                    ?SLOG(warning, Warning);
-                false ->
-                    ok
-            end,
-            case Ready of
-                [] ->
-                    %% Other core nodes running but no one replicated it successfully.
-                    ?SLOG(error, #{
-                        msg => "copy_override_conf_from_core_node_failed",
+                    %% Act as if this node is alone, so it can
+                    %% finish the boot sequence and load the
+                    %% config for other nodes to copy it.
+                    ?SLOG(info, #{
+                        msg => "skip_sync_cluster_conf",
+                        loading_from_disk => true,
                         nodes => Nodes,
                         failed => Failed,
                         not_ready => NotReady
                     }),
-
-                    case should_proceed_with_boot() of
-                        true ->
-                            %% Act as if this node is alone, so it can
-                            %% finish the boot sequence and load the
-                            %% config for other nodes to copy it.
-                            ?SLOG(info, #{
-                                msg => "skip_copy_override_conf_from_core_node",
-                                loading_from_disk => true,
-                                nodes => Nodes,
-                                failed => Failed,
-                                not_ready => NotReady
-                            }),
-                            {ok, ?DEFAULT_INIT_TXN_ID};
-                        false ->
-                            %% retry in some time
-                            Jitter = rand:uniform(2000),
-                            Timeout = 10000 + Jitter,
-                            ?SLOG(info, #{
-                                msg => "copy_cluster_conf_from_core_node_retry",
-                                timeout => Timeout,
-                                nodes => Nodes,
-                                failed => Failed,
-                                not_ready => NotReady
-                            }),
-                            timer:sleep(Timeout),
-                            copy_override_conf_from_core_node()
-                    end;
-                _ ->
-                    [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
-                    #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
-                    HasDeprecatedFile = has_deprecated_file(Info),
-                    ?SLOG(debug, #{
-                        msg => "copy_cluster_conf_from_core_node_success",
-                        node => Node,
-                        has_deprecated_file => HasDeprecatedFile,
-                        local_release => emqx_app:get_release(),
-                        remote_release => maps:get(release, Info, "before_v5.0.24|e5.0.3"),
-                        data_dir => emqx:data_dir(),
-                        tnx_id => TnxId
+                    {ok, ?DEFAULT_INIT_TXN_ID};
+                false ->
+                    %% retry in some time
+                    Jitter = rand:uniform(2000),
+                    Timeout = 10000 + Jitter,
+                    timer:sleep(Timeout),
+                    ?SLOG(warning, #{
+                        msg => "sync_cluster_conf_retry",
+                        timeout => Timeout,
+                        nodes => Nodes,
+                        failed => Failed,
+                        not_ready => NotReady
                     }),
-                    ok = emqx_config:save_to_override_conf(
-                        HasDeprecatedFile,
-                        RawOverrideConf,
-                        #{override_to => cluster}
-                    ),
-                    ok = sync_data_from_node(Node),
-                    {ok, TnxId}
-            end
+                    sync_cluster_conf()
+            end;
+        _ ->
+            sync_cluster_conf3(Ready)
     end.
 
+%% @priv Filter out the nodes which are running a newer version than this node.
+sync_cluster_conf3(Ready) ->
+    NotNewer = fun({ok, #{release := RemoteRelease}}) ->
+        try
+            emqx_release:vsn_compare(RemoteRelease) =/= newer
+        catch
+            _:_ ->
+                %% If the version is not valid (without v or e prefix),
+                %% we know it's older than v5.1.0/e5.1.0
+                true
+        end
+    end,
+    case lists:filter(NotNewer, Ready) of
+        [] ->
+            %% All available core nodes are running a newer version than this node.
+            %% Start this node without syncing cluster config from them.
+            %% This is likely a restart of an older version node during cluster upgrade.
+            NodesAndVersions = lists:map(
+                fun({ok, #{node := Node, release := Release}}) ->
+                    #{node => Node, version => Release}
+                end,
+                Ready
+            ),
+            ?SLOG(warning, #{
+                msg => "all_available_nodes_running_newer_version",
+                hint => "Booting this node without syncing cluster config from peer core nodes",
+                peer_nodes => NodesAndVersions
+            }),
+            {ok, ?DEFAULT_INIT_TXN_ID};
+        Ready2 ->
+            sync_cluster_conf4(Ready2)
+    end.
+
+%% @priv Some core nodes are running and replied with their configs successfully.
+%% Try to sort the results and save the first one for local use.
+sync_cluster_conf4(Ready) ->
+    [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
+    #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
+    HasDeprecatedFile = has_deprecated_file(Info),
+    ?SLOG(debug, #{
+        msg => "sync_cluster_conf_success",
+        synced_from_node => Node,
+        has_deprecated_file => HasDeprecatedFile,
+        local_release => emqx_app:get_release(),
+        remote_release => maps:get(release, Info, "before_v5.0.24|e5.0.3"),
+        data_dir => emqx:data_dir(),
+        tnx_id => TnxId
+    }),
+    ok = emqx_config:save_to_override_conf(
+        HasDeprecatedFile,
+        RawOverrideConf,
+        #{override_to => cluster}
+    ),
+    ok = sync_data_from_node(Node),
+    {ok, TnxId}.
+
 should_proceed_with_boot() ->
     TablesStatus = emqx_cluster_rpc:get_tables_status(),
     LocalNode = node(),
diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl
index 583405158..95b4ce697 100644
--- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl
+++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl
@@ -98,6 +98,34 @@ t_copy_deprecated_data_dir(_Config) ->
         stop_cluster(Nodes)
     end.
 
+t_no_copy_from_newer_version_node(_Config) ->
+    net_kernel:start(['master2@127.0.0.1', longnames]),
+    ct:timetrap({seconds, 120}),
+    snabbkaffe:fix_ct_logging(),
+    Cluster = cluster([cluster_spec({core, 10}), cluster_spec({core, 11}), cluster_spec({core, 12})]),
+    OKs = [ok, ok, ok],
+    [First | Rest] = Nodes = start_cluster(Cluster),
+    try
+        File = "/configs/cluster.hocon",
+        assert_config_load_done(Nodes),
+        rpc:call(First, ?MODULE, create_data_dir, [File]),
+        {OKs, []} = rpc:multicall(Nodes, application, stop, [emqx_conf]),
+        {OKs, []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []),
+        {OKs, []} = rpc:multicall(Nodes, meck, new, [
+            emqx_release, [passthrough, no_history, no_link, non_strict]
+        ]),
+        %% 99.9.9 is always newer than the current version
+        {OKs, []} = rpc:multicall(Nodes, meck, expect, [
+            emqx_release, version_with_prefix, 0, "e99.9.9"
+        ]),
+        ok = rpc:call(First, application, start, [emqx_conf]),
+        {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]),
+        ok = assert_no_cluster_conf_copied(Rest, File),
+        stop_cluster(Nodes),
+        ok
+    after
+        stop_cluster(Nodes)
+    end.
 %%------------------------------------------------------------------------------
 %% Helper functions
 %%------------------------------------------------------------------------------
@@ -158,6 +186,17 @@ assert_data_copy_done([First0 | Rest], File) ->
         Rest
     ).
 
+assert_no_cluster_conf_copied([], _) ->
+    ok;
+assert_no_cluster_conf_copied([Node | Nodes], File) ->
+    NodeStr = atom_to_list(Node),
+    ?assertEqual(
+        {error, enoent},
+        file:read_file(NodeStr ++ File),
+        #{node => Node}
+    ),
+    assert_no_cluster_conf_copied(Nodes, File).
+
 assert_config_load_done(Nodes) ->
     lists:foreach(
         fun(Node) ->

From 5146de5b1cad540e1cd2f0f75a5a3cdf84ca0308 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 31 May 2023 17:34:39 +0200
Subject: [PATCH 03/29] feat: add a backup copies for cluster.hocon

---
 apps/emqx/src/emqx_config.erl        | 94 +++++++++++++++++++++-------
 apps/emqx/test/emqx_config_SUITE.erl | 66 ++++++++++++++++++-
 2 files changed, 136 insertions(+), 24 deletions(-)

diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl
index 91809134c..c84a1fb9c 100644
--- a/apps/emqx/src/emqx_config.erl
+++ b/apps/emqx/src/emqx_config.erl
@@ -91,7 +91,7 @@
 -export([ensure_atom_conf_path/2]).
 
 -ifdef(TEST).
--export([erase_all/0]).
+-export([erase_all/0, backup_and_write/2]).
 -endif.
 
 -include("logger.hrl").
@@ -105,6 +105,7 @@
 -define(LISTENER_CONF_PATH(TYPE, LISTENER, PATH), [listeners, TYPE, LISTENER | PATH]).
 
 -define(CONFIG_NOT_FOUND_MAGIC, '$0tFound').
+-define(MAX_KEEP_BACKUP_CONFIGS, 10).
 
 -export_type([
     update_request/0,
@@ -601,43 +602,94 @@ save_to_config_map(Conf, RawConf) ->
 -spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}.
 save_to_override_conf(_, undefined, _) ->
     ok;
-%% TODO: Remove deprecated override conf file when 5.1
 save_to_override_conf(true, RawConf, Opts) ->
     case deprecated_conf_file(Opts) of
         undefined ->
             ok;
         FileName ->
-            ok = filelib:ensure_dir(FileName),
-            case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
-                ok ->
-                    ok;
-                {error, Reason} ->
-                    ?SLOG(error, #{
-                        msg => "failed_to_write_override_file",
-                        filename => FileName,
-                        reason => Reason
-                    }),
-                    {error, Reason}
-            end
+            backup_and_write(FileName, hocon_pp:do(RawConf, #{}))
     end;
 save_to_override_conf(false, RawConf, _Opts) ->
     case cluster_hocon_file() of
         undefined ->
             ok;
         FileName ->
-            ok = filelib:ensure_dir(FileName),
-            case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
+            backup_and_write(FileName, hocon_pp:do(RawConf, #{}))
+    end.
+
+%% @priv This is the same human-readable timestamp format as
+%% hocon-cli generated app.<time>.config file name.
+now_time() ->
+    Ts = os:system_time(millisecond),
+    {{Y, M, D}, {HH, MM, SS}} = calendar:system_time_to_local_time(Ts, millisecond),
+    Res = io_lib:format(
+        "~0p.~2..0b.~2..0b.~2..0b.~2..0b.~2..0b.~3..0b",
+        [Y, M, D, HH, MM, SS, Ts rem 1000]
+    ),
+    lists:flatten(Res).
+
+%% @private Backup the current config to a file with a timestamp suffix and
+%% then save the new config to the config file.
+backup_and_write(Path, Content) ->
+    %% this may fail, but we don't care
+    %% e.g. read-only file system
+    _ = filelib:ensure_dir(Path),
+    TmpFile = Path ++ ".tmp",
+    case file:write_file(TmpFile, Content) of
+        ok ->
+            backup_and_replace(Path, TmpFile);
+        {error, Reason} ->
+            ?SLOG(error, #{
+                msg => "failed_to_save_conf_file",
+                hint =>
+                    "The updated cluster config is note saved on this node, please check the file system.",
+                filename => TmpFile,
+                reason => Reason
+            }),
+            %% e.g. read-only, it's not the end of the world
+            ok
+    end.
+
+backup_and_replace(Path, TmpPath) ->
+    Backup = Path ++ "." ++ now_time() ++ ".bak",
+    case file:rename(Path, Backup) of
+        ok ->
+            ok = file:rename(TmpPath, Path),
+            ok = prune_backup_files(Path);
+        {error, enoent} ->
+            %% not created yet
+            ok = file:rename(TmpPath, Path);
+        {error, Reason} ->
+            ?SLOG(warning, #{
+                msg => "failed_to_backup_conf_file",
+                filename => Backup,
+                reason => Reason
+            }),
+            ok
+    end.
+
+prune_backup_files(Path) ->
+    Files0 = filelib:wildcard(Path ++ ".*"),
+    Re = "\\.[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{3}\\.bak$",
+    Files = lists:filter(fun(F) -> re:run(F, Re) =/= nomatch end, Files0),
+    Sorted = lists:reverse(lists:sort(Files)),
+    {_Keeps, Deletes} = lists:split(min(?MAX_KEEP_BACKUP_CONFIGS, length(Sorted)), Sorted),
+    lists:foreach(
+        fun(F) ->
+            case file:delete(F) of
                 ok ->
                     ok;
                 {error, Reason} ->
-                    ?SLOG(error, #{
-                        msg => "failed_to_save_conf_file",
-                        filename => FileName,
+                    ?SLOG(warning, #{
+                        msg => "failed_to_delete_backup_conf_file",
+                        filename => F,
                         reason => Reason
                     }),
-                    {error, Reason}
+                    ok
             end
-    end.
+        end,
+        Deletes
+    ).
 
 add_handlers() ->
     ok = emqx_config_logger:add_handler(),
diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl
index a55531c2d..41701b0f4 100644
--- a/apps/emqx/test/emqx_config_SUITE.erl
+++ b/apps/emqx/test/emqx_config_SUITE.erl
@@ -31,7 +31,24 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     emqx_common_test_helpers:stop_apps([]).
 
-t_fill_default_values(_) ->
+init_per_testcase(TestCase, Config) ->
+    try
+        ?MODULE:TestCase({init, Config})
+    catch
+        error:function_clause ->
+            ok
+    end,
+    Config.
+
+end_per_testcase(TestCase, Config) ->
+    try
+        ?MODULE:TestCase({'end', Config})
+    catch
+        error:function_clause ->
+            ok
+    end.
+
+t_fill_default_values(C) when is_list(C) ->
     Conf = #{
         <<"broker">> => #{
             <<"perf">> => #{},
@@ -60,7 +77,7 @@ t_fill_default_values(_) ->
     _ = emqx_utils_json:encode(WithDefaults),
     ok.
 
-t_init_load(_Config) ->
+t_init_load(C) when is_list(C) ->
     ConfFile = "./test_emqx.conf",
     ok = file:write_file(ConfFile, <<"">>),
     ExpectRootNames = lists:sort(hocon_schema:root_names(emqx_schema)),
@@ -79,7 +96,7 @@ t_init_load(_Config) ->
     ?assertMatch({ok, #{raw_config := 128}}, emqx:update_config([mqtt, max_topic_levels], 128)),
     ok = file:delete(DeprecatedFile).
 
-t_unknown_rook_keys(_) ->
+t_unknown_rook_keys(C) when is_list(C) ->
     ?check_trace(
         #{timetrap => 1000},
         begin
@@ -96,3 +113,46 @@ t_unknown_rook_keys(_) ->
         end
     ),
     ok.
+
+t_cluster_hocon_backup({init, C}) ->
+    C;
+t_cluster_hocon_backup({'end', _C}) ->
+    File = "backup-test.hocon",
+    Files = [File | filelib:wildcard(File ++ ".*.bak")],
+    lists:foreach(fun file:delete/1, Files);
+t_cluster_hocon_backup(C) when is_list(C) ->
+    Write = fun(Path, Content) ->
+        %% avoid name clash
+        timer:sleep(1),
+        emqx_config:backup_and_write(Path, Content)
+    end,
+    File = "backup-test.hocon",
+    %% write 12 times, 10 backups should be kept
+    %% the latest one is File itself without suffix
+    %% the oldest one is expected to be deleted
+    N = 12,
+    Inputs = lists:seq(1, N),
+    Backups = lists:seq(N - 10, N - 1),
+    InputContents = [integer_to_binary(I) || I <- Inputs],
+    BackupContents = [integer_to_binary(I) || I <- Backups],
+    lists:foreach(
+        fun(Content) ->
+            Write(File, Content)
+        end,
+        InputContents
+    ),
+    LatestContent = integer_to_binary(N),
+    ?assertEqual({ok, LatestContent}, file:read_file(File)),
+    Re = "\\.[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{3}\\.bak$",
+    Files = filelib:wildcard(File ++ ".*.bak"),
+    ?assert(lists:all(fun(F) -> re:run(F, Re) =/= nomatch end, Files)),
+    %% keep only the latest 10
+    ?assertEqual(10, length(Files)),
+    FilesSorted = lists:zip(lists:sort(Files), BackupContents),
+    lists:foreach(
+        fun({BackupFile, ExpectedContent}) ->
+            ?assertEqual({ok, ExpectedContent}, file:read_file(BackupFile))
+        end,
+        FilesSorted
+    ),
+    ok.

From be2d25515a8e12d42ee2d499178cd5074fdd9b35 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 31 May 2023 20:44:00 +0200
Subject: [PATCH 04/29] docs: add changelog for #10902

---
 changes/ce/fix-10902.en.md | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 changes/ce/fix-10902.en.md

diff --git a/changes/ce/fix-10902.en.md b/changes/ce/fix-10902.en.md
new file mode 100644
index 000000000..b7ca80803
--- /dev/null
+++ b/changes/ce/fix-10902.en.md
@@ -0,0 +1,6 @@
+Avoid syncing cluser.hocon file from the nodes runing a newer version than self.
+
+During cluster rolling upgrade, if an older version node has to restart due to whatever reason,
+if it copies the cluster.hocon file from a newer version node, it may fail to start.
+After this fix, the older version node will not copy the cluster.hocon file from a newer,
+so it will use its own cluster.hocon file to start.

From cb2be1811e3f99280e4d827520e6fb628a72573a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 1 Jun 2023 08:17:36 +0200
Subject: [PATCH 05/29] docs: fix function doc annotation

---
 apps/emqx/src/emqx_config.erl           | 2 +-
 apps/emqx/src/emqx_release.erl          | 4 ++--
 apps/emqx_conf/src/emqx_cluster_rpc.erl | 2 +-
 apps/emqx_conf/src/emqx_conf_app.erl    | 6 +++---
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl
index c84a1fb9c..3891f4bdf 100644
--- a/apps/emqx/src/emqx_config.erl
+++ b/apps/emqx/src/emqx_config.erl
@@ -617,7 +617,7 @@ save_to_override_conf(false, RawConf, _Opts) ->
             backup_and_write(FileName, hocon_pp:do(RawConf, #{}))
     end.
 
-%% @priv This is the same human-readable timestamp format as
+%% @private This is the same human-readable timestamp format as
 %% hocon-cli generated app.<time>.config file name.
 now_time() ->
     Ts = os:system_time(millisecond),
diff --git a/apps/emqx/src/emqx_release.erl b/apps/emqx/src/emqx_release.erl
index 9a930649e..c052eecc9 100644
--- a/apps/emqx/src/emqx_release.erl
+++ b/apps/emqx/src/emqx_release.erl
@@ -107,7 +107,7 @@ vsn_compare("v" ++ Vsn) ->
 vsn_compare("e" ++ Vsn) ->
     vsn_compare(?EMQX_RELEASE_EE, Vsn).
 
-%% @priv Compare the second argument with the first argument, return
+%% @private Compare the second argument with the first argument, return
 %% 'newer' 'older' or 'same' semver comparison result.
 vsn_compare(Vsn1, Vsn2) ->
     ParsedVsn1 = parse_vsn(Vsn1),
@@ -121,7 +121,7 @@ vsn_compare(Vsn1, Vsn2) ->
             older
     end.
 
-%% @priv Parse the version string to a tuple.
+%% @private Parse the version string to a tuple.
 %% Return {{Major, Minor, Patch}, Suffix}.
 %% Where Suffix is either an empty string or a tuple like {"rc", 1}.
 %% NOTE: taking the nature ordering of the suffix:
diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl
index c82191bc3..91558630f 100644
--- a/apps/emqx_conf/src/emqx_cluster_rpc.erl
+++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl
@@ -575,7 +575,7 @@ maybe_init_tnx_id(Node, TnxId) ->
     {atomic, _} = transaction(fun ?MODULE:commit/2, [Node, TnxId]),
     ok.
 
-%% @priv Cannot proceed until emqx app is ready.
+%% @private Cannot proceed until emqx app is ready.
 %% Otherwise the committed transaction catch up may fail.
 wait_for_emqx_ready() ->
     %% wait 10 seconds for emqx to start
diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl
index 363e367b6..c1a3b5df5 100644
--- a/apps/emqx_conf/src/emqx_conf_app.erl
+++ b/apps/emqx_conf/src/emqx_conf_app.erl
@@ -118,7 +118,7 @@ sync_cluster_conf() ->
             sync_cluster_conf2(Nodes)
     end.
 
-%% @priv Some core nodes are running, try to sync the cluster config from them.
+%% @private Some core nodes are running, try to sync the cluster config from them.
 sync_cluster_conf2(Nodes) ->
     {Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes),
     {Ready, NotReady0} = lists:partition(fun(Res) -> element(1, Res) =:= ok end, Results),
@@ -177,7 +177,7 @@ sync_cluster_conf2(Nodes) ->
             sync_cluster_conf3(Ready)
     end.
 
-%% @priv Filter out the nodes which are running a newer version than this node.
+%% @private Filter out the nodes which are running a newer version than this node.
 sync_cluster_conf3(Ready) ->
     NotNewer = fun({ok, #{release := RemoteRelease}}) ->
         try
@@ -210,7 +210,7 @@ sync_cluster_conf3(Ready) ->
             sync_cluster_conf4(Ready2)
     end.
 
-%% @priv Some core nodes are running and replied with their configs successfully.
+%% @private Some core nodes are running and replied with their configs successfully.
 %% Try to sort the results and save the first one for local use.
 sync_cluster_conf4(Ready) ->
     [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),

From 15ffa98bc566a0ea47be75760c2809f1d9f9a874 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 1 Jun 2023 08:18:26 +0200
Subject: [PATCH 06/29] chore: fix a typo in test case function name

---
 apps/emqx/test/emqx_config_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl
index 41701b0f4..a5da9ca4e 100644
--- a/apps/emqx/test/emqx_config_SUITE.erl
+++ b/apps/emqx/test/emqx_config_SUITE.erl
@@ -96,7 +96,7 @@ t_init_load(C) when is_list(C) ->
     ?assertMatch({ok, #{raw_config := 128}}, emqx:update_config([mqtt, max_topic_levels], 128)),
     ok = file:delete(DeprecatedFile).
 
-t_unknown_rook_keys(C) when is_list(C) ->
+t_unknown_root_keys(C) when is_list(C) ->
     ?check_trace(
         #{timetrap => 1000},
         begin

From 760de7becf4a8c1a71995e225aa0ca9506da89b2 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 1 Jun 2023 08:26:53 +0200
Subject: [PATCH 07/29] chore: imporve logging message

---
 apps/emqx_conf/src/emqx_conf_app.erl | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl
index c1a3b5df5..c09faec89 100644
--- a/apps/emqx_conf/src/emqx_conf_app.erl
+++ b/apps/emqx_conf/src/emqx_conf_app.erl
@@ -111,7 +111,8 @@ sync_cluster_conf() ->
         [] ->
             %% The first core nodes is self.
             ?SLOG(debug, #{
-                msg => "skip_sync_cluster_conf", reason => "Running single node"
+                msg => "skip_sync_cluster_conf",
+                reason => "This is a single node, or the first node in the cluster"
             }),
             {ok, ?DEFAULT_INIT_TXN_ID};
         Nodes ->
@@ -202,7 +203,9 @@ sync_cluster_conf3(Ready) ->
             ),
             ?SLOG(warning, #{
                 msg => "all_available_nodes_running_newer_version",
-                hint => "Booting this node without syncing cluster config from peer core nodes",
+                hint =>
+                    "Booting this node without syncing cluster config from peer core nodes "
+                    "because other nodes are running a newer version",
                 peer_nodes => NodesAndVersions
             }),
             {ok, ?DEFAULT_INIT_TXN_ID};

From bd29433997df051f4eeb593dfa955dac745e54a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9F=90=E6=96=87?= <zhongwencool@gmail.com>
Date: Sun, 28 May 2023 23:16:37 +0800
Subject: [PATCH 08/29] feat: support emqx_conf:update([exhook],Conf)

---
 apps/emqx_exhook/src/emqx_exhook.app.src |   2 +-
 apps/emqx_exhook/src/emqx_exhook_app.erl |   7 +-
 apps/emqx_exhook/src/emqx_exhook_mgr.erl | 109 +++++++++++++---
 apps/emqx_utils/src/emqx_utils.app.src   |   2 +-
 apps/emqx_utils/src/emqx_utils.erl       | 152 ++++++++++++++++++++++-
 5 files changed, 243 insertions(+), 29 deletions(-)

diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src
index 194c91206..92a70cf37 100644
--- a/apps/emqx_exhook/src/emqx_exhook.app.src
+++ b/apps/emqx_exhook/src/emqx_exhook.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_exhook, [
     {description, "EMQX Extension for Hook"},
-    {vsn, "5.0.12"},
+    {vsn, "5.0.13"},
     {modules, []},
     {registered, []},
     {mod, {emqx_exhook_app, []}},
diff --git a/apps/emqx_exhook/src/emqx_exhook_app.erl b/apps/emqx_exhook/src/emqx_exhook_app.erl
index 6e9bc5242..2fc562ae8 100644
--- a/apps/emqx_exhook/src/emqx_exhook_app.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_app.erl
@@ -22,8 +22,7 @@
 
 -export([
     start/2,
-    stop/1,
-    prep_stop/1
+    stop/1
 ]).
 
 %%--------------------------------------------------------------------
@@ -34,10 +33,6 @@ start(_StartType, _StartArgs) ->
     {ok, Sup} = emqx_exhook_sup:start_link(),
     {ok, Sup}.
 
-prep_stop(State) ->
-    emqx_ctl:unregister_command(exhook),
-    State.
-
 stop(_State) ->
     ok.
 
diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl
index 0647c80ea..573dded9e 100644
--- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl
@@ -181,10 +181,14 @@ pre_config_update(_, {enable, Name, Enable}, OldConf) ->
     of
         not_found -> throw(not_found);
         NewConf -> {ok, lists:map(fun maybe_write_certs/1, NewConf)}
-    end.
+    end;
+pre_config_update(_, NewConf, _OldConf) when NewConf =:= #{} ->
+    {ok, NewConf#{<<"servers">> => []}};
+pre_config_update(_, NewConf = #{<<"servers">> := Servers}, _OldConf) ->
+    {ok, NewConf#{<<"servers">> => lists:map(fun maybe_write_certs/1, Servers)}}.
 
 post_config_update(_KeyPath, UpdateReq, NewConf, OldConf, _AppEnvs) ->
-    Result = call({update_config, UpdateReq, NewConf}),
+    Result = call({update_config, UpdateReq, NewConf, OldConf}),
     try_clear_ssl_files(UpdateReq, NewConf, OldConf),
     {ok, Result}.
 
@@ -197,6 +201,7 @@ post_config_update(_KeyPath, UpdateReq, NewConf, OldConf, _AppEnvs) ->
 init([]) ->
     process_flag(trap_exit, true),
     emqx_conf:add_handler([exhook, servers], ?MODULE),
+    emqx_conf:add_handler([exhook], ?MODULE),
     ServerL = emqx:get_config([exhook, servers]),
     Servers = load_all_servers(ServerL),
     Servers2 = reorder(ServerL, Servers),
@@ -222,22 +227,16 @@ handle_call(
     OrderServers = sort_name_by_order(Infos, Servers),
     {reply, OrderServers, State};
 handle_call(
-    {update_config, {move, _Name, _Position}, NewConfL},
+    {update_config, {move, _Name, _Position}, NewConfL, _},
     _From,
     #{servers := Servers} = State
 ) ->
     Servers2 = reorder(NewConfL, Servers),
     {reply, ok, State#{servers := Servers2}};
-handle_call({update_config, {delete, ToDelete}, _}, _From, State) ->
-    emqx_exhook_metrics:on_server_deleted(ToDelete),
-
-    #{servers := Servers} = State2 = do_unload_server(ToDelete, State),
-
-    Servers2 = maps:remove(ToDelete, Servers),
-
-    {reply, ok, update_servers(Servers2, State2)};
+handle_call({update_config, {delete, ToDelete}, _, _}, _From, State) ->
+    {reply, ok, remove_server(ToDelete, State)};
 handle_call(
-    {update_config, {add, RawConf}, NewConfL},
+    {update_config, {add, RawConf}, NewConfL, _},
     _From,
     #{servers := Servers} = State
 ) ->
@@ -246,14 +245,68 @@ handle_call(
     Servers2 = Servers#{Name => Server},
     Servers3 = reorder(NewConfL, Servers2),
     {reply, Result, State#{servers := Servers3}};
+handle_call({update_config, {update, Name, _Conf}, NewConfL, _}, _From, State) ->
+    {Result, State2} = restart_server(Name, NewConfL, State),
+    {reply, Result, State2};
+handle_call({update_config, {enable, Name, _Enable}, NewConfL, _}, _From, State) ->
+    {Result, State2} = restart_server(Name, NewConfL, State),
+    {reply, Result, State2};
+handle_call({update_config, _, ConfL, ConfL}, _From, State) ->
+    {reply, ok, State};
+handle_call({update_config, _, #{servers := NewConfL}, #{servers := OldConfL}}, _From, State) ->
+    #{
+        removed := Removed,
+        added := Added,
+        changed := Updated
+    } = emqx_utils:diff_lists(NewConfL, OldConfL, fun(#{name := Name}) -> Name end),
+    %% remove servers
+    State2 = lists:foldl(
+        fun(Conf, Acc) ->
+            ToDelete = maps:get(name, Conf),
+            remove_server(ToDelete, Acc)
+        end,
+        State,
+        Removed
+    ),
+    %% update servers
+    {UpdateRes, State3} =
+        lists:foldl(
+            fun({_Old, Conf}, {ResAcc, StateAcc}) ->
+                Name = maps:get(name, Conf),
+                case restart_server(Name, NewConfL, StateAcc) of
+                    {ok, StateAcc1} -> {ResAcc, StateAcc1};
+                    {Err, StateAcc1} -> {[Err | ResAcc], StateAcc1}
+                end
+            end,
+            {[], State2},
+            Updated
+        ),
+    %% Add servers
+    {AddRes, State4} =
+        lists:foldl(
+            fun(Conf, {ResAcc, StateAcc}) ->
+                case do_load_server(options_to_server(Conf)) of
+                    {ok, Server} ->
+                        #{servers := Servers} = StateAcc,
+                        Name = maps:get(name, Conf),
+                        Servers2 = Servers#{Name => Server},
+                        {ResAcc, update_servers(Servers2, StateAcc)};
+                    {Err, StateAcc1} ->
+                        {[Err | ResAcc], StateAcc1}
+                end
+            end,
+            {[], State3},
+            Added
+        ),
+    %% update order
+    #{servers := Servers4} = State4,
+    State5 = State4#{servers => reorder(NewConfL, Servers4)},
+    case lists:append([UpdateRes, AddRes]) of
+        [] -> {reply, ok, State5};
+        _ -> {reply, {error, #{added => AddRes, updated => UpdateRes}}, State5}
+    end;
 handle_call({lookup, Name}, _From, State) ->
     {reply, where_is_server(Name, State), State};
-handle_call({update_config, {update, Name, _Conf}, NewConfL}, _From, State) ->
-    {Result, State2} = restart_server(Name, NewConfL, State),
-    {reply, Result, State2};
-handle_call({update_config, {enable, Name, _Enable}, NewConfL}, _From, State) ->
-    {Result, State2} = restart_server(Name, NewConfL, State),
-    {reply, Result, State2};
 handle_call({server_info, Name}, _From, State) ->
     case where_is_server(Name, State) of
         not_found ->
@@ -287,6 +340,12 @@ handle_call(_Request, _From, State) ->
     Reply = ok,
     {reply, Reply, State}.
 
+remove_server(ToDelete, State) ->
+    emqx_exhook_metrics:on_server_deleted(ToDelete),
+    #{servers := Servers} = State2 = do_unload_server(ToDelete, State),
+    Servers2 = maps:remove(ToDelete, Servers),
+    update_servers(Servers2, State2).
+
 handle_cast(_Msg, State) ->
     {noreply, State}.
 
@@ -310,6 +369,8 @@ terminate(Reason, State = #{servers := Servers}) ->
         Servers
     ),
     ?tp(info, exhook_mgr_terminated, #{reason => Reason, servers => Servers}),
+    emqx_conf:remove_handler([exhook, servers]),
+    emqx_conf:remove_handler([exhook]),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->
@@ -612,8 +673,16 @@ try_clear_ssl_files({Op, Name, _}, NewConfs, OldConfs) when
     NewSSL = find_server_ssl_cfg(Name, NewConfs),
     OldSSL = find_server_ssl_cfg(Name, OldConfs),
     emqx_tls_lib:delete_ssl_files(ssl_file_path(Name), NewSSL, OldSSL);
-try_clear_ssl_files(_Req, _NewConf, _OldConf) ->
-    ok.
+%% replace the whole config from the file
+try_clear_ssl_files(_Req, #{servers := NewServers}, #{servers := OldServers}) ->
+    lists:foreach(
+        fun(#{name := Name} = Conf) ->
+            NewSSL = find_server_ssl_cfg(Name, NewServers),
+            OldSSL = maps:get(ssl, Conf, undefined),
+            emqx_tls_lib:delete_ssl_files(ssl_file_path(Name), NewSSL, OldSSL)
+        end,
+        OldServers
+    ).
 
 search_server_cfg(Name, Confs) ->
     lists:search(
diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src
index 605093875..0b172565a 100644
--- a/apps/emqx_utils/src/emqx_utils.app.src
+++ b/apps/emqx_utils/src/emqx_utils.app.src
@@ -2,7 +2,7 @@
 {application, emqx_utils, [
     {description, "Miscellaneous utilities for EMQX apps"},
     % strict semver, bump manually!
-    {vsn, "5.0.2"},
+    {vsn, "5.0.3"},
     {modules, [
         emqx_utils,
         emqx_utils_api,
diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl
index 2c6ddd9c1..edc76a1e6 100644
--- a/apps/emqx_utils/src/emqx_utils.erl
+++ b/apps/emqx_utils/src/emqx_utils.erl
@@ -54,7 +54,8 @@
     safe_to_existing_atom/1,
     safe_to_existing_atom/2,
     pub_props_to_packet/1,
-    safe_filename/1
+    safe_filename/1,
+    diff_lists/3
 ]).
 
 -export([
@@ -748,3 +749,152 @@ safe_filename(Filename) when is_binary(Filename) ->
     binary:replace(Filename, <<":">>, <<"-">>, [global]);
 safe_filename(Filename) when is_list(Filename) ->
     lists:flatten(string:replace(Filename, ":", "-", all)).
+
+%% @doc Compares two lists of maps and returns the differences between them in a
+%% map containing four keys – 'removed', 'added', 'identical', and 'changed' –
+%% each holding a list of maps. Elements are compared using key function KeyFunc
+%% to extract the comparison key used for matching.
+%%
+%% The return value is a map with the following keys and the list of maps as its values:
+%% * 'removed' – a list of maps that were present in the Old list, but not found in the New list.
+%% * 'added' – a list of maps that were present in the New list, but not found in the Old list.
+%% * 'identical' – a list of maps that were present in both lists and have the same comparison key value.
+%% * 'changed' – a list of pairs of maps representing the changes between maps present in the New and Old lists.
+%% The first map in the pair represents the map in the Old list, and the second map
+%% represents the potential modification in the New list.
+
+%% The KeyFunc parameter is a function that extracts the comparison key used
+%% for matching from each map. The function should return a comparable term,
+%% such as an atom, a number, or a string. This is used to determine if each
+%% element is the same in both lists.
+
+-spec diff_lists(list(T), list(T), Func) ->
+    #{
+        added := list(T),
+        identical := list(T),
+        removed := list(T),
+        changed := list({Old :: T, New :: T})
+    }
+when
+    Func :: fun((T) -> any()),
+    T :: any().
+
+diff_lists(New, Old, KeyFunc) when is_list(New) andalso is_list(Old) ->
+    Removed =
+        lists:foldl(
+            fun(E, RemovedAcc) ->
+                case search(KeyFunc(E), KeyFunc, New) of
+                    false -> [E | RemovedAcc];
+                    _ -> RemovedAcc
+                end
+            end,
+            [],
+            Old
+        ),
+    {Added, Identical, Changed} =
+        lists:foldl(
+            fun(E, Acc) ->
+                {Added0, Identical0, Changed0} = Acc,
+                case search(KeyFunc(E), KeyFunc, Old) of
+                    false ->
+                        {[E | Added0], Identical0, Changed0};
+                    E ->
+                        {Added0, [E | Identical0], Changed0};
+                    E1 ->
+                        {Added0, Identical0, [{E1, E} | Changed0]}
+                end
+            end,
+            {[], [], []},
+            New
+        ),
+    #{
+        removed => lists:reverse(Removed),
+        added => lists:reverse(Added),
+        identical => lists:reverse(Identical),
+        changed => lists:reverse(Changed)
+    }.
+
+search(_ExpectValue, _KeyFunc, []) ->
+    false;
+search(ExpectValue, KeyFunc, [Item | List]) ->
+    case KeyFunc(Item) =:= ExpectValue of
+        true -> Item;
+        false -> search(ExpectValue, KeyFunc, List)
+    end.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+diff_lists_test() ->
+    KeyFunc = fun(#{name := Name}) -> Name end,
+    ?assertEqual(
+        #{
+            removed => [],
+            added => [],
+            identical => [],
+            changed => []
+        },
+        diff_lists([], [], KeyFunc)
+    ),
+    %% test removed list
+    ?assertEqual(
+        #{
+            removed => [#{name => a, value => 1}],
+            added => [],
+            identical => [],
+            changed => []
+        },
+        diff_lists([], [#{name => a, value => 1}], KeyFunc)
+    ),
+    %% test added list
+    ?assertEqual(
+        #{
+            removed => [],
+            added => [#{name => a, value => 1}],
+            identical => [],
+            changed => []
+        },
+        diff_lists([#{name => a, value => 1}], [], KeyFunc)
+    ),
+    %% test identical list
+    ?assertEqual(
+        #{
+            removed => [],
+            added => [],
+            identical => [#{name => a, value => 1}],
+            changed => []
+        },
+        diff_lists([#{name => a, value => 1}], [#{name => a, value => 1}], KeyFunc)
+    ),
+    Old = [
+        #{name => a, value => 1},
+        #{name => b, value => 4},
+        #{name => e, value => 2},
+        #{name => d, value => 4}
+    ],
+    New = [
+        #{name => a, value => 1},
+        #{name => b, value => 2},
+        #{name => e, value => 2},
+        #{name => c, value => 3}
+    ],
+    Diff = diff_lists(New, Old, KeyFunc),
+    ?assertEqual(
+        #{
+            added => [
+                #{name => c, value => 3}
+            ],
+            identical => [
+                #{name => a, value => 1},
+                #{name => e, value => 2}
+            ],
+            removed => [
+                #{name => d, value => 4}
+            ],
+            changed => [{#{name => b, value => 4}, #{name => b, value => 2}}]
+        },
+        Diff
+    ),
+    ok.
+
+-endif.

From 03160ef599781d82b2f0f9d6f271044fab22ece4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9F=90=E6=96=87?= <zhongwencool@gmail.com>
Date: Sun, 4 Jun 2023 12:02:02 +0800
Subject: [PATCH 09/29] test: add more test for update_config

---
 apps/emqx_exhook/src/emqx_exhook_mgr.erl    | 169 +++++++++-----------
 apps/emqx_exhook/test/emqx_exhook_SUITE.erl |  45 +++++-
 2 files changed, 114 insertions(+), 100 deletions(-)

diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl
index 573dded9e..173a1fecb 100644
--- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl
@@ -23,6 +23,9 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
+-define(SERVERS, [exhook, servers]).
+-define(EXHOOK, [exhook]).
+
 %% APIs
 -export([start_link/0]).
 
@@ -148,7 +151,7 @@ update_config(KeyPath, UpdateReq) ->
             Error
     end.
 
-pre_config_update(_, {add, #{<<"name">> := Name} = Conf}, OldConf) ->
+pre_config_update(?SERVERS, {add, #{<<"name">> := Name} = Conf}, OldConf) ->
     case lists:any(fun(#{<<"name">> := ExistedName}) -> ExistedName =:= Name end, OldConf) of
         true ->
             throw(already_exists);
@@ -156,35 +159,20 @@ pre_config_update(_, {add, #{<<"name">> := Name} = Conf}, OldConf) ->
             NConf = maybe_write_certs(Conf),
             {ok, OldConf ++ [NConf]}
     end;
-pre_config_update(_, {update, Name, Conf}, OldConf) ->
-    case replace_conf(Name, fun(_) -> Conf end, OldConf) of
-        not_found -> throw(not_found);
-        NewConf -> {ok, lists:map(fun maybe_write_certs/1, NewConf)}
-    end;
-pre_config_update(_, {delete, ToDelete}, OldConf) ->
-    case do_delete(ToDelete, OldConf) of
-        not_found -> throw(not_found);
-        NewConf -> {ok, NewConf}
-    end;
-pre_config_update(_, {move, Name, Position}, OldConf) ->
-    case do_move(Name, Position, OldConf) of
-        not_found -> throw(not_found);
-        NewConf -> {ok, NewConf}
-    end;
-pre_config_update(_, {enable, Name, Enable}, OldConf) ->
-    case
-        replace_conf(
-            Name,
-            fun(Conf) -> Conf#{<<"enable">> => Enable} end,
-            OldConf
-        )
-    of
-        not_found -> throw(not_found);
-        NewConf -> {ok, lists:map(fun maybe_write_certs/1, NewConf)}
-    end;
-pre_config_update(_, NewConf, _OldConf) when NewConf =:= #{} ->
+pre_config_update(?SERVERS, {update, Name, Conf}, OldConf) ->
+    NewConf = replace_conf(Name, fun(_) -> Conf end, OldConf),
+    {ok, lists:map(fun maybe_write_certs/1, NewConf)};
+pre_config_update(?SERVERS, {delete, ToDelete}, OldConf) ->
+    {ok, do_delete(ToDelete, OldConf)};
+pre_config_update(?SERVERS, {move, Name, Position}, OldConf) ->
+    {ok, do_move(Name, Position, OldConf)};
+pre_config_update(?SERVERS, {enable, Name, Enable}, OldConf) ->
+    ReplaceFun = fun(Conf) -> Conf#{<<"enable">> => Enable} end,
+    NewConf = replace_conf(Name, ReplaceFun, OldConf),
+    {ok, lists:map(fun maybe_write_certs/1, NewConf)};
+pre_config_update(?EXHOOK, NewConf, _OldConf) when NewConf =:= #{} ->
     {ok, NewConf#{<<"servers">> => []}};
-pre_config_update(_, NewConf = #{<<"servers">> := Servers}, _OldConf) ->
+pre_config_update(?EXHOOK, NewConf = #{<<"servers">> := Servers}, _OldConf) ->
     {ok, NewConf#{<<"servers">> => lists:map(fun maybe_write_certs/1, Servers)}}.
 
 post_config_update(_KeyPath, UpdateReq, NewConf, OldConf, _AppEnvs) ->
@@ -192,17 +180,15 @@ post_config_update(_KeyPath, UpdateReq, NewConf, OldConf, _AppEnvs) ->
     try_clear_ssl_files(UpdateReq, NewConf, OldConf),
     {ok, Result}.
 
-%%=====================================================================
-
 %%--------------------------------------------------------------------
 %% gen_server callbacks
 %%--------------------------------------------------------------------
 
 init([]) ->
     process_flag(trap_exit, true),
-    emqx_conf:add_handler([exhook, servers], ?MODULE),
-    emqx_conf:add_handler([exhook], ?MODULE),
-    ServerL = emqx:get_config([exhook, servers]),
+    emqx_conf:add_handler(?EXHOOK, ?MODULE),
+    emqx_conf:add_handler(?SERVERS, ?MODULE),
+    ServerL = emqx:get_config(?SERVERS),
     Servers = load_all_servers(ServerL),
     Servers2 = reorder(ServerL, Servers),
     refresh_tick(),
@@ -259,51 +245,13 @@ handle_call({update_config, _, #{servers := NewConfL}, #{servers := OldConfL}},
         added := Added,
         changed := Updated
     } = emqx_utils:diff_lists(NewConfL, OldConfL, fun(#{name := Name}) -> Name end),
-    %% remove servers
-    State2 = lists:foldl(
-        fun(Conf, Acc) ->
-            ToDelete = maps:get(name, Conf),
-            remove_server(ToDelete, Acc)
-        end,
-        State,
-        Removed
-    ),
-    %% update servers
-    {UpdateRes, State3} =
-        lists:foldl(
-            fun({_Old, Conf}, {ResAcc, StateAcc}) ->
-                Name = maps:get(name, Conf),
-                case restart_server(Name, NewConfL, StateAcc) of
-                    {ok, StateAcc1} -> {ResAcc, StateAcc1};
-                    {Err, StateAcc1} -> {[Err | ResAcc], StateAcc1}
-                end
-            end,
-            {[], State2},
-            Updated
-        ),
-    %% Add servers
-    {AddRes, State4} =
-        lists:foldl(
-            fun(Conf, {ResAcc, StateAcc}) ->
-                case do_load_server(options_to_server(Conf)) of
-                    {ok, Server} ->
-                        #{servers := Servers} = StateAcc,
-                        Name = maps:get(name, Conf),
-                        Servers2 = Servers#{Name => Server},
-                        {ResAcc, update_servers(Servers2, StateAcc)};
-                    {Err, StateAcc1} ->
-                        {[Err | ResAcc], StateAcc1}
-                end
-            end,
-            {[], State3},
-            Added
-        ),
-    %% update order
-    #{servers := Servers4} = State4,
+    State2 = remove_servers(Removed, State),
+    {UpdateRes, State3} = restart_servers(Updated, NewConfL, State2),
+    {AddRes, State4 = #{servers := Servers4}} = add_servers(Added, State3),
     State5 = State4#{servers => reorder(NewConfL, Servers4)},
-    case lists:append([UpdateRes, AddRes]) of
-        [] -> {reply, ok, State5};
-        _ -> {reply, {error, #{added => AddRes, updated => UpdateRes}}, State5}
+    case UpdateRes =:= [] andalso AddRes =:= [] of
+        true -> {reply, ok, State5};
+        false -> {reply, {error, #{added => AddRes, updated => UpdateRes}}, State5}
     end;
 handle_call({lookup, Name}, _From, State) ->
     {reply, where_is_server(Name, State), State};
@@ -340,6 +288,16 @@ handle_call(_Request, _From, State) ->
     Reply = ok,
     {reply, Reply, State}.
 
+remove_servers(Removes, State) ->
+    lists:foldl(
+        fun(Conf, Acc) ->
+            ToDelete = maps:get(name, Conf),
+            remove_server(ToDelete, Acc)
+        end,
+        State,
+        Removes
+    ).
+
 remove_server(ToDelete, State) ->
     emqx_exhook_metrics:on_server_deleted(ToDelete),
     #{servers := Servers} = State2 = do_unload_server(ToDelete, State),
@@ -369,8 +327,8 @@ terminate(Reason, State = #{servers := Servers}) ->
         Servers
     ),
     ?tp(info, exhook_mgr_terminated, #{reason => Reason, servers => Servers}),
-    emqx_conf:remove_handler([exhook, servers]),
-    emqx_conf:remove_handler([exhook]),
+    emqx_conf:remove_handler(?SERVERS),
+    emqx_conf:remove_handler(?EXHOOK),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->
@@ -386,6 +344,22 @@ unload_exhooks() ->
      || {Name, {M, F, _A}} <- ?ENABLED_HOOKS
     ].
 
+add_servers(Added, State) ->
+    lists:foldl(
+        fun(Conf = #{name := Name}, {ResAcc, StateAcc}) ->
+            case do_load_server(options_to_server(Conf)) of
+                {ok, Server} ->
+                    #{servers := Servers} = StateAcc,
+                    Servers2 = Servers#{Name => Server},
+                    {ResAcc, update_servers(Servers2, StateAcc)};
+                {Err, StateAcc1} ->
+                    {[Err | ResAcc], StateAcc1}
+            end
+        end,
+        {[], State},
+        Added
+    ).
+
 do_load_server(#{name := Name} = Server) ->
     case emqx_exhook_server:load(Name, Server) of
         {ok, ServerState} ->
@@ -462,8 +436,7 @@ clean_reload_timer(#{timer := Timer}) ->
     _ = erlang:cancel_timer(Timer),
     ok.
 
--spec do_move(binary(), position(), list(server_options())) ->
-    not_found | list(server_options()).
+-spec do_move(binary(), position(), list(server_options())) -> list(server_options()).
 do_move(Name, Position, ConfL) ->
     move(ConfL, Name, Position, []).
 
@@ -472,7 +445,7 @@ move([#{<<"name">> := Name} = Server | T], Name, Position, HeadL) ->
 move([Server | T], Name, Position, HeadL) ->
     move(T, Name, Position, [Server | HeadL]);
 move([], _Name, _Position, _HeadL) ->
-    not_found.
+    throw(not_found).
 
 move_to(?CMD_MOVE_FRONT, Server, ServerL) ->
     [Server | ServerL];
@@ -490,8 +463,7 @@ move_to([H | T], Position, Server, HeadL) ->
 move_to([], _Position, _Server, _HeadL) ->
     not_found.
 
--spec do_delete(binary(), list(server_options())) ->
-    not_found | list(server_options()).
+-spec do_delete(binary(), list(server_options())) -> list(server_options()).
 do_delete(ToDelete, OldConf) ->
     case lists:any(fun(#{<<"name">> := ExistedName}) -> ExistedName =:= ToDelete end, OldConf) of
         true ->
@@ -500,7 +472,7 @@ do_delete(ToDelete, OldConf) ->
                 OldConf
             );
         false ->
-            not_found
+            throw(not_found)
     end.
 
 -spec reorder(list(server_options()), servers()) -> servers().
@@ -532,9 +504,7 @@ where_is_server(Name, #{servers := Servers}) ->
 
 -type replace_fun() :: fun((server_options()) -> server_options()).
 
--spec replace_conf(binary(), replace_fun(), list(server_options())) ->
-    not_found
-    | list(server_options()).
+-spec replace_conf(binary(), replace_fun(), list(server_options())) -> list(server_options()).
 replace_conf(Name, ReplaceFun, ConfL) ->
     replace_conf(ConfL, Name, ReplaceFun, []).
 
@@ -544,7 +514,20 @@ replace_conf([#{<<"name">> := Name} = H | T], Name, ReplaceFun, HeadL) ->
 replace_conf([H | T], Name, ReplaceFun, HeadL) ->
     replace_conf(T, Name, ReplaceFun, [H | HeadL]);
 replace_conf([], _, _, _) ->
-    not_found.
+    throw(not_found).
+
+restart_servers(Updated, NewConfL, State) ->
+    lists:foldl(
+        fun({_Old, Conf}, {ResAcc, StateAcc}) ->
+            Name = maps:get(name, Conf),
+            case restart_server(Name, NewConfL, StateAcc) of
+                {ok, StateAcc1} -> {ResAcc, StateAcc1};
+                {Err, StateAcc1} -> {[Err | ResAcc], StateAcc1}
+            end
+        end,
+        {[], State},
+        Updated
+    ).
 
 -spec restart_server(binary(), list(server_options()), state()) ->
     {ok, state()}
@@ -673,7 +656,7 @@ try_clear_ssl_files({Op, Name, _}, NewConfs, OldConfs) when
     NewSSL = find_server_ssl_cfg(Name, NewConfs),
     OldSSL = find_server_ssl_cfg(Name, OldConfs),
     emqx_tls_lib:delete_ssl_files(ssl_file_path(Name), NewSSL, OldSSL);
-%% replace the whole config from the file
+%% replace the whole config from the cli
 try_clear_ssl_files(_Req, #{servers := NewServers}, #{servers := OldServers}) ->
     lists:foreach(
         fun(#{name := Name} = Conf) ->
@@ -682,7 +665,9 @@ try_clear_ssl_files(_Req, #{servers := NewServers}, #{servers := OldServers}) ->
             emqx_tls_lib:delete_ssl_files(ssl_file_path(Name), NewSSL, OldSSL)
         end,
         OldServers
-    ).
+    );
+try_clear_ssl_files(_Req, _NewConf, _OldConf) ->
+    ok.
 
 search_server_cfg(Name, Confs) ->
     lists:search(
diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
index ff313c8c8..d9297fff6 100644
--- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
+++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
@@ -196,9 +196,9 @@ t_error_update_conf(_) ->
     Path = [exhook, servers],
     Name = <<"error_update">>,
     ErrorCfg = #{<<"name">> => Name},
-    {error, _} = emqx_exhook_mgr:update_config(Path, {update, Name, ErrorCfg}),
-    {error, _} = emqx_exhook_mgr:update_config(Path, {move, Name, top, <<>>}),
-    {error, _} = emqx_exhook_mgr:update_config(Path, {enable, Name, true}),
+    {error, not_found} = emqx_exhook_mgr:update_config(Path, {update, Name, ErrorCfg}),
+    {error, not_found} = emqx_exhook_mgr:update_config(Path, {move, Name, top}),
+    {error, not_found} = emqx_exhook_mgr:update_config(Path, {enable, Name, true}),
 
     ErrorAnd = #{<<"name">> => Name, <<"url">> => <<"http://127.0.0.1:9001">>},
     {ok, _} = emqx_exhook_mgr:update_config(Path, {add, ErrorAnd}),
@@ -210,12 +210,37 @@ t_error_update_conf(_) ->
     },
     {ok, _} = emqx_exhook_mgr:update_config(Path, {update, Name, DisableAnd}),
 
-    {ok, _} = emqx_exhook_mgr:update_config(Path, {delete, <<"error">>}),
-    {error, not_found} = emqx_exhook_mgr:update_config(
-        Path, {delete, <<"delete_not_exists">>}
-    ),
+    {ok, _} = emqx_exhook_mgr:update_config(Path, {delete, Name}),
+    {error, not_found} = emqx_exhook_mgr:update_config(Path, {delete, Name}),
     ok.
 
+t_update_conf(_Config) ->
+    Path = [exhook],
+    Conf = #{<<"servers">> := Servers} = emqx_config:get_raw(Path),
+    ?assert(length(Servers) > 1),
+    Servers1 = shuffle(Servers),
+    ReOrderedConf = Conf#{<<"servers">> => Servers1},
+    validate_servers(Path, ReOrderedConf, Servers1),
+    [_ | Servers2] = Servers,
+    DeletedConf = Conf#{<<"servers">> => Servers2},
+    validate_servers(Path, DeletedConf, Servers2),
+    [L1, L2 | Servers3] = Servers,
+    UpdateL2 = L2#{<<"pool_size">> => 1, <<"request_timeout">> => 1000},
+    UpdatedServers = [L1, UpdateL2 | Servers3],
+    UpdatedConf = Conf#{<<"servers">> => UpdatedServers},
+    validate_servers(Path, UpdatedConf, UpdatedServers),
+    %% reset
+    validate_servers(Path, Conf, Servers),
+    ok.
+
+validate_servers(Path, ReOrderConf, Servers1) ->
+    {ok, _} = emqx_exhook_mgr:update_config(Path, ReOrderConf),
+    ?assertEqual(ReOrderConf, emqx_config:get_raw(Path)),
+    List = emqx_exhook_mgr:list(),
+    ExpectL = lists:map(fun(#{<<"name">> := Name}) -> Name end, Servers1),
+    L1 = lists:map(fun(#{name := Name}) -> Name end, List),
+    ?assertEqual(ExpectL, L1).
+
 t_error_server_info(_) ->
     not_found = emqx_exhook_mgr:server_info(<<"not_exists">>),
     ok.
@@ -490,6 +515,10 @@ data_file(Name) ->
 cert_file(Name) ->
     data_file(filename:join(["certs", Name])).
 
-%% FIXME: this creats inter-test dependency
+%% FIXME: this creates inter-test dependency
 stop_apps(Apps) ->
     emqx_common_test_helpers:stop_apps(Apps, #{erase_all_configs => false}).
+
+shuffle(List) ->
+    Sorted = lists:sort(lists:map(fun(L) -> {rand:uniform(), L} end, List)),
+    lists:map(fun({_, L}) -> L end, Sorted).

From d0d6992a1471bb90b48bba873bc44b6e84a58155 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9F=90=E6=96=87?= <zhongwencool@gmail.com>
Date: Sun, 4 Jun 2023 20:07:33 +0800
Subject: [PATCH 10/29] feat: update listeners from cli

---
 apps/emqx/src/emqx_listeners.erl              | 183 +++++++++++++-----
 .../emqx/test/emqx_listeners_update_SUITE.erl | 154 +++++++++++++++
 2 files changed, 288 insertions(+), 49 deletions(-)
 create mode 100644 apps/emqx/test/emqx_listeners_update_SUITE.erl

diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index acdb9ff96..c0116fd8c 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -57,6 +57,7 @@
 ]).
 
 -export([pre_config_update/3, post_config_update/5]).
+-export([create_listener/3, remove_listener/3, update_listener/3]).
 
 -export([format_bind/1]).
 
@@ -65,8 +66,8 @@
 -endif.
 
 -type listener_id() :: atom() | binary().
-
--define(CONF_KEY_PATH, [listeners, '?', '?']).
+-define(ROOT_KEY, listeners).
+-define(CONF_KEY_PATH, [?ROOT_KEY, '?', '?']).
 -define(TYPES_STRING, ["tcp", "ssl", "ws", "wss", "quic"]).
 -define(MARK_DEL, ?TOMBSTONE_CONFIG_CHANGE_REQ).
 
@@ -212,7 +213,10 @@ shutdown_count(_, _, _) ->
 start() ->
     %% The ?MODULE:start/0 will be called by emqx_app when emqx get started,
     %% so we install the config handler here.
+    %% callback when http api request
     ok = emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE),
+    %% callback when reload from config file
+    ok = emqx_config_handler:add_handler([?ROOT_KEY], ?MODULE),
     foreach_listeners(fun start_listener/3).
 
 -spec start_listener(listener_id()) -> ok | {error, term()}.
@@ -287,7 +291,8 @@ restart_listener(Type, ListenerName, OldConf, NewConf) ->
 stop() ->
     %% The ?MODULE:stop/0 will be called by emqx_app when emqx is going to shutdown,
     %% so we uninstall the config handler here.
-    _ = emqx_config_handler:remove_handler(?CONF_KEY_PATH),
+    ok = emqx_config_handler:remove_handler(?CONF_KEY_PATH),
+    ok = emqx_config_handler:remove_handler([?ROOT_KEY]),
     foreach_listeners(fun stop_listener/3).
 
 -spec stop_listener(listener_id()) -> ok | {error, term()}.
@@ -463,50 +468,34 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) ->
     end.
 
 %% Update the listeners at runtime
-pre_config_update([listeners, Type, Name], {create, NewConf}, V) when
+pre_config_update([?ROOT_KEY, Type, Name], {create, NewConf}, V) when
     V =:= undefined orelse V =:= ?TOMBSTONE_VALUE
 ->
-    CertsDir = certs_dir(Type, Name),
-    {ok, convert_certs(CertsDir, NewConf)};
-pre_config_update([listeners, _Type, _Name], {create, _NewConf}, _RawConf) ->
+    {ok, convert_certs(Type, Name, NewConf)};
+pre_config_update([?ROOT_KEY, _Type, _Name], {create, _NewConf}, _RawConf) ->
     {error, already_exist};
-pre_config_update([listeners, _Type, _Name], {update, _Request}, undefined) ->
+pre_config_update([?ROOT_KEY, _Type, _Name], {update, _Request}, undefined) ->
     {error, not_found};
-pre_config_update([listeners, Type, Name], {update, Request}, RawConf) ->
-    NewConfT = emqx_utils_maps:deep_merge(RawConf, Request),
-    NewConf = ensure_override_limiter_conf(NewConfT, Request),
-    CertsDir = certs_dir(Type, Name),
-    {ok, convert_certs(CertsDir, NewConf)};
-pre_config_update([listeners, _Type, _Name], {action, _Action, Updated}, RawConf) ->
-    NewConf = emqx_utils_maps:deep_merge(RawConf, Updated),
-    {ok, NewConf};
-pre_config_update([listeners, _Type, _Name], ?MARK_DEL, _RawConf) ->
+pre_config_update([?ROOT_KEY, Type, Name], {update, Request}, RawConf) ->
+    RawConf1 = emqx_utils_maps:deep_merge(RawConf, Request),
+    RawConf2 = ensure_override_limiter_conf(RawConf1, Request),
+    {ok, convert_certs(Type, Name, RawConf2)};
+pre_config_update([?ROOT_KEY, _Type, _Name], {action, _Action, Updated}, RawConf) ->
+    {ok, emqx_utils_maps:deep_merge(RawConf, Updated)};
+pre_config_update([?ROOT_KEY, _Type, _Name], ?MARK_DEL, _RawConf) ->
     {ok, ?TOMBSTONE_VALUE};
-pre_config_update(_Path, _Request, RawConf) ->
-    {ok, RawConf}.
+pre_config_update([?ROOT_KEY], RawConf, RawConf) ->
+    {ok, RawConf};
+pre_config_update([?ROOT_KEY], NewConf, _RawConf) ->
+    {ok, convert_certs(NewConf)}.
 
-post_config_update([listeners, Type, Name], {create, _Request}, NewConf, undefined, _AppEnvs) ->
-    start_listener(Type, Name, NewConf);
-post_config_update([listeners, Type, Name], {update, _Request}, NewConf, OldConf, _AppEnvs) ->
-    try_clear_ssl_files(certs_dir(Type, Name), NewConf, OldConf),
-    ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf),
-    case NewConf of
-        #{enabled := true} -> restart_listener(Type, Name, {OldConf, NewConf});
-        _ -> ok
-    end;
-post_config_update([listeners, Type, Name], Op, _, OldConf, _AppEnvs) when
-    Op =:= ?MARK_DEL andalso is_map(OldConf)
-->
-    ok = unregister_ocsp_stapling_refresh(Type, Name),
-    case stop_listener(Type, Name, OldConf) of
-        ok ->
-            _ = emqx_authentication:delete_chain(listener_id(Type, Name)),
-            CertsDir = certs_dir(Type, Name),
-            clear_certs(CertsDir, OldConf);
-        Err ->
-            Err
-    end;
-post_config_update([listeners, Type, Name], {action, _Action, _}, NewConf, OldConf, _AppEnvs) ->
+post_config_update([?ROOT_KEY, Type, Name], {create, _Request}, NewConf, undefined, _AppEnvs) ->
+    create_listener(Type, Name, NewConf);
+post_config_update([?ROOT_KEY, Type, Name], {update, _Request}, NewConf, OldConf, _AppEnvs) ->
+    update_listener(Type, Name, {OldConf, NewConf});
+post_config_update([?ROOT_KEY, Type, Name], ?MARK_DEL, _, OldConf = #{}, _AppEnvs) ->
+    remove_listener(Type, Name, OldConf);
+post_config_update([?ROOT_KEY, Type, Name], {action, _Action, _}, NewConf, OldConf, _AppEnvs) ->
     #{enabled := NewEnabled} = NewConf,
     #{enabled := OldEnabled} = OldConf,
     case {NewEnabled, OldEnabled} of
@@ -523,9 +512,65 @@ post_config_update([listeners, Type, Name], {action, _Action, _}, NewConf, OldCo
             ok = unregister_ocsp_stapling_refresh(Type, Name),
             stop_listener(Type, Name, OldConf)
     end;
+post_config_update([?ROOT_KEY], _Request, OldConf, OldConf, _AppEnvs) ->
+    ok;
+post_config_update([?ROOT_KEY], _Request, NewConf, OldConf, _AppEnvs) ->
+    #{added := Added, removed := Removed, changed := Changed} = diff_confs(NewConf, OldConf),
+    Updated = lists:map(fun({{{T, N}, Old}, {_, New}}) -> {{T, N}, {Old, New}} end, Changed),
+    perform_listener_changes([
+        {fun ?MODULE:remove_listener/3, Removed},
+        {fun ?MODULE:update_listener/3, Updated},
+        {fun ?MODULE:create_listener/3, Added}
+    ]);
 post_config_update(_Path, _Request, _NewConf, _OldConf, _AppEnvs) ->
     ok.
 
+create_listener(Type, Name, NewConf) ->
+    Res = start_listener(Type, Name, NewConf),
+    recreate_authenticator(Res, Type, Name, NewConf).
+
+recreate_authenticator(ok, Type, Name, Conf) ->
+    Chain = listener_id(Type, Name),
+    _ = emqx_authentication:delete_chain(Chain),
+    case maps:get(authentication, Conf, []) of
+        [] -> ok;
+        AuthN -> emqx_authentication:create_authenticator(Chain, AuthN)
+    end;
+recreate_authenticator(Error, _Type, _Name, _NewConf) ->
+    Error.
+
+remove_listener(Type, Name, OldConf) ->
+    ok = unregister_ocsp_stapling_refresh(Type, Name),
+    case stop_listener(Type, Name, OldConf) of
+        ok ->
+            _ = emqx_authentication:delete_chain(listener_id(Type, Name)),
+            clear_certs(certs_dir(Type, Name), OldConf);
+        Err ->
+            Err
+    end.
+
+update_listener(Type, Name, {OldConf, NewConf}) ->
+    try_clear_ssl_files(certs_dir(Type, Name), NewConf, OldConf),
+    ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf),
+    Res = restart_listener(Type, Name, {OldConf, NewConf}),
+    recreate_authenticator(Res, Type, Name, NewConf).
+
+perform_listener_changes([]) ->
+    ok;
+perform_listener_changes([{Action, ConfL} | Tasks]) ->
+    case perform_listener_changes(Action, ConfL) of
+        ok -> perform_listener_changes(Tasks);
+        {error, Reason} -> {error, Reason}
+    end.
+
+perform_listener_changes(_Action, []) ->
+    ok;
+perform_listener_changes(Action, [{{Type, Name}, Diff} | MapConf]) ->
+    case Action(Type, Name, Diff) of
+        ok -> perform_listener_changes(Action, MapConf);
+        {error, Reason} -> {error, Reason}
+    end.
+
 esockd_opts(ListenerId, Type, Opts0) ->
     Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0),
     Limiter = limiter(Opts0),
@@ -701,6 +746,29 @@ del_limiter_bucket(Id, Conf) ->
             )
     end.
 
+diff_confs(NewConfs, OldConfs) ->
+    emqx_utils:diff_lists(
+        flatten_confs(NewConfs),
+        flatten_confs(OldConfs),
+        fun({Key, _}) -> Key end
+    ).
+
+flatten_confs(Conf0) ->
+    lists:flatmap(
+        fun({Type, Conf}) ->
+            do_flatten_confs(Type, Conf)
+        end,
+        maps:to_list(Conf0)
+    ).
+
+do_flatten_confs(Type, Conf0) ->
+    FilterFun =
+        fun
+            ({_Name, ?TOMBSTONE_TYPE}) -> false;
+            ({Name, Conf}) -> {true, {{Type, Name}, Conf}}
+        end,
+    lists:filtermap(FilterFun, maps:to_list(Conf0)).
+
 enable_authn(Opts) ->
     maps:get(enable_authn, Opts, true).
 
@@ -762,14 +830,32 @@ parse_bind(#{<<"bind">> := Bind}) ->
 certs_dir(Type, Name) ->
     iolist_to_binary(filename:join(["listeners", Type, Name])).
 
-convert_certs(CertsDir, Conf) ->
+convert_certs(ListenerConf) ->
+    maps:fold(
+        fun(Type, Listeners0, Acc) ->
+            Listeners1 =
+                maps:fold(
+                    fun(Name, Conf, Acc1) ->
+                        Acc1#{Name => convert_certs(Type, Name, Conf)}
+                    end,
+                    #{},
+                    Listeners0
+                ),
+            Acc#{Type => Listeners1}
+        end,
+        #{},
+        ListenerConf
+    ).
+
+convert_certs(Type, Name, Conf) ->
+    CertsDir = certs_dir(Type, Name),
     case emqx_tls_lib:ensure_ssl_files(CertsDir, get_ssl_options(Conf)) of
         {ok, undefined} ->
             Conf;
         {ok, SSL} ->
             Conf#{<<"ssl_options">> => SSL};
         {error, Reason} ->
-            ?SLOG(error, Reason#{msg => "bad_ssl_config"}),
+            ?SLOG(error, Reason#{msg => "bad_ssl_config", type => Type, name => Name}),
             throw({bad_ssl_config, Reason})
     end.
 
@@ -791,13 +877,15 @@ try_clear_ssl_files(CertsDir, NewConf, OldConf) ->
     OldSSL = get_ssl_options(OldConf),
     emqx_tls_lib:delete_ssl_files(CertsDir, NewSSL, OldSSL).
 
-get_ssl_options(Conf) ->
+get_ssl_options(Conf = #{}) ->
     case maps:find(ssl_options, Conf) of
         {ok, SSL} ->
             SSL;
         error ->
             maps:get(<<"ssl_options">>, Conf, undefined)
-    end.
+    end;
+get_ssl_options(_) ->
+    undefined.
 
 %% @doc Get QUIC optional settings for low level tunings.
 %% @see quicer:quic_settings()
@@ -889,8 +977,5 @@ unregister_ocsp_stapling_refresh(Type, Name) ->
     emqx_ocsp_cache:unregister_listener(ListenerId),
     ok.
 
-%% There is currently an issue with frontend
-%% infinity is not a good value for it, so we use 5m for now
 default_max_conn() ->
-    %% TODO: <<"infinity">>
-    5_000_000.
+    <<"infinity">>.
diff --git a/apps/emqx/test/emqx_listeners_update_SUITE.erl b/apps/emqx/test/emqx_listeners_update_SUITE.erl
new file mode 100644
index 000000000..c16a26f3a
--- /dev/null
+++ b/apps/emqx/test/emqx_listeners_update_SUITE.erl
@@ -0,0 +1,154 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_listeners_update_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_schema.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-import(emqx_listeners, [current_conns/2, is_running/1]).
+
+-define(LISTENERS, [listeners]).
+
+all() -> emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_common_test_helpers:boot_modules(all),
+    emqx_common_test_helpers:start_apps([]),
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_common_test_helpers:stop_apps([]).
+
+init_per_testcase(_TestCase, Config) ->
+    Init = emqx:get_raw_config(?LISTENERS),
+    [{init_conf, Init} | Config].
+
+end_per_testcase(_TestCase, Config) ->
+    Conf = ?config(init_conf, Config),
+    {ok, _} = emqx:update_config(?LISTENERS, Conf),
+    ok.
+
+t_default_conf(_Config) ->
+    ?assertMatch(
+        #{
+            <<"tcp">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:1883">>}},
+            <<"ssl">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8883">>}},
+            <<"ws">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8083">>}},
+            <<"wss">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8084">>}}
+        },
+        emqx:get_raw_config(?LISTENERS)
+    ),
+    ?assertMatch(
+        #{
+            tcp := #{default := #{bind := {{0, 0, 0, 0}, 1883}}},
+            ssl := #{default := #{bind := {{0, 0, 0, 0}, 8883}}},
+            ws := #{default := #{bind := {{0, 0, 0, 0}, 8083}}},
+            wss := #{default := #{bind := {{0, 0, 0, 0}, 8084}}}
+        },
+        emqx:get_config(?LISTENERS)
+    ),
+    ok.
+
+t_update_conf(_Conf) ->
+    Raw = emqx:get_raw_config(?LISTENERS),
+    Raw1 = emqx_utils_maps:deep_put(
+        [<<"tcp">>, <<"default">>, <<"bind">>], Raw, <<"127.0.0.1:1883">>
+    ),
+    Raw2 = emqx_utils_maps:deep_put(
+        [<<"ssl">>, <<"default">>, <<"bind">>], Raw1, <<"127.0.0.1:8883">>
+    ),
+    Raw3 = emqx_utils_maps:deep_put(
+        [<<"ws">>, <<"default">>, <<"bind">>], Raw2, <<"0.0.0.0:8083">>
+    ),
+    Raw4 = emqx_utils_maps:deep_put(
+        [<<"wss">>, <<"default">>, <<"bind">>], Raw3, <<"127.0.0.1:8084">>
+    ),
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw4)),
+    ?assertMatch(
+        #{
+            <<"tcp">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:1883">>}},
+            <<"ssl">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:8883">>}},
+            <<"ws">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8083">>}},
+            <<"wss">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:8084">>}}
+        },
+        emqx:get_raw_config(?LISTENERS)
+    ),
+    BindTcp = {{127, 0, 0, 1}, 1883},
+    BindSsl = {{127, 0, 0, 1}, 8883},
+    BindWs = {{0, 0, 0, 0}, 8083},
+    BindWss = {{127, 0, 0, 1}, 8084},
+    ?assertMatch(
+        #{
+            tcp := #{default := #{bind := BindTcp}},
+            ssl := #{default := #{bind := BindSsl}},
+            ws := #{default := #{bind := BindWs}},
+            wss := #{default := #{bind := BindWss}}
+        },
+        emqx:get_config(?LISTENERS)
+    ),
+    ?assertError(not_found, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})),
+    ?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+
+    ?assertEqual(0, current_conns(<<"tcp:default">>, BindTcp)),
+    ?assertEqual(0, current_conns(<<"ssl:default">>, BindSsl)),
+
+    ?assertEqual({0, 0, 0, 0}, proplists:get_value(ip, ranch:info('ws:default'))),
+    ?assertEqual({127, 0, 0, 1}, proplists:get_value(ip, ranch:info('wss:default'))),
+    ?assert(is_running('ws:default')),
+    ?assert(is_running('wss:default')),
+    ok.
+
+t_add_delete_conf(_Conf) ->
+    Raw = emqx:get_raw_config(?LISTENERS),
+    %% add
+    #{<<"tcp">> := #{<<"default">> := Tcp}} = Raw,
+    NewBind = <<"127.0.0.1:1987">>,
+    Raw1 = emqx_utils_maps:deep_put([<<"tcp">>, <<"new">>], Raw, Tcp#{<<"bind">> => NewBind}),
+    Raw2 = emqx_utils_maps:deep_put([<<"ssl">>, <<"default">>], Raw1, ?TOMBSTONE_VALUE),
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw2)),
+    ?assertEqual(0, current_conns(<<"tcp:new">>, {{127, 0, 0, 1}, 1987})),
+    ?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+    %% deleted
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw)),
+    ?assertError(not_found, current_conns(<<"tcp:new">>, {{127, 0, 0, 1}, 1987})),
+    ?assertEqual(0, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+    ok.
+
+t_delete_default_conf(_Conf) ->
+    Raw = emqx:get_raw_config(?LISTENERS),
+    %% delete default listeners
+    Raw1 = emqx_utils_maps:deep_put([<<"tcp">>, <<"default">>], Raw, ?TOMBSTONE_VALUE),
+    Raw2 = emqx_utils_maps:deep_put([<<"ssl">>, <<"default">>], Raw1, ?TOMBSTONE_VALUE),
+    Raw3 = emqx_utils_maps:deep_put([<<"ws">>, <<"default">>], Raw2, ?TOMBSTONE_VALUE),
+    Raw4 = emqx_utils_maps:deep_put([<<"wss">>, <<"default">>], Raw3, ?TOMBSTONE_VALUE),
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw4)),
+    ?assertError(not_found, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})),
+    ?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+    ?assertMatch({error, not_found}, is_running('ws:default')),
+    ?assertMatch({error, not_found}, is_running('wss:default')),
+
+    %% reset
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw)),
+    ?assertEqual(0, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})),
+    ?assertEqual(0, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+    ?assert(is_running('ws:default')),
+    ?assert(is_running('wss:default')),
+    ok.

From bdcc069aac5c0b31666a4e7d63854d44620d6cae Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 5 Jun 2023 14:58:33 +0200
Subject: [PATCH 11/29] chore: start deprecating mcast cluster discovery

---
 apps/emqx_conf/src/emqx_conf_schema.erl | 4 ++--
 rel/i18n/emqx_conf_schema.hocon         | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index b4ba2bf47..f339f78a3 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -143,7 +143,7 @@ fields("cluster") ->
             )},
         {"discovery_strategy",
             sc(
-                hoconsc:enum([manual, static, mcast, dns, etcd, k8s]),
+                hoconsc:enum([manual, static, dns, etcd, k8s, mcast]),
                 #{
                     default => manual,
                     desc => ?DESC(cluster_discovery_strategy),
@@ -198,7 +198,7 @@ fields("cluster") ->
         {"mcast",
             sc(
                 ?R_REF(cluster_mcast),
-                #{}
+                #{importance => ?IMPORTANCE_HIDDEN}
             )},
         {"dns",
             sc(
diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon
index 079132454..416b3ca76 100644
--- a/rel/i18n/emqx_conf_schema.hocon
+++ b/rel/i18n/emqx_conf_schema.hocon
@@ -755,7 +755,9 @@ cluster_discovery_strategy.desc:
 - static: Configure static nodes list by setting <code>seeds</code> in config file.<br/>
 - dns: Use DNS A record to discover peer nodes.<br/>
 - etcd: Use etcd to discover peer nodes.<br/>
-- k8s: Use Kubernetes API to discover peer pods."""
+- k8s: Use Kubernetes API to discover peer pods.
+- mcast: Deprecated since 5.1, will be removed in 5.2.
+  This supports discovery via UDP multicast."""
 
 cluster_discovery_strategy.label:
 """Cluster Discovery Strategy"""

From 1dd2109b1fcbc8d787e6132cd9cb3eb166b3a418 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 5 Jun 2023 15:03:21 +0200
Subject: [PATCH 12/29] docs: add changelog for 10943

---
 changes/ce/fix-10943.en.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 changes/ce/fix-10943.en.md

diff --git a/changes/ce/fix-10943.en.md b/changes/ce/fix-10943.en.md
new file mode 100644
index 000000000..081f94c24
--- /dev/null
+++ b/changes/ce/fix-10943.en.md
@@ -0,0 +1,5 @@
+Deprecated UDP mcast mechanism for cluster discovery.
+
+This feature has been planed for deprecation since 5.0 mainly due to the lack of
+actual production use.
+This feature code is not yet removed in 5.1, but the document interface is demoted.

From 3caf114f6890670d13535fa5b0b7293069e1d23d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 5 Jun 2023 23:00:41 +0200
Subject: [PATCH 13/29] test: fix test case is_list(Config) function guard

---
 apps/emqx/test/emqx_config_SUITE.erl | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl
index 13bcc2601..765d0ced8 100644
--- a/apps/emqx/test/emqx_config_SUITE.erl
+++ b/apps/emqx/test/emqx_config_SUITE.erl
@@ -37,9 +37,8 @@ init_per_testcase(TestCase, Config) ->
         ?MODULE:TestCase({init, Config})
     catch
         error:function_clause ->
-            ok
-    end,
-    Config.
+            Config
+    end.
 
 end_per_testcase(TestCase, Config) ->
     try
@@ -236,7 +235,7 @@ t_init_zones_with_user_defined_default_zone(Config) when is_list(Config) ->
     %% Then others are defaults
     ?assertEqual(ExpectedOthers, Others).
 
-t_init_zones_with_user_defined_other_zone(Config) ->
+t_init_zones_with_user_defined_other_zone(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given user defined config for default zone
     ConfFile = prepare_conf_file(

From 47d7e6ce016181931583649db43b18333a8748d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9F=90=E6=96=87?= <zhongwencool@gmail.com>
Date: Mon, 5 Jun 2023 15:55:54 +0800
Subject: [PATCH 14/29] feat: update gateway from cli

---
 apps/emqx_gateway/src/emqx_gateway.erl        |   2 +-
 apps/emqx_gateway/src/emqx_gateway_conf.erl   | 294 ++++++++++++++----
 .../emqx_gateway/src/emqx_gateway_metrics.erl |   2 +-
 .../test/emqx_gateway_conf_SUITE.erl          | 188 +++++++++++
 .../test/emqx_gateway_test_utils.erl          |   8 +-
 5 files changed, 425 insertions(+), 69 deletions(-)

diff --git a/apps/emqx_gateway/src/emqx_gateway.erl b/apps/emqx_gateway/src/emqx_gateway.erl
index 0e6871db2..0d5ea79b1 100644
--- a/apps/emqx_gateway/src/emqx_gateway.erl
+++ b/apps/emqx_gateway/src/emqx_gateway.erl
@@ -16,7 +16,7 @@
 
 -module(emqx_gateway).
 
--include("include/emqx_gateway.hrl").
+-include("emqx_gateway.hrl").
 
 %% Gateway APIs
 -export([
diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl
index da86d6a58..e7b8a1c65 100644
--- a/apps/emqx_gateway/src/emqx_gateway_conf.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl
@@ -74,18 +74,20 @@
 -type listener_ref() :: {ListenerType :: atom_or_bin(), ListenerName :: atom_or_bin()}.
 
 -define(IS_SSL(T), (T == <<"ssl_options">> orelse T == <<"dtls_options">>)).
+-define(IGNORE_KEYS, [<<"listeners">>, ?AUTHN_BIN]).
 
 %%--------------------------------------------------------------------
 %%  Load/Unload
 %%--------------------------------------------------------------------
+-define(GATEWAY, [gateway]).
 
 -spec load() -> ok.
 load() ->
-    emqx_conf:add_handler([gateway], ?MODULE).
+    emqx_conf:add_handler(?GATEWAY, ?MODULE).
 
 -spec unload() -> ok.
 unload() ->
-    emqx_conf:remove_handler([gateway]).
+    emqx_conf:remove_handler(?GATEWAY).
 
 %%--------------------------------------------------------------------
 %% APIs
@@ -104,7 +106,7 @@ unconvert_listeners(Ls) when is_list(Ls) ->
     lists:foldl(
         fun(Lis, Acc) ->
             {[Type, Name], Lis1} = maps_key_take([<<"type">>, <<"name">>], Lis),
-            _ = vaildate_listener_name(Name),
+            _ = validate_listener_name(Name),
             NLis1 = maps:without([<<"id">>, <<"running">>], Lis1),
             emqx_utils_maps:deep_merge(Acc, #{Type => #{Name => NLis1}})
         end,
@@ -122,7 +124,7 @@ maps_key_take([K | Ks], M, Acc) ->
         {V, M1} -> maps_key_take(Ks, M1, [V | Acc])
     end.
 
-vaildate_listener_name(Name) ->
+validate_listener_name(Name) ->
     try
         {match, _} = re:run(Name, "^[0-9a-zA-Z_-]+$"),
         ok
@@ -373,7 +375,7 @@ ret_listener_or_err(_, _, Err) ->
     emqx_config:raw_config()
 ) ->
     {ok, emqx_config:update_request()} | {error, term()}.
-pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) ->
+pre_config_update(?GATEWAY, {load_gateway, GwName, Conf}, RawConf) ->
     case maps:get(GwName, RawConf, undefined) of
         undefined ->
             NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf),
@@ -381,29 +383,25 @@ pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) ->
         _ ->
             badres_gateway(already_exist, GwName)
     end;
-pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) ->
+pre_config_update(?GATEWAY, {update_gateway, GwName, Conf}, RawConf) ->
     case maps:get(GwName, RawConf, undefined) of
         undefined ->
             badres_gateway(not_found, GwName);
         GwRawConf ->
-            Conf1 = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf),
+            Conf1 = maps:without(?IGNORE_KEYS, Conf),
             NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf1),
             NConf1 = maps:merge(GwRawConf, NConf),
             {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf1)}
     end;
-pre_config_update(_, {unload_gateway, GwName}, RawConf) ->
+pre_config_update(?GATEWAY, {unload_gateway, GwName}, RawConf) ->
     _ = tune_gw_certs(
         fun clear_certs/2,
         GwName,
         maps:get(GwName, RawConf, #{})
     ),
     {ok, maps:remove(GwName, RawConf)};
-pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, <<"listeners">>, LType, LName], RawConf, undefined
-        )
-    of
+pre_config_update(?GATEWAY, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             NConf = convert_certs(certs_dir(GwName), Conf),
             NListener = #{LType => #{LName => NConf}},
@@ -415,12 +413,8 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
         _ ->
             badres_listener(already_exist, GwName, LType, LName)
     end;
-pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, <<"listeners">>, LType, LName], RawConf, undefined
-        )
-    of
+pre_config_update(?GATEWAY, {update_listener, GwName, {LType, LName}, Conf}, RawConf) ->
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             badres_listener(not_found, GwName, LType, LName);
         OldConf ->
@@ -432,21 +426,17 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) -
             ),
             {ok, NRawConf}
     end;
-pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) ->
-    Path = [GwName, <<"listeners">>, LType, LName],
-    case emqx_utils_maps:deep_get(Path, RawConf, undefined) of
+pre_config_update(?GATEWAY, {remove_listener, GwName, {LType, LName}}, RawConf) ->
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             {ok, RawConf};
         OldConf ->
             clear_certs(certs_dir(GwName), OldConf),
+            Path = [GwName, <<"listeners">>, LType, LName],
             {ok, emqx_utils_maps:deep_remove(Path, RawConf)}
     end;
-pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, ?AUTHN_BIN], RawConf, undefined
-        )
-    of
+pre_config_update(?GATEWAY, {add_authn, GwName, Conf}, RawConf) ->
+    case get_authn(GwName, RawConf) of
         undefined ->
             CertsDir = authn_certs_dir(GwName, Conf),
             Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf),
@@ -458,14 +448,8 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
         _ ->
             badres_authn(already_exist, GwName)
     end;
-pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, <<"listeners">>, LType, LName],
-            RawConf,
-            undefined
-        )
-    of
+pre_config_update(?GATEWAY, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             badres_listener(not_found, GwName, LType, LName);
         Listener ->
@@ -486,12 +470,8 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
                     badres_listener_authn(already_exist, GwName, LType, LName)
             end
     end;
-pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, ?AUTHN_BIN], RawConf, undefined
-        )
-    of
+pre_config_update(?GATEWAY, {update_authn, GwName, Conf}, RawConf) ->
+    case get_authn(GwName, RawConf) of
         undefined ->
             badres_authn(not_found, GwName);
         OldAuthnConf ->
@@ -499,14 +479,8 @@ pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
             Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf, OldAuthnConf),
             {ok, emqx_utils_maps:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf1)}
     end;
-pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, <<"listeners">>, LType, LName],
-            RawConf,
-            undefined
-        )
-    of
+pre_config_update(?GATEWAY, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             badres_listener(not_found, GwName, LType, LName);
         Listener ->
@@ -533,12 +507,8 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
                         )}
             end
     end;
-pre_config_update(_, {remove_authn, GwName}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, ?AUTHN_BIN], RawConf, undefined
-        )
-    of
+pre_config_update(?GATEWAY, {remove_authn, GwName}, RawConf) ->
+    case get_authn(GwName, RawConf) of
         undefined ->
             ok;
         OldAuthnConf ->
@@ -549,7 +519,7 @@ pre_config_update(_, {remove_authn, GwName}, RawConf) ->
         emqx_utils_maps:deep_remove(
             [GwName, ?AUTHN_BIN], RawConf
         )};
-pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) ->
+pre_config_update(?GATEWAY, {remove_authn, GwName, {LType, LName}}, RawConf) ->
     Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN],
     case
         emqx_utils_maps:deep_get(
@@ -565,10 +535,184 @@ pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) ->
             emqx_authentication_config:clear_certs(CertsDir, OldAuthnConf)
     end,
     {ok, emqx_utils_maps:deep_remove(Path, RawConf)};
-pre_config_update(_, UnknownReq, _RawConf) ->
-    logger:error("Unknown configuration update request: ~0p", [UnknownReq]),
+pre_config_update(?GATEWAY, NewRawConf0 = #{}, OldRawConf = #{}) ->
+    %% FIXME don't support gateway's listener's authn update.
+    %% load all authentications
+    NewRawConf1 = pre_load_authentications(NewRawConf0, OldRawConf),
+    %% load all listeners
+    NewRawConf2 = pre_load_listeners(NewRawConf1, OldRawConf),
+    %% load all gateway
+    NewRawConf3 = pre_load_gateways(NewRawConf2, OldRawConf),
+    {ok, NewRawConf3};
+pre_config_update(Path, UnknownReq, _RawConf) ->
+    ?SLOG(error, #{
+        msg => "unknown_gateway_update_request",
+        request => UnknownReq,
+        path => Path
+    }),
     {error, badreq}.
 
+pre_load_gateways(NewConf, OldConf) ->
+    %% unload old gateways
+    maps:foreach(
+        fun(GwName, _OldGwConf) ->
+            case maps:find(GwName, NewConf) of
+                error -> pre_config_update(?GATEWAY, {unload_gateway, GwName}, OldConf);
+                _ -> ok
+            end
+        end,
+        OldConf
+    ),
+    %% load/update gateways
+    maps:map(
+        fun(GwName, NewGwConf) ->
+            case maps:find(GwName, OldConf) of
+                {ok, NewGwConf} ->
+                    NewGwConf;
+                {ok, _OldGwConf} ->
+                    {ok, #{GwName := NewGwConf1}} = pre_config_update(
+                        ?GATEWAY, {update_gateway, GwName, NewGwConf}, OldConf
+                    ),
+                    %% update gateway should pass through ignore keys(listener/authn)
+                    PassThroughConf = maps:with(?IGNORE_KEYS, NewGwConf),
+                    NewGwConf2 = maps:without(?IGNORE_KEYS, NewGwConf1),
+                    maps:merge(NewGwConf2, PassThroughConf);
+                error ->
+                    {ok, #{GwName := NewGwConf1}} = pre_config_update(
+                        ?GATEWAY, {load_gateway, GwName, NewGwConf}, OldConf
+                    ),
+                    NewGwConf1
+            end
+        end,
+        NewConf
+    ).
+
+pre_load_listeners(NewConf, OldConf) ->
+    %% remove listeners
+    maps:foreach(
+        fun(GwName, GwConf) ->
+            Listeners = maps:get(<<"listeners">>, GwConf, #{}),
+            remove_listeners(GwName, NewConf, OldConf, Listeners)
+        end,
+        OldConf
+    ),
+    %% add/update listeners
+    maps:map(
+        fun(GwName, GwConf) ->
+            Listeners = maps:get(<<"listeners">>, GwConf, #{}),
+            NewListeners = create_or_update_listeners(GwName, OldConf, Listeners),
+            maps:put(<<"listeners">>, NewListeners, GwConf)
+        end,
+        NewConf
+    ).
+
+create_or_update_listeners(GwName, OldConf, Listeners) ->
+    maps:map(
+        fun(LType, LConf) ->
+            maps:map(
+                fun(LName, LConf1) ->
+                    NConf =
+                        case get_listener(GwName, LType, LName, OldConf) of
+                            undefined ->
+                                {ok, NConf0} =
+                                    pre_config_update(
+                                        ?GATEWAY,
+                                        {add_listener, GwName, {LType, LName}, LConf1},
+                                        OldConf
+                                    ),
+                                NConf0;
+                            _ ->
+                                {ok, NConf0} =
+                                    pre_config_update(
+                                        ?GATEWAY,
+                                        {update_listener, GwName, {LType, LName}, LConf1},
+                                        OldConf
+                                    ),
+                                NConf0
+                        end,
+                    get_listener(GwName, LType, LName, NConf)
+                end,
+                LConf
+            )
+        end,
+        Listeners
+    ).
+
+remove_listeners(GwName, NewConf, OldConf, Listeners) ->
+    maps:foreach(
+        fun(LType, LConf) ->
+            maps:foreach(
+                fun(LName, _LConf1) ->
+                    case get_listener(GwName, LType, LName, NewConf) of
+                        undefined ->
+                            pre_config_update(
+                                ?GATEWAY, {remove_listener, GwName, {LType, LName}}, OldConf
+                            );
+                        _ ->
+                            ok
+                    end
+                end,
+                LConf
+            )
+        end,
+        Listeners
+    ).
+
+get_listener(GwName, LType, LName, NewConf) ->
+    emqx_utils_maps:deep_get(
+        [GwName, <<"listeners">>, LType, LName], NewConf, undefined
+    ).
+
+get_authn(GwName, Conf) ->
+    emqx_utils_maps:deep_get([GwName, ?AUTHN_BIN], Conf, undefined).
+
+pre_load_authentications(NewConf, OldConf) ->
+    %% remove authentications when not in new config
+    maps:foreach(
+        fun(GwName, OldGwConf) ->
+            case
+                maps:get(?AUTHN_BIN, OldGwConf, undefined) =/= undefined andalso
+                    get_authn(GwName, NewConf) =:= undefined
+            of
+                true ->
+                    pre_config_update(?GATEWAY, {remove_authn, GwName}, OldConf);
+                false ->
+                    ok
+            end
+        end,
+        OldConf
+    ),
+    %% add/update authentications
+    maps:map(
+        fun(GwName, NewGwConf) ->
+            case get_authn(GwName, OldConf) of
+                undefined ->
+                    case maps:get(?AUTHN_BIN, NewGwConf, undefined) of
+                        undefined ->
+                            NewGwConf;
+                        AuthN ->
+                            {ok, #{GwName := #{?AUTHN_BIN := NAuthN}}} =
+                                pre_config_update(?GATEWAY, {add_authn, GwName, AuthN}, OldConf),
+                            maps:put(?AUTHN_BIN, NAuthN, NewGwConf)
+                    end;
+                OldAuthN ->
+                    case maps:get(?AUTHN_BIN, NewGwConf, undefined) of
+                        undefined ->
+                            NewGwConf;
+                        OldAuthN ->
+                            NewGwConf;
+                        NewAuthN ->
+                            {ok, #{GwName := #{?AUTHN_BIN := NAuthN}}} =
+                                pre_config_update(
+                                    ?GATEWAY, {update_authn, GwName, NewAuthN}, OldConf
+                                ),
+                            maps:put(?AUTHN_BIN, NAuthN, NewGwConf)
+                    end
+            end
+        end,
+        NewConf
+    ).
+
 badres_gateway(not_found, GwName) ->
     {error,
         {badres, #{
@@ -642,7 +786,7 @@ badres_listener_authn(already_exist, GwName, LType, LName) ->
 ) ->
     ok | {ok, Result :: any()} | {error, Reason :: term()}.
 
-post_config_update(_, Req, NewConfig, OldConfig, _AppEnvs) when is_tuple(Req) ->
+post_config_update(?GATEWAY, Req, NewConfig, OldConfig, _AppEnvs) when is_tuple(Req) ->
     [_Tag, GwName0 | _] = tuple_to_list(Req),
     GwName = binary_to_existing_atom(GwName0),
 
@@ -657,11 +801,35 @@ post_config_update(_, Req, NewConfig, OldConfig, _AppEnvs) when is_tuple(Req) ->
         {New, Old} when is_map(New), is_map(Old) ->
             emqx_gateway:update(GwName, New)
     end;
-post_config_update(_, _Req, _NewConfig, _OldConfig, _AppEnvs) ->
+post_config_update(?GATEWAY, _Req = #{}, NewConfig, OldConfig, _AppEnvs) ->
+    %% unload gateways
+    maps:foreach(
+        fun(GwName, _OldGwConf) ->
+            case maps:get(GwName, NewConfig, undefined) of
+                undefined ->
+                    emqx_gateway:unload(GwName);
+                _ ->
+                    ok
+            end
+        end,
+        OldConfig
+    ),
+    %% load/update gateways
+    maps:foreach(
+        fun(GwName, NewGwConf) ->
+            case maps:get(GwName, OldConfig, undefined) of
+                undefined ->
+                    emqx_gateway:load(GwName, NewGwConf);
+                _ ->
+                    emqx_gateway:update(GwName, NewGwConf)
+            end
+        end,
+        NewConfig
+    ),
     ok.
 
 %%--------------------------------------------------------------------
-%% Internal funcs
+%% Internal functions
 %%--------------------------------------------------------------------
 
 tune_gw_certs(Fun, GwName, Conf) ->
diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl
index 0aa2ff210..ef17b0954 100644
--- a/apps/emqx_gateway/src/emqx_gateway_metrics.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl
@@ -18,7 +18,7 @@
 
 -behaviour(gen_server).
 
--include_lib("emqx_gateway/include/emqx_gateway.hrl").
+-include("emqx_gateway.hrl").
 
 %% APIs
 -export([start_link/1]).
diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
index ce709efc3..2b169f64b 100644
--- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl
@@ -277,6 +277,48 @@ t_load_unload_gateway(_) ->
         {config_not_found, [<<"gateway">>, stomp]},
         emqx:get_raw_config([gateway, stomp])
     ),
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    #{<<"listeners">> := StompConfL1} = StompConf1,
+    StompConf11 = StompConf1#{
+        <<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL1)
+    },
+    #{<<"listeners">> := StompConfL2} = StompConf2,
+    StompConf22 = StompConf2#{
+        <<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL2)
+    },
+    Raw1 = Raw0#{<<"stomp">> => StompConf11},
+    Raw2 = Raw0#{<<"stomp">> => StompConf22},
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
+    ?assertMatch(
+        #{
+            config := #{
+                authentication := #{backend := built_in_database, enable := true},
+                listeners := #{tcp := #{default := #{bind := 61613}}},
+                mountpoint := <<"t/">>,
+                idle_timeout := 10000
+            }
+        },
+        emqx_gateway:lookup('stomp')
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw2)),
+    assert_confs(StompConf2, emqx:get_raw_config([gateway, stomp])),
+    ?assertMatch(
+        #{
+            config :=
+                #{
+                    authentication := #{backend := built_in_database, enable := true},
+                    listeners := #{tcp := #{default := #{bind := 61613}}},
+                    idle_timeout := 20000,
+                    mountpoint := <<"t2/">>
+                }
+        },
+        emqx_gateway:lookup('stomp')
+    ),
+    %% reset
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    ?assertEqual(undefined, emqx_gateway:lookup('stomp')),
     ok.
 
 t_load_remove_authn(_) ->
@@ -310,6 +352,40 @@ t_load_remove_authn(_) ->
         {config_not_found, [<<"gateway">>, stomp, authentication]},
         emqx:get_raw_config([gateway, stomp, authentication])
     ),
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    #{<<"listeners">> := StompConfL} = StompConf,
+    StompConf1 = StompConf#{
+        <<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL),
+        <<"authentication">> => ?CONF_STOMP_AUTHN_1
+    },
+    Raw1 = maps:put(<<"stomp">>, StompConf1, Raw0),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"password_based:built_in_database">>,
+                    listeners := [#{authn := <<"undefined">>, type := tcp}],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
+    %% reset(remove authn)
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"undefined">>,
+                    listeners := [#{authn := <<"undefined">>, type := tcp}],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
     ok.
 
 t_load_remove_listeners(_) ->
@@ -324,6 +400,7 @@ t_load_remove_listeners(_) ->
         {<<"tcp">>, <<"default">>},
         ?CONF_STOMP_LISTENER_1
     ),
+
     assert_confs(
         maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_1)),
         emqx:get_raw_config([gateway, stomp])
@@ -355,6 +432,59 @@ t_load_remove_listeners(_) ->
         {config_not_found, [<<"gateway">>, stomp, listeners, tcp, default]},
         emqx:get_raw_config([gateway, stomp, listeners, tcp, default])
     ),
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    Raw1 = emqx_utils_maps:deep_put(
+        [<<"stomp">>, <<"listeners">>, <<"tcp">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_1
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    assert_confs(
+        maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_1)),
+        emqx:get_raw_config([gateway, stomp])
+    ),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"password_based:built_in_database">>,
+                    listeners := [#{authn := <<"undefined">>, type := tcp}],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
+    Raw2 = emqx_utils_maps:deep_put(
+        [<<"stomp">>, <<"listeners">>, <<"tcp">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_2
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw2)),
+    assert_confs(
+        maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_2)),
+        emqx:get_raw_config([gateway, stomp])
+    ),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"password_based:built_in_database">>,
+                    listeners := [#{authn := <<"undefined">>, type := tcp}],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
+    %% reset(remove listener)
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"password_based:built_in_database">>,
+                    listeners := [],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
     ok.
 
 t_load_remove_listener_authn(_) ->
@@ -417,6 +547,7 @@ t_load_gateway_with_certs_content(_) ->
         [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
         emqx:get_raw_config([gateway, stomp])
     ),
+    assert_ssl_confs_files_exist(SslConf),
     ok = emqx_gateway_conf:unload_gateway(<<"stomp">>),
     assert_ssl_confs_files_deleted(SslConf),
     ?assertException(
@@ -424,6 +555,25 @@ t_load_gateway_with_certs_content(_) ->
         {config_not_found, [<<"gateway">>, stomp]},
         emqx:get_raw_config([gateway, stomp])
     ),
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    #{<<"listeners">> := StompConfL} = StompConf,
+    StompConf1 = StompConf#{
+        <<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL)
+    },
+    Raw1 = emqx_utils_maps:deep_put([<<"stomp">>], Raw0, StompConf1),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    assert_ssl_confs_files_exist(SslConf),
+    ?assertEqual(
+        SslConf,
+        emqx_utils_maps:deep_get(
+            [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
+            emqx:get_raw_config([gateway, stomp])
+        )
+    ),
+    %% reset
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    assert_ssl_confs_files_deleted(SslConf),
     ok.
 
 %% TODO: Comment out this test case for now, because emqx_tls_lib
@@ -475,6 +625,7 @@ t_add_listener_with_certs_content(_) ->
         [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
         emqx:get_raw_config([gateway, stomp])
     ),
+    assert_ssl_confs_files_exist(SslConf),
     ok = emqx_gateway_conf:remove_listener(
         <<"stomp">>, {<<"ssl">>, <<"default">>}
     ),
@@ -492,6 +643,34 @@ t_add_listener_with_certs_content(_) ->
         {config_not_found, [<<"gateway">>, stomp, listeners, ssl, default]},
         emqx:get_raw_config([gateway, stomp, listeners, ssl, default])
     ),
+
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    Raw1 = emqx_utils_maps:deep_put(
+        [<<"stomp">>, <<"listeners">>, <<"ssl">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_SSL
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    SslConf1 = emqx_utils_maps:deep_get(
+        [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
+        emqx:get_raw_config([gateway, stomp])
+    ),
+    assert_ssl_confs_files_exist(SslConf1),
+    %% update
+    Raw2 = emqx_utils_maps:deep_put(
+        [<<"stomp">>, <<"listeners">>, <<"ssl">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_SSL_2
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw2)),
+    SslConf2 =
+        emqx_utils_maps:deep_get(
+            [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
+            emqx:get_raw_config([gateway, stomp])
+        ),
+    assert_ssl_confs_files_exist(SslConf2),
+    %% reset
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    assert_ssl_confs_files_deleted(SslConf),
+    assert_ssl_confs_files_deleted(SslConf1),
+    assert_ssl_confs_files_deleted(SslConf2),
     ok.
 
 assert_ssl_confs_files_deleted(SslConf) when is_map(SslConf) ->
@@ -503,6 +682,15 @@ assert_ssl_confs_files_deleted(SslConf) when is_map(SslConf) ->
         end,
         Ks
     ).
+assert_ssl_confs_files_exist(SslConf) when is_map(SslConf) ->
+    Ks = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>],
+    lists:foreach(
+        fun(K) ->
+            Path = maps:get(K, SslConf),
+            {ok, _} = file:read_file(Path)
+        end,
+        Ks
+    ).
 
 %%--------------------------------------------------------------------
 %% Utils
diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl
index bb378ef10..2e8a3a583 100644
--- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl
+++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl
@@ -83,13 +83,13 @@ do_assert_confs(Key, Expected, Effected) ->
 
 maybe_unconvert_listeners(Conf) when is_map(Conf) ->
     case maps:take(<<"listeners">>, Conf) of
-        error ->
-            Conf;
-        {Ls, Conf1} ->
+        {Ls, Conf1} when is_list(Ls) ->
             Conf1#{
                 <<"listeners">> =>
                     emqx_gateway_conf:unconvert_listeners(Ls)
-            }
+            };
+        _ ->
+            Conf
     end;
 maybe_unconvert_listeners(Conf) ->
     Conf.

From bcc47442eb10d980a11a8ddf254f6523a4d781f7 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 6 Jun 2023 18:44:27 +0800
Subject: [PATCH 15/29] fix(mqttsn): make mountpoint works for publish

---
 .../src/emqx_gateway_mqttsn.app.src           |  2 +-
 .../src/emqx_mqttsn_channel.erl               |  5 ++-
 .../test/emqx_sn_protocol_SUITE.erl           | 45 +++++++++++++++++++
 3 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src
index 76f0f45b5..b43201e1a 100644
--- a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src
+++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src
@@ -1,6 +1,6 @@
 {application, emqx_gateway_mqttsn, [
     {description, "MQTT-SN Gateway"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {applications, [kernel, stdlib, emqx, emqx_gateway]},
     {env, []},
diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl
index 914f837e1..84334875b 100644
--- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl
+++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl
@@ -1111,15 +1111,16 @@ check_pub_authz(
 
 convert_pub_to_msg(
     {TopicName, Flags, Data},
-    Channel = #channel{clientinfo = #{clientid := ClientId}}
+    Channel = #channel{clientinfo = #{clientid := ClientId, mountpoint := Mountpoint}}
 ) ->
     #mqtt_sn_flags{qos = QoS, dup = Dup, retain = Retain} = Flags,
     NewQoS = get_corrected_qos(QoS),
+    NTopicName = emqx_mountpoint:mount(Mountpoint, TopicName),
     Message = put_message_headers(
         emqx_message:make(
             ClientId,
             NewQoS,
-            TopicName,
+            NTopicName,
             Data,
             #{dup => Dup, retain => Retain},
             #{}
diff --git a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl
index cce4ce904..b22d7b4b0 100644
--- a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl
+++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl
@@ -120,6 +120,13 @@ restart_mqttsn_with_subs_resume_off() ->
         Conf#{<<"subs_resume">> => <<"false">>}
     ).
 
+restart_mqttsn_with_mountpoint(Mp) ->
+    Conf = emqx:get_raw_config([gateway, mqttsn]),
+    emqx_gateway_conf:update_gateway(
+        mqttsn,
+        Conf#{<<"mountpoint">> => Mp}
+    ).
+
 default_config() ->
     ?CONF_DEFAULT.
 
@@ -990,6 +997,44 @@ t_publish_qos2_case03(_) ->
     ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
     gen_udp:close(Socket).
 
+t_publish_mountpoint(_) ->
+    restart_mqttsn_with_mountpoint(<<"mp/">>),
+    Dup = 0,
+    QoS = 1,
+    Retain = 0,
+    Will = 0,
+    CleanSession = 0,
+    MsgId = 1,
+    TopicId1 = ?MAX_PRED_TOPIC_ID + 1,
+    Topic = <<"abc">>,
+    {ok, Socket} = gen_udp:open(0, [binary]),
+    ClientId = ?CLIENTID,
+    send_connect_msg(Socket, ClientId),
+    ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
+    send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId),
+    ?assertEqual(
+        <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+            TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+        receive_response(Socket)
+    ),
+
+    Payload1 = <<20, 21, 22, 23>>,
+    send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1),
+    ?assertEqual(
+        <<7, ?SN_PUBACK, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)
+    ),
+    timer:sleep(100),
+
+    ?assertEqual(
+        <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+            TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>,
+        receive_response(Socket)
+    ),
+
+    send_disconnect_msg(Socket, undefined),
+    restart_mqttsn_with_mountpoint(<<>>),
+    gen_udp:close(Socket).
+
 t_delivery_qos1_register_invalid_topic_id(_) ->
     Dup = 0,
     QoS = 1,

From f26b372e51507aca327b9d04af2a7b5c3a5292b4 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 6 Jun 2023 18:47:41 +0800
Subject: [PATCH 16/29] chore: update changes

---
 changes/ce/fix-10951.en.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/ce/fix-10951.en.md

diff --git a/changes/ce/fix-10951.en.md b/changes/ce/fix-10951.en.md
new file mode 100644
index 000000000..89dabb4a7
--- /dev/null
+++ b/changes/ce/fix-10951.en.md
@@ -0,0 +1 @@
+Fix the issue in MQTT-SN gateway where the `mountpoint` does not take effect on message publishing.

From 1968589f81d543b8c6b68c855b0c5e488c9f7e4d Mon Sep 17 00:00:00 2001
From: Serge Tupchii <serge.tupchii@protonmail.com>
Date: Tue, 6 Jun 2023 14:02:58 +0300
Subject: [PATCH 17/29] fix(emqx_schema): don't allow enabling
 `fail_if_no_peer_cert` if `verify_none` is set

Setting `fail_if_no_peer_cert = true` and `verify = verify_none` causes connection errors.

Closes: EMQX-9586
---
 apps/emqx/src/emqx_schema.erl                 | 18 +++++-
 apps/emqx/test/emqx_schema_tests.erl          | 61 +++++++++++++++++++
 apps/emqx_gateway/src/emqx_gateway_schema.erl | 13 +++-
 changes/ce/fix-10952.en.md                    |  8 +++
 4 files changed, 96 insertions(+), 4 deletions(-)
 create mode 100644 changes/ce/fix-10952.en.md

diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 37d5350a5..521293f7a 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -94,7 +94,8 @@
     validate_keepalive_multiplier/1,
     non_empty_string/1,
     validations/0,
-    naive_env_interpolation/1
+    naive_env_interpolation/1,
+    validate_server_ssl_opts/1
 ]).
 
 -export([qos/0]).
@@ -958,7 +959,7 @@ fields("mqtt_wss_listener") ->
             {"ssl_options",
                 sc(
                     ref("listener_wss_opts"),
-                    #{}
+                    #{validator => fun validate_server_ssl_opts/1}
                 )},
             {"websocket",
                 sc(
@@ -2426,8 +2427,21 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
             ]
         ].
 
+validate_server_ssl_opts(#{<<"fail_if_no_peer_cert">> := true, <<"verify">> := Verify}) ->
+    validate_verify(Verify);
+validate_server_ssl_opts(#{fail_if_no_peer_cert := true, verify := Verify}) ->
+    validate_verify(Verify);
+validate_server_ssl_opts(_SSLOpts) ->
+    ok.
+
+validate_verify(verify_peer) ->
+    ok;
+validate_verify(_) ->
+    {error, "verify must be verify_peer when fail_if_no_peer_cert is true"}.
+
 mqtt_ssl_listener_ssl_options_validator(Conf) ->
     Checks = [
+        fun validate_server_ssl_opts/1,
         fun ocsp_outer_validator/1,
         fun crl_outer_validator/1
     ],
diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl
index ad2341460..58f9a94d5 100644
--- a/apps/emqx/test/emqx_schema_tests.erl
+++ b/apps/emqx/test/emqx_schema_tests.erl
@@ -106,6 +106,67 @@ bad_cipher_test() ->
     ),
     ok.
 
+fail_if_no_peer_cert_test_() ->
+    Sc = #{
+        roots => [mqtt_ssl_listener],
+        fields => #{mqtt_ssl_listener => emqx_schema:fields("mqtt_ssl_listener")}
+    },
+    Opts = #{atom_key => false, required => false},
+    OptsAtomKey = #{atom_key => true, required => false},
+    InvalidConf = #{
+        <<"bind">> => <<"0.0.0.0:9883">>,
+        <<"ssl_options">> => #{
+            <<"fail_if_no_peer_cert">> => true,
+            <<"verify">> => <<"verify_none">>
+        }
+    },
+    InvalidListener = #{<<"mqtt_ssl_listener">> => InvalidConf},
+    ValidListener = #{
+        <<"mqtt_ssl_listener">> => InvalidConf#{
+            <<"ssl_options">> =>
+                #{
+                    <<"fail_if_no_peer_cert">> => true,
+                    <<"verify">> => <<"verify_peer">>
+                }
+        }
+    },
+    ValidListener1 = #{
+        <<"mqtt_ssl_listener">> => InvalidConf#{
+            <<"ssl_options">> =>
+                #{
+                    <<"fail_if_no_peer_cert">> => false,
+                    <<"verify">> => <<"verify_none">>
+                }
+        }
+    },
+    Reason = "verify must be verify_peer when fail_if_no_peer_cert is true",
+    [
+        ?_assertThrow(
+            {_Sc, [#{kind := validation_error, reason := Reason}]},
+            hocon_tconf:check_plain(Sc, InvalidListener, Opts)
+        ),
+        ?_assertThrow(
+            {_Sc, [#{kind := validation_error, reason := Reason}]},
+            hocon_tconf:check_plain(Sc, InvalidListener, OptsAtomKey)
+        ),
+        ?_assertMatch(
+            #{mqtt_ssl_listener := #{}},
+            hocon_tconf:check_plain(Sc, ValidListener, OptsAtomKey)
+        ),
+        ?_assertMatch(
+            #{mqtt_ssl_listener := #{}},
+            hocon_tconf:check_plain(Sc, ValidListener1, OptsAtomKey)
+        ),
+        ?_assertMatch(
+            #{<<"mqtt_ssl_listener">> := #{}},
+            hocon_tconf:check_plain(Sc, ValidListener, Opts)
+        ),
+        ?_assertMatch(
+            #{<<"mqtt_ssl_listener">> := #{}},
+            hocon_tconf:check_plain(Sc, ValidListener1, Opts)
+        )
+    ].
+
 validate(Schema, Data0) ->
     Sc = #{
         roots => [ssl_opts],
diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl
index 8c80fc1fa..3c5706e82 100644
--- a/apps/emqx_gateway/src/emqx_gateway_schema.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl
@@ -120,7 +120,10 @@ fields(ssl_listener) ->
             {ssl_options,
                 sc(
                     hoconsc:ref(emqx_schema, "listener_ssl_opts"),
-                    #{desc => ?DESC(ssl_listener_options)}
+                    #{
+                        desc => ?DESC(ssl_listener_options),
+                        validator => fun emqx_schema:validate_server_ssl_opts/1
+                    }
                 )}
         ];
 fields(udp_listener) ->
@@ -132,7 +135,13 @@ fields(udp_listener) ->
 fields(dtls_listener) ->
     [{acceptors, sc(integer(), #{default => 16, desc => ?DESC(dtls_listener_acceptors)})}] ++
         fields(udp_listener) ++
-        [{dtls_options, sc(ref(dtls_opts), #{desc => ?DESC(dtls_listener_dtls_opts)})}];
+        [
+            {dtls_options,
+                sc(ref(dtls_opts), #{
+                    desc => ?DESC(dtls_listener_dtls_opts),
+                    validator => fun emqx_schema:validate_server_ssl_opts/1
+                })}
+        ];
 fields(udp_opts) ->
     [
         {active_n,
diff --git a/changes/ce/fix-10952.en.md b/changes/ce/fix-10952.en.md
new file mode 100644
index 000000000..20792906f
--- /dev/null
+++ b/changes/ce/fix-10952.en.md
@@ -0,0 +1,8 @@
+Disallow enabling `fail_if_no_peer_cert` in listener SSL options if `verify_none` is set.
+
+Setting `fail_if_no_peer_cert = true` and `verify = verify_none` caused connection errors
+due to incompatible options.
+This fix validates the options when creating or updating a listener to avoid these errors.
+
+Note: any old listener configuration with `fail_if_no_peer_cert = true` and `verify = verify_none`
+that was previously allowed will fail to load after applying this fix and must be manually fixed.

From bf2b5a9f24eb99f88feb21cdf727af522c722a0b Mon Sep 17 00:00:00 2001
From: Serge Tupchii <serge.tupchii@protonmail.com>
Date: Tue, 6 Jun 2023 18:53:17 +0300
Subject: [PATCH 18/29] fix(emqx_listeners): fix listener authentication
 create/update

Listener authentication is a list of authenticators, so each of them must be created
or updated individually.
---
 apps/emqx/src/emqx_listeners.erl | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index c0116fd8c..9a59db6e1 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -527,18 +527,26 @@ post_config_update(_Path, _Request, _NewConf, _OldConf, _AppEnvs) ->
 
 create_listener(Type, Name, NewConf) ->
     Res = start_listener(Type, Name, NewConf),
-    recreate_authenticator(Res, Type, Name, NewConf).
+    recreate_authenticators(Res, Type, Name, NewConf).
 
-recreate_authenticator(ok, Type, Name, Conf) ->
+recreate_authenticators(ok, Type, Name, Conf) ->
     Chain = listener_id(Type, Name),
     _ = emqx_authentication:delete_chain(Chain),
-    case maps:get(authentication, Conf, []) of
-        [] -> ok;
-        AuthN -> emqx_authentication:create_authenticator(Chain, AuthN)
-    end;
-recreate_authenticator(Error, _Type, _Name, _NewConf) ->
+    do_create_authneticators(Chain, maps:get(authentication, Conf, []));
+recreate_authenticators(Error, _Type, _Name, _NewConf) ->
     Error.
 
+do_create_authneticators(Chain, [AuthN | T]) ->
+    case emqx_authentication:create_authenticator(Chain, AuthN) of
+        {ok, _} ->
+            do_create_authneticators(Chain, T);
+        Error ->
+            _ = emqx_authentication:delete_chain(Chain),
+            Error
+    end;
+do_create_authneticators(_Chain, []) ->
+    ok.
+
 remove_listener(Type, Name, OldConf) ->
     ok = unregister_ocsp_stapling_refresh(Type, Name),
     case stop_listener(Type, Name, OldConf) of
@@ -553,7 +561,7 @@ update_listener(Type, Name, {OldConf, NewConf}) ->
     try_clear_ssl_files(certs_dir(Type, Name), NewConf, OldConf),
     ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf),
     Res = restart_listener(Type, Name, {OldConf, NewConf}),
-    recreate_authenticator(Res, Type, Name, NewConf).
+    recreate_authenticators(Res, Type, Name, NewConf).
 
 perform_listener_changes([]) ->
     ok;

From 260fae296bfff8b025282ba8d2a112b2ad25bc50 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 5 Jun 2023 15:05:21 -0300
Subject: [PATCH 19/29] feat(gcp_pubsub): generate jwt tokens on demand without
 workers (5.1)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes https://emqx.atlassian.net/browse/EMQX-9603

Rather than relying on a JWT worker to produce and refresh tokens, we
could just produce then on demand when pushing the messages to GCP
PubSub.  That can generate a bit of extra work (as multiple processes
might realize it’s time to refresh the JWT and do so), but that
shouldn’t be much.  In return, we avoid any possibility of not having
a fresh JWT when pushing messages.
---
 .../src/emqx_bridge_gcp_pubsub_connector.erl  | 107 ++++++--------
 .../test/emqx_bridge_gcp_pubsub_SUITE.erl     | 132 ++++++++++--------
 .../emqx_connector/src/emqx_connector_jwt.erl |  87 +++++++++++-
 .../src/emqx_connector_jwt_worker.erl         |  39 +-----
 .../test/emqx_connector_jwt_SUITE.erl         |  66 +++++++++
 .../test/emqx_connector_jwt_worker_SUITE.erl  |   4 +-
 changes/ee/feat-10944.en.md                   |   1 +
 7 files changed, 273 insertions(+), 163 deletions(-)
 create mode 100644 changes/ee/feat-10944.en.md

diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
index d2f6bbee4..6d278c65f 100644
--- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
+++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
@@ -6,6 +6,7 @@
 
 -behaviour(emqx_resource).
 
+-include_lib("jose/include/jose_jwk.hrl").
 -include_lib("emqx_connector/include/emqx_connector_tables.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 -include_lib("typerefl/include/types.hrl").
@@ -26,7 +27,6 @@
 ]).
 -export([reply_delegator/3]).
 
--type jwt_worker() :: binary().
 -type service_account_json() :: emqx_bridge_gcp_pubsub:service_account_json().
 -type config() :: #{
     connect_timeout := emqx_schema:duration_ms(),
@@ -38,7 +38,7 @@
 }.
 -type state() :: #{
     connect_timeout := timer:time(),
-    jwt_worker_id := jwt_worker(),
+    jwt_config := emqx_connector_jwt:jwt_config(),
     max_retries := non_neg_integer(),
     payload_template := emqx_plugin_libs_rule:tmpl_token(),
     pool_name := binary(),
@@ -97,12 +97,12 @@ on_start(
         {enable_pipelining, maps:get(enable_pipelining, Config, ?DEFAULT_PIPELINE_SIZE)}
     ],
     #{
-        jwt_worker_id := JWTWorkerId,
+        jwt_config := JWTConfig,
         project_id := ProjectId
-    } = ensure_jwt_worker(ResourceId, Config),
+    } = parse_jwt_config(ResourceId, Config),
     State = #{
         connect_timeout => ConnectTimeout,
-        jwt_worker_id => JWTWorkerId,
+        jwt_config => JWTConfig,
         max_retries => MaxRetries,
         payload_template => emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate),
         pool_name => ResourceId,
@@ -136,14 +136,13 @@ on_start(
 -spec on_stop(resource_id(), state()) -> ok | {error, term()}.
 on_stop(
     ResourceId,
-    _State = #{jwt_worker_id := JWTWorkerId}
+    _State = #{jwt_config := JWTConfig}
 ) ->
-    ?tp(gcp_pubsub_stop, #{resource_id => ResourceId, jwt_worker_id => JWTWorkerId}),
+    ?tp(gcp_pubsub_stop, #{resource_id => ResourceId, jwt_config => JWTConfig}),
     ?SLOG(info, #{
         msg => "stopping_gcp_pubsub_bridge",
         connector => ResourceId
     }),
-    emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
     emqx_connector_jwt:delete_jwt(?JWT_TABLE, ResourceId),
     ehttpc_sup:stop_pool(ResourceId).
 
@@ -228,12 +227,12 @@ on_get_status(ResourceId, #{connect_timeout := Timeout} = State) ->
 %% Helper fns
 %%-------------------------------------------------------------------------------------------------
 
--spec ensure_jwt_worker(resource_id(), config()) ->
+-spec parse_jwt_config(resource_id(), config()) ->
     #{
-        jwt_worker_id := jwt_worker(),
+        jwt_config := emqx_connector_jwt:jwt_config(),
         project_id := binary()
     }.
-ensure_jwt_worker(ResourceId, #{
+parse_jwt_config(ResourceId, #{
     service_account_json := ServiceAccountJSON
 }) ->
     #{
@@ -246,8 +245,32 @@ ensure_jwt_worker(ResourceId, #{
     Aud = <<"https://pubsub.googleapis.com/">>,
     ExpirationMS = timer:hours(1),
     Alg = <<"RS256">>,
-    Config = #{
-        private_key => PrivateKeyPEM,
+    JWK =
+        try jose_jwk:from_pem(PrivateKeyPEM) of
+            JWK0 = #jose_jwk{} ->
+                %% Don't wrap the JWK with `emqx_secret:wrap' here;
+                %% this is stored in mnesia and synchronized among the
+                %% nodes, and will easily become a bad fun.
+                JWK0;
+            [] ->
+                ?tp(error, gcp_pubsub_connector_startup_error, #{error => empty_key}),
+                throw("empty private in service account json");
+            {error, Reason} ->
+                Error = {invalid_private_key, Reason},
+                ?tp(error, gcp_pubsub_connector_startup_error, #{error => Error}),
+                throw("invalid private key in service account json");
+            Error0 ->
+                Error = {invalid_private_key, Error0},
+                ?tp(error, gcp_pubsub_connector_startup_error, #{error => Error}),
+                throw("invalid private key in service account json")
+        catch
+            Kind:Reason ->
+                Error = {Kind, Reason},
+                ?tp(error, gcp_pubsub_connector_startup_error, #{error => Error}),
+                throw("invalid private key in service account json")
+        end,
+    JWTConfig = #{
+        jwk => emqx_secret:wrap(JWK),
         resource_id => ResourceId,
         expiration => ExpirationMS,
         table => ?JWT_TABLE,
@@ -257,46 +280,8 @@ ensure_jwt_worker(ResourceId, #{
         kid => KId,
         alg => Alg
     },
-
-    JWTWorkerId = <<"gcp_pubsub_jwt_worker:", ResourceId/binary>>,
-    Worker =
-        case emqx_connector_jwt_sup:ensure_worker_present(JWTWorkerId, Config) of
-            {ok, Worker0} ->
-                Worker0;
-            Error ->
-                ?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{
-                    connector => ResourceId,
-                    reason => Error
-                }),
-                _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
-                throw(failed_to_start_jwt_worker)
-        end,
-    MRef = monitor(process, Worker),
-    Ref = emqx_connector_jwt_worker:ensure_jwt(Worker),
-
-    %% to ensure that this resource and its actions will be ready to
-    %% serve when started, we must ensure that the first JWT has been
-    %% produced by the worker.
-    receive
-        {Ref, token_created} ->
-            ?tp(gcp_pubsub_bridge_jwt_created, #{resource_id => ResourceId}),
-            demonitor(MRef, [flush]),
-            ok;
-        {'DOWN', MRef, process, Worker, Reason} ->
-            ?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{
-                connector => ResourceId,
-                reason => Reason
-            }),
-            _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
-            throw(failed_to_start_jwt_worker)
-    after 10_000 ->
-        ?tp(warning, "gcp_pubsub_bridge_jwt_timeout", #{connector => ResourceId}),
-        demonitor(MRef, [flush]),
-        _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
-        throw(timeout_creating_jwt)
-    end,
     #{
-        jwt_worker_id => JWTWorkerId,
+        jwt_config => JWTConfig,
         project_id => ProjectId
     }.
 
@@ -322,14 +307,10 @@ publish_path(
 ) ->
     <<"/v1/projects/", ProjectId/binary, "/topics/", PubSubTopic/binary, ":publish">>.
 
--spec get_jwt_authorization_header(resource_id()) -> [{binary(), binary()}].
-get_jwt_authorization_header(ResourceId) ->
-    case emqx_connector_jwt:lookup_jwt(?JWT_TABLE, ResourceId) of
-        %% Since we synchronize the JWT creation during resource start
-        %% (see `on_start/2'), this will be always be populated.
-        {ok, JWT} ->
-            [{<<"Authorization">>, <<"Bearer ", JWT/binary>>}]
-    end.
+-spec get_jwt_authorization_header(emqx_connector_jwt:jwt_config()) -> [{binary(), binary()}].
+get_jwt_authorization_header(JWTConfig) ->
+    JWT = emqx_connector_jwt:ensure_jwt(JWTConfig),
+    [{<<"Authorization">>, <<"Bearer ", JWT/binary>>}].
 
 -spec do_send_requests_sync(
     state(),
@@ -342,6 +323,7 @@ get_jwt_authorization_header(ResourceId) ->
     | {error, term()}.
 do_send_requests_sync(State, Requests, ResourceId) ->
     #{
+        jwt_config := JWTConfig,
         pool_name := PoolName,
         max_retries := MaxRetries,
         request_ttl := RequestTTL
@@ -354,7 +336,7 @@ do_send_requests_sync(State, Requests, ResourceId) ->
             requests => Requests
         }
     ),
-    Headers = get_jwt_authorization_header(ResourceId),
+    Headers = get_jwt_authorization_header(JWTConfig),
     Payloads =
         lists:map(
             fun({send_message, Selected}) ->
@@ -466,6 +448,7 @@ do_send_requests_sync(State, Requests, ResourceId) ->
 ) -> {ok, pid()}.
 do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId) ->
     #{
+        jwt_config := JWTConfig,
         pool_name := PoolName,
         request_ttl := RequestTTL
     } = State,
@@ -477,7 +460,7 @@ do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId) ->
             requests => Requests
         }
     ),
-    Headers = get_jwt_authorization_header(ResourceId),
+    Headers = get_jwt_authorization_header(JWTConfig),
     Payloads =
         lists:map(
             fun({send_message, Selected}) ->
diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl
index b5f22f727..4dd2682fb 100644
--- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl
+++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl
@@ -55,8 +55,9 @@ single_config_tests() ->
         t_not_of_service_account_type,
         t_json_missing_fields,
         t_invalid_private_key,
-        t_jwt_worker_start_timeout,
-        t_failed_to_start_jwt_worker,
+        t_truncated_private_key,
+        t_jose_error_tuple,
+        t_jose_other_error,
         t_stop,
         t_get_status_ok,
         t_get_status_down,
@@ -580,14 +581,7 @@ t_publish_success(Config) ->
     ServiceAccountJSON = ?config(service_account_json, Config),
     TelemetryTable = ?config(telemetry_table, Config),
     Topic = <<"t/topic">>,
-    ?check_trace(
-        create_bridge(Config),
-        fun(Res, Trace) ->
-            ?assertMatch({ok, _}, Res),
-            ?assertMatch([_], ?of_kind(gcp_pubsub_bridge_jwt_created, Trace)),
-            ok
-        end
-    ),
+    ?assertMatch({ok, _}, create_bridge(Config)),
     {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config),
     on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end),
     assert_empty_metrics(ResourceId),
@@ -686,14 +680,7 @@ t_publish_success_local_topic(Config) ->
     ok.
 
 t_create_via_http(Config) ->
-    ?check_trace(
-        create_bridge_http(Config),
-        fun(Res, Trace) ->
-            ?assertMatch({ok, _}, Res),
-            ?assertMatch([_, _], ?of_kind(gcp_pubsub_bridge_jwt_created, Trace)),
-            ok
-        end
-    ),
+    ?assertMatch({ok, _}, create_bridge_http(Config)),
     ok.
 
 t_publish_templated(Config) ->
@@ -705,16 +692,12 @@ t_publish_templated(Config) ->
         "{\"payload\": \"${payload}\","
         " \"pub_props\": ${pub_props}}"
     >>,
-    ?check_trace(
+    ?assertMatch(
+        {ok, _},
         create_bridge(
             Config,
             #{<<"payload_template">> => PayloadTemplate}
-        ),
-        fun(Res, Trace) ->
-            ?assertMatch({ok, _}, Res),
-            ?assertMatch([_], ?of_kind(gcp_pubsub_bridge_jwt_created, Trace)),
-            ok
-        end
+        )
     ),
     {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config),
     on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end),
@@ -908,36 +891,26 @@ t_invalid_private_key(Config) ->
                                 #{<<"private_key">> => InvalidPrivateKeyPEM}
                         }
                     ),
-                    #{?snk_kind := "gcp_pubsub_bridge_jwt_worker_failed_to_start"},
+                    #{?snk_kind := gcp_pubsub_connector_startup_error},
                     20_000
                 ),
             Res
         end,
         fun(Res, Trace) ->
             ?assertMatch({ok, _}, Res),
-            ?assertMatch(
-                [#{reason := Reason}] when
-                    Reason =:= noproc orelse
-                        Reason =:= {shutdown, {error, empty_key}},
-                ?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace)
-            ),
             ?assertMatch(
                 [#{error := empty_key}],
-                ?of_kind(connector_jwt_worker_startup_error, Trace)
+                ?of_kind(gcp_pubsub_connector_startup_error, Trace)
             ),
             ok
         end
     ),
     ok.
 
-t_jwt_worker_start_timeout(Config) ->
-    InvalidPrivateKeyPEM = <<"xxxxxx">>,
+t_truncated_private_key(Config) ->
+    InvalidPrivateKeyPEM = <<"-----BEGIN PRIVATE KEY-----\nMIIEvQI...">>,
     ?check_trace(
         begin
-            ?force_ordering(
-                #{?snk_kind := will_never_happen},
-                #{?snk_kind := connector_jwt_worker_make_key}
-            ),
             {Res, {ok, _Event}} =
                 ?wait_async_action(
                     create_bridge(
@@ -947,14 +920,71 @@ t_jwt_worker_start_timeout(Config) ->
                                 #{<<"private_key">> => InvalidPrivateKeyPEM}
                         }
                     ),
-                    #{?snk_kind := "gcp_pubsub_bridge_jwt_timeout"},
+                    #{?snk_kind := gcp_pubsub_connector_startup_error},
                     20_000
                 ),
             Res
         end,
         fun(Res, Trace) ->
             ?assertMatch({ok, _}, Res),
-            ?assertMatch([_], ?of_kind("gcp_pubsub_bridge_jwt_timeout", Trace)),
+            ?assertMatch(
+                [#{error := {error, function_clause}}],
+                ?of_kind(gcp_pubsub_connector_startup_error, Trace)
+            ),
+            ok
+        end
+    ),
+    ok.
+
+t_jose_error_tuple(Config) ->
+    ?check_trace(
+        begin
+            {Res, {ok, _Event}} =
+                ?wait_async_action(
+                    emqx_common_test_helpers:with_mock(
+                        jose_jwk,
+                        from_pem,
+                        fun(_PrivateKeyPEM) -> {error, some_error} end,
+                        fun() -> create_bridge(Config) end
+                    ),
+                    #{?snk_kind := gcp_pubsub_connector_startup_error},
+                    20_000
+                ),
+            Res
+        end,
+        fun(Res, Trace) ->
+            ?assertMatch({ok, _}, Res),
+            ?assertMatch(
+                [#{error := {invalid_private_key, some_error}}],
+                ?of_kind(gcp_pubsub_connector_startup_error, Trace)
+            ),
+            ok
+        end
+    ),
+    ok.
+
+t_jose_other_error(Config) ->
+    ?check_trace(
+        begin
+            {Res, {ok, _Event}} =
+                ?wait_async_action(
+                    emqx_common_test_helpers:with_mock(
+                        jose_jwk,
+                        from_pem,
+                        fun(_PrivateKeyPEM) -> {unknown, error} end,
+                        fun() -> create_bridge(Config) end
+                    ),
+                    #{?snk_kind := gcp_pubsub_connector_startup_error},
+                    20_000
+                ),
+            Res
+        end,
+        fun(Res, Trace) ->
+            ?assertMatch({ok, _}, Res),
+            ?assertMatch(
+                [#{error := {invalid_private_key, {unknown, error}}}],
+                ?of_kind(gcp_pubsub_connector_startup_error, Trace)
+            ),
             ok
         end
     ),
@@ -1309,26 +1339,6 @@ t_unrecoverable_error(Config) ->
     ),
     ok.
 
-t_failed_to_start_jwt_worker(Config) ->
-    ?check_trace(
-        emqx_common_test_helpers:with_mock(
-            emqx_connector_jwt_sup,
-            ensure_worker_present,
-            fun(_JWTWorkerId, _Config) -> {error, restarting} end,
-            fun() ->
-                ?assertMatch({ok, _}, create_bridge(Config))
-            end
-        ),
-        fun(Trace) ->
-            ?assertMatch(
-                [#{reason := {error, restarting}}],
-                ?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace)
-            ),
-            ok
-        end
-    ),
-    ok.
-
 t_stop(Config) ->
     Name = ?config(gcp_pubsub_name, Config),
     {ok, _} = create_bridge(Config),
diff --git a/apps/emqx_connector/src/emqx_connector_jwt.erl b/apps/emqx_connector/src/emqx_connector_jwt.erl
index 1e695faad..49b7b3a0e 100644
--- a/apps/emqx_connector/src/emqx_connector_jwt.erl
+++ b/apps/emqx_connector/src/emqx_connector_jwt.erl
@@ -19,15 +19,33 @@
 -include_lib("emqx_connector/include/emqx_connector_tables.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("jose/include/jose_jwt.hrl").
+-include_lib("jose/include/jose_jws.hrl").
 
 %% API
 -export([
     lookup_jwt/1,
     lookup_jwt/2,
-    delete_jwt/2
+    delete_jwt/2,
+    ensure_jwt/1
 ]).
 
 -type jwt() :: binary().
+-type wrapped_jwk() :: fun(() -> jose_jwk:key()).
+-type jwk() :: jose_jwk:key().
+-type jwt_config() :: #{
+    expiration := timer:time(),
+    resource_id := resource_id(),
+    table := ets:table(),
+    jwk := wrapped_jwk() | jwk(),
+    iss := binary(),
+    sub := binary(),
+    aud := binary(),
+    kid := binary(),
+    alg := binary()
+}.
+
+-export_type([jwt_config/0, jwt/0]).
 
 -spec lookup_jwt(resource_id()) -> {ok, jwt()} | {error, not_found}.
 lookup_jwt(ResourceId) ->
@@ -57,3 +75,70 @@ delete_jwt(TId, ResourceId) ->
         error:badarg ->
             ok
     end.
+
+%% @doc Attempts to retrieve a valid JWT from the cache.  If there is
+%% none or if the cached token is expired, generates an caches a fresh
+%% one.
+-spec ensure_jwt(jwt_config()) -> jwt().
+ensure_jwt(JWTConfig) ->
+    #{resource_id := ResourceId, table := Table} = JWTConfig,
+    case lookup_jwt(Table, ResourceId) of
+        {error, not_found} ->
+            JWT = do_generate_jwt(JWTConfig),
+            store_jwt(JWTConfig, JWT),
+            JWT;
+        {ok, JWT0} ->
+            case is_about_to_expire(JWT0) of
+                true ->
+                    JWT = do_generate_jwt(JWTConfig),
+                    store_jwt(JWTConfig, JWT),
+                    JWT;
+                false ->
+                    JWT0
+            end
+    end.
+
+%%-----------------------------------------------------------------------------------------
+%% Helper fns
+%%-----------------------------------------------------------------------------------------
+
+-spec do_generate_jwt(jwt_config()) -> jwt().
+do_generate_jwt(#{
+    expiration := ExpirationMS,
+    iss := Iss,
+    sub := Sub,
+    aud := Aud,
+    kid := KId,
+    alg := Alg,
+    jwk := WrappedJWK
+}) ->
+    JWK = emqx_secret:unwrap(WrappedJWK),
+    Headers = #{
+        <<"alg">> => Alg,
+        <<"kid">> => KId
+    },
+    Now = erlang:system_time(seconds),
+    ExpirationS = erlang:convert_time_unit(ExpirationMS, millisecond, second),
+    Claims = #{
+        <<"iss">> => Iss,
+        <<"sub">> => Sub,
+        <<"aud">> => Aud,
+        <<"iat">> => Now,
+        <<"exp">> => Now + ExpirationS
+    },
+    JWT0 = jose_jwt:sign(JWK, Headers, Claims),
+    {_, JWT} = jose_jws:compact(JWT0),
+    JWT.
+
+-spec store_jwt(jwt_config(), jwt()) -> ok.
+store_jwt(#{resource_id := ResourceId, table := TId}, JWT) ->
+    true = ets:insert(TId, {{ResourceId, jwt}, JWT}),
+    ?tp(emqx_connector_jwt_token_stored, #{resource_id => ResourceId}),
+    ok.
+
+-spec is_about_to_expire(jwt()) -> boolean().
+is_about_to_expire(JWT) ->
+    #jose_jwt{fields = #{<<"exp">> := Exp}} = jose_jwt:peek(JWT),
+    Now = erlang:system_time(seconds),
+    GraceExp = Exp - timer:seconds(5),
+    Now >= GraceExp.
diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl
index b13e74a4d..c814b32e1 100644
--- a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl
+++ b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl
@@ -189,49 +189,14 @@ terminate(_Reason, State) ->
 %% Helper fns
 %%-----------------------------------------------------------------------------------------
 
--spec do_generate_jwt(state()) -> jwt().
-do_generate_jwt(
-    #{
-        expiration := ExpirationMS,
-        iss := Iss,
-        sub := Sub,
-        aud := Aud,
-        kid := KId,
-        alg := Alg,
-        jwk := JWK
-    } = _State
-) ->
-    Headers = #{
-        <<"alg">> => Alg,
-        <<"kid">> => KId
-    },
-    Now = erlang:system_time(seconds),
-    ExpirationS = erlang:convert_time_unit(ExpirationMS, millisecond, second),
-    Claims = #{
-        <<"iss">> => Iss,
-        <<"sub">> => Sub,
-        <<"aud">> => Aud,
-        <<"iat">> => Now,
-        <<"exp">> => Now + ExpirationS
-    },
-    JWT0 = jose_jwt:sign(JWK, Headers, Claims),
-    {_, JWT} = jose_jws:compact(JWT0),
-    JWT.
-
 -spec generate_and_store_jwt(state()) -> state().
 generate_and_store_jwt(State0) ->
-    JWT = do_generate_jwt(State0),
-    store_jwt(State0, JWT),
+    JWTConfig = maps:without([jwt, refresh_timer], State0),
+    JWT = emqx_connector_jwt:ensure_jwt(JWTConfig),
     ?tp(connector_jwt_worker_refresh, #{jwt => JWT}),
     State1 = State0#{jwt := JWT},
     ensure_timer(State1).
 
--spec store_jwt(state(), jwt()) -> ok.
-store_jwt(#{resource_id := ResourceId, table := TId}, JWT) ->
-    true = ets:insert(TId, {{ResourceId, jwt}, JWT}),
-    ?tp(connector_jwt_worker_token_stored, #{resource_id => ResourceId}),
-    ok.
-
 -spec ensure_timer(state()) -> state().
 ensure_timer(
     State = #{
diff --git a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl b/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl
index 3105c95ea..a89d67f4f 100644
--- a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl
@@ -18,7 +18,10 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("jose/include/jose_jwt.hrl").
+-include_lib("jose/include/jose_jws.hrl").
 -include("emqx_connector_tables.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -compile([export_all, nowarn_export_all]).
 
@@ -51,6 +54,33 @@ end_per_testcase(_TestCase, _Config) ->
 insert_jwt(TId, ResourceId, JWT) ->
     ets:insert(TId, {{ResourceId, jwt}, JWT}).
 
+generate_private_key_pem() ->
+    PublicExponent = 65537,
+    Size = 2048,
+    Key = public_key:generate_key({rsa, Size, PublicExponent}),
+    DERKey = public_key:der_encode('PrivateKeyInfo', Key),
+    public_key:pem_encode([{'PrivateKeyInfo', DERKey, not_encrypted}]).
+
+generate_config() ->
+    PrivateKeyPEM = generate_private_key_pem(),
+    ResourceID = emqx_guid:gen(),
+    #{
+        private_key => PrivateKeyPEM,
+        expiration => timer:hours(1),
+        resource_id => ResourceID,
+        table => ets:new(test_jwt_table, [ordered_set, public]),
+        iss => <<"issuer">>,
+        sub => <<"subject">>,
+        aud => <<"audience">>,
+        kid => <<"key id">>,
+        alg => <<"RS256">>
+    }.
+
+is_expired(JWT) ->
+    #jose_jwt{fields = #{<<"exp">> := Exp}} = jose_jwt:peek(JWT),
+    Now = erlang:system_time(seconds),
+    Now >= Exp.
+
 %%-----------------------------------------------------------------------------
 %% Test cases
 %%-----------------------------------------------------------------------------
@@ -77,3 +107,39 @@ t_delete_jwt(_Config) ->
     ?assertEqual(ok, emqx_connector_jwt:delete_jwt(TId, ResourceId)),
     ?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(TId, ResourceId)),
     ok.
+
+t_ensure_jwt(_Config) ->
+    Config0 =
+        #{
+            table := Table,
+            resource_id := ResourceId,
+            private_key := PrivateKeyPEM
+        } = generate_config(),
+    JWK = jose_jwk:from_pem(PrivateKeyPEM),
+    Config1 = maps:without([private_key], Config0),
+    Expiration = timer:seconds(10),
+    JWTConfig = Config1#{jwk => JWK, expiration := Expiration},
+    ?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(Table, ResourceId)),
+    ?check_trace(
+        begin
+            JWT0 = emqx_connector_jwt:ensure_jwt(JWTConfig),
+            ?assertNot(is_expired(JWT0)),
+            %% should refresh 5 s before expiration
+            ct:sleep(Expiration - 5500),
+            JWT1 = emqx_connector_jwt:ensure_jwt(JWTConfig),
+            ?assertNot(is_expired(JWT1)),
+            %% fully expired
+            ct:sleep(2 * Expiration),
+            JWT2 = emqx_connector_jwt:ensure_jwt(JWTConfig),
+            ?assertNot(is_expired(JWT2)),
+            {JWT0, JWT1, JWT2}
+        end,
+        fun({JWT0, JWT1, JWT2}, Trace) ->
+            ?assertNotEqual(JWT0, JWT1),
+            ?assertNotEqual(JWT1, JWT2),
+            ?assertNotEqual(JWT2, JWT0),
+            ?assertMatch([_, _, _], ?of_kind(emqx_connector_jwt_token_stored, Trace)),
+            ok
+        end
+    ),
+    ok.
diff --git a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl b/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl
index a079d632f..fdeb980d4 100644
--- a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl
+++ b/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl
@@ -176,7 +176,7 @@ t_refresh(_Config) ->
             {{ok, _Pid}, {ok, _Event}} =
                 ?wait_async_action(
                     emqx_connector_jwt_worker:start_link(Config),
-                    #{?snk_kind := connector_jwt_worker_token_stored},
+                    #{?snk_kind := emqx_connector_jwt_token_stored},
                     5_000
                 ),
             {ok, FirstJWT} = emqx_connector_jwt:lookup_jwt(Table, ResourceId),
@@ -209,7 +209,7 @@ t_refresh(_Config) ->
         fun({FirstJWT, SecondJWT, ThirdJWT}, Trace) ->
             ?assertMatch(
                 [_, _, _ | _],
-                ?of_kind(connector_jwt_worker_token_stored, Trace)
+                ?of_kind(emqx_connector_jwt_token_stored, Trace)
             ),
             ?assertNotEqual(FirstJWT, SecondJWT),
             ?assertNotEqual(SecondJWT, ThirdJWT),
diff --git a/changes/ee/feat-10944.en.md b/changes/ee/feat-10944.en.md
new file mode 100644
index 000000000..9a6c0b624
--- /dev/null
+++ b/changes/ee/feat-10944.en.md
@@ -0,0 +1 @@
+Improved the GCP PubSub bridge to avoid a potential issue where messages could fail to be sent when restarting a node.

From bed1efd99faa0c2e8a0891957a30322643b3db6e Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Mon, 5 Jun 2023 14:12:31 +0800
Subject: [PATCH 20/29] build: bump dashboard tag `v1.2.6-beta.1` and
 `e1.1.0-beta.2`

The dashboard asset zip file will contain version after these two tags.
---
 Makefile                 | 4 ++--
 scripts/get-dashboard.sh | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index dc8564c38..f322a0a80 100644
--- a/Makefile
+++ b/Makefile
@@ -15,8 +15,8 @@ endif
 
 # Dashbord version
 # from https://github.com/emqx/emqx-dashboard5
-export EMQX_DASHBOARD_VERSION ?= v1.2.5-1
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.8-beta.1
+export EMQX_DASHBOARD_VERSION ?= v1.2.6-beta.1
+export EMQX_EE_DASHBOARD_VERSION ?= e1.1.0-beta.2
 
 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used
 # In make 4.4+, for backward-compatibility the value from the original environment is used.
diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh
index ace795aa5..9ca71482e 100755
--- a/scripts/get-dashboard.sh
+++ b/scripts/get-dashboard.sh
@@ -8,10 +8,10 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
 VERSION="${1}"
 case "$VERSION" in
     v*)
-        RELEASE_ASSET_FILE="emqx-dashboard.zip"
+        RELEASE_ASSET_FILE="emqx-dashboard-$VERSION.zip"
         ;;
     e*)
-        RELEASE_ASSET_FILE="emqx-enterprise-dashboard.zip"
+        RELEASE_ASSET_FILE="emqx-enterprise-dashboard-$VERSION.zip"
         ;;
     *)
         echo "Unknown version $VERSION"

From 085ffb617f10a98fc7997f08ca0d4016391feaab Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 7 Jun 2023 12:05:24 +0200
Subject: [PATCH 21/29] chore: add 'mcast' to spellcheck dict

---
 scripts/spellcheck/dicts/emqx.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt
index ea1c76d96..533269c87 100644
--- a/scripts/spellcheck/dicts/emqx.txt
+++ b/scripts/spellcheck/dicts/emqx.txt
@@ -170,6 +170,7 @@ libcoap
 lifecycle
 localhost
 lwm
+mcast
 mnesia
 mountpoint
 mqueue

From a4407764f3e896e2c14890854fd3d146ae67ef1c Mon Sep 17 00:00:00 2001
From: Paulo Zulato <paulozulato@gmail.com>
Date: Fri, 2 Jun 2023 16:47:47 -0300
Subject: [PATCH 22/29] feat(connection): configurable TCP keepalive

Fixes https://emqx.atlassian.net/browse/EMQX-9852
---
 apps/emqx/src/emqx_connection.erl             | 49 +++++++++++++++---
 apps/emqx/src/emqx_listeners.erl              |  2 +-
 apps/emqx/src/emqx_schema.erl                 | 51 ++++++++++++++++++-
 apps/emqx/test/emqx_mqtt_SUITE.erl            | 26 +++++-----
 .../src/emqx_gateway_api_listeners.erl        |  3 +-
 changes/ce/feat-10933.en.md                   |  1 +
 rel/i18n/emqx_schema.hocon                    | 14 +++++
 7 files changed, 125 insertions(+), 21 deletions(-)
 create mode 100644 changes/ce/feat-10933.en.md

diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 79654e510..385c20393 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -49,7 +49,7 @@
 
 -export([
     async_set_keepalive/3,
-    async_set_keepalive/4,
+    async_set_keepalive/5,
     async_set_socket_options/2
 ]).
 
@@ -273,16 +273,30 @@ stats(#state{
 %% NOTE: This API sets TCP socket options, which has nothing to do with
 %%       the MQTT layer's keepalive (PINGREQ and PINGRESP).
 async_set_keepalive(Idle, Interval, Probes) ->
-    async_set_keepalive(self(), Idle, Interval, Probes).
+    async_set_keepalive(os:type(), self(), Idle, Interval, Probes).
 
-async_set_keepalive(Pid, Idle, Interval, Probes) ->
+async_set_keepalive({unix, linux}, Pid, Idle, Interval, Probes) ->
     Options = [
         {keepalive, true},
         {raw, 6, 4, <<Idle:32/native>>},
         {raw, 6, 5, <<Interval:32/native>>},
         {raw, 6, 6, <<Probes:32/native>>}
     ],
-    async_set_socket_options(Pid, Options).
+    async_set_socket_options(Pid, Options);
+async_set_keepalive({unix, darwin}, Pid, Idle, Interval, Probes) ->
+    Options = [
+        {keepalive, true},
+        {raw, 6, 16#10, <<Idle:32/native>>},
+        {raw, 6, 16#101, <<Interval:32/native>>},
+        {raw, 6, 16#102, <<Probes:32/native>>}
+    ],
+    async_set_socket_options(Pid, Options);
+async_set_keepalive(OS, _Pid, _Idle, _Interval, _Probes) ->
+    ?SLOG(warning, #{
+        msg => "Unsupported operation: set TCP keepalive",
+        os => OS
+    }),
+    ok.
 
 %% @doc Set custom socket options.
 %% This API is made async because the call might be originated from
@@ -353,6 +367,9 @@ init_state(
             false -> disabled
         end,
     IdleTimeout = emqx_channel:get_mqtt_conf(Zone, idle_timeout),
+
+    set_tcp_keepalive(Listener),
+
     IdleTimer = start_timer(IdleTimeout, idle_timeout),
     #state{
         transport = Transport,
@@ -948,8 +965,15 @@ handle_cast(
     }
 ) ->
     case Transport:setopts(Socket, Opts) of
-        ok -> ?tp(info, "custom_socket_options_successfully", #{opts => Opts});
-        Err -> ?tp(error, "failed_to_set_custom_socket_optionn", #{reason => Err})
+        ok ->
+            ?tp(debug, "custom_socket_options_successfully", #{opts => Opts});
+        {error, einval} ->
+            %% socket is already closed, ignore this error
+            ?tp(debug, "socket already closed", #{reason => socket_already_closed}),
+            ok;
+        Err ->
+            %% other errors
+            ?tp(error, "failed_to_set_custom_socket_option", #{reason => Err})
     end,
     State;
 handle_cast(Req, State) ->
@@ -1199,6 +1223,19 @@ inc_counter(Key, Inc) ->
     _ = emqx_pd:inc_counter(Key, Inc),
     ok.
 
+set_tcp_keepalive({quic, _Listener}) ->
+    ok;
+set_tcp_keepalive({Type, Id}) ->
+    Conf = emqx_config:get_listener_conf(Type, Id, [tcp_options, keepalive], <<"none">>),
+    case iolist_to_binary(Conf) of
+        <<"none">> ->
+            ok;
+        Value ->
+            %% the value is already validated by schema, so we do not validate it again.
+            {Idle, Interval, Probes} = emqx_schema:parse_tcp_keepalive(Value),
+            async_set_keepalive(Idle, Interval, Probes)
+    end.
+
 %%--------------------------------------------------------------------
 %% For CT tests
 %%--------------------------------------------------------------------
diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index 9a59db6e1..8c7f0ec17 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -786,7 +786,7 @@ ssl_opts(Opts) ->
 tcp_opts(Opts) ->
     maps:to_list(
         maps:without(
-            [active_n],
+            [active_n, keepalive],
             maps:get(tcp_options, Opts, #{})
         )
     ).
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 521293f7a..d65da8316 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -95,7 +95,9 @@
     non_empty_string/1,
     validations/0,
     naive_env_interpolation/1,
-    validate_server_ssl_opts/1
+    validate_server_ssl_opts/1,
+    validate_tcp_keepalive/1,
+    parse_tcp_keepalive/1
 ]).
 
 -export([qos/0]).
@@ -1389,6 +1391,15 @@ fields("tcp_opts") ->
                     default => true,
                     desc => ?DESC(fields_tcp_opts_reuseaddr)
                 }
+            )},
+        {"keepalive",
+            sc(
+                string(),
+                #{
+                    default => <<"none">>,
+                    desc => ?DESC(fields_tcp_opts_keepalive),
+                    validator => fun validate_tcp_keepalive/1
+                }
             )}
     ];
 fields("listener_ssl_opts") ->
@@ -2842,6 +2853,44 @@ validate_alarm_actions(Actions) ->
         Error -> {error, Error}
     end.
 
+validate_tcp_keepalive(Value) ->
+    case iolist_to_binary(Value) of
+        <<"none">> ->
+            ok;
+        _ ->
+            _ = parse_tcp_keepalive(Value),
+            ok
+    end.
+
+%% @doc This function is used as value validator and also run-time parser.
+parse_tcp_keepalive(Str) ->
+    try
+        [Idle, Interval, Probes] = binary:split(iolist_to_binary(Str), <<",">>, [global]),
+        %% use 10 times the Linux defaults as range limit
+        IdleInt = parse_ka_int(Idle, "Idle", 1, 7200_0),
+        IntervalInt = parse_ka_int(Interval, "Interval", 1, 75_0),
+        ProbesInt = parse_ka_int(Probes, "Probes", 1, 9_0),
+        {IdleInt, IntervalInt, ProbesInt}
+    catch
+        error:_ ->
+            throw(#{
+                reason => "Not comma separated positive integers of 'Idle,Interval,Probes' format",
+                value => Str
+            })
+    end.
+
+parse_ka_int(Bin, Name, Min, Max) ->
+    I = binary_to_integer(string:trim(Bin)),
+    case I >= Min andalso I =< Max of
+        true ->
+            I;
+        false ->
+            Msg = io_lib:format("TCP-Keepalive '~s' value must be in the rage of [~p, ~p].", [
+                Name, Min, Max
+            ]),
+            throw(#{reason => lists:flatten(Msg), value => I})
+    end.
+
 user_lookup_fun_tr(Lookup, #{make_serializable := true}) ->
     fmt_user_lookup_fun(Lookup);
 user_lookup_fun_tr(Lookup, _) ->
diff --git a/apps/emqx/test/emqx_mqtt_SUITE.erl b/apps/emqx/test/emqx_mqtt_SUITE.erl
index d0162b34b..f03c7af83 100644
--- a/apps/emqx/test/emqx_mqtt_SUITE.erl
+++ b/apps/emqx/test/emqx_mqtt_SUITE.erl
@@ -219,13 +219,15 @@ t_async_set_keepalive('end', _Config) ->
 t_async_set_keepalive(_) ->
     case os:type() of
         {unix, darwin} ->
-            %% Mac OSX don't support the feature
-            ok;
+            do_async_set_keepalive(16#10, 16#101, 16#102);
+        {unix, linux} ->
+            do_async_set_keepalive(4, 5, 6);
         _ ->
-            do_async_set_keepalive()
+            %% don't support the feature on other OS
+            ok
     end.
 
-do_async_set_keepalive() ->
+do_async_set_keepalive(OptKeepIdle, OptKeepInterval, OptKeepCount) ->
     ClientID = <<"client-tcp-keepalive">>,
     {ok, Client} = emqtt:start_link([
         {host, "localhost"},
@@ -247,19 +249,19 @@ do_async_set_keepalive() ->
     Transport = maps:get(transport, State),
     Socket = maps:get(socket, State),
     ?assert(is_port(Socket)),
-    Opts = [{raw, 6, 4, 4}, {raw, 6, 5, 4}, {raw, 6, 6, 4}],
+    Opts = [{raw, 6, OptKeepIdle, 4}, {raw, 6, OptKeepInterval, 4}, {raw, 6, OptKeepCount, 4}],
     {ok, [
-        {raw, 6, 4, <<Idle:32/native>>},
-        {raw, 6, 5, <<Interval:32/native>>},
-        {raw, 6, 6, <<Probes:32/native>>}
+        {raw, 6, OptKeepIdle, <<Idle:32/native>>},
+        {raw, 6, OptKeepInterval, <<Interval:32/native>>},
+        {raw, 6, OptKeepCount, <<Probes:32/native>>}
     ]} = Transport:getopts(Socket, Opts),
     ct:pal("Idle=~p, Interval=~p, Probes=~p", [Idle, Interval, Probes]),
-    emqx_connection:async_set_keepalive(Pid, Idle + 1, Interval + 1, Probes + 1),
+    emqx_connection:async_set_keepalive(os:type(), Pid, Idle + 1, Interval + 1, Probes + 1),
     {ok, _} = ?block_until(#{?snk_kind := "custom_socket_options_successfully"}, 1000),
     {ok, [
-        {raw, 6, 4, <<NewIdle:32/native>>},
-        {raw, 6, 5, <<NewInterval:32/native>>},
-        {raw, 6, 6, <<NewProbes:32/native>>}
+        {raw, 6, OptKeepIdle, <<NewIdle:32/native>>},
+        {raw, 6, OptKeepInterval, <<NewInterval:32/native>>},
+        {raw, 6, OptKeepCount, <<NewProbes:32/native>>}
     ]} = Transport:getopts(Socket, Opts),
     ?assertEqual(NewIdle, Idle + 1),
     ?assertEqual(NewInterval, Interval + 1),
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
index 8eea3c522..7c0b8c4bf 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
@@ -723,7 +723,8 @@ examples_listener() ->
                                 buffer => <<"10KB">>,
                                 high_watermark => <<"1MB">>,
                                 nodelay => false,
-                                reuseaddr => true
+                                reuseaddr => true,
+                                keepalive => "none"
                             }
                     }
             },
diff --git a/changes/ce/feat-10933.en.md b/changes/ce/feat-10933.en.md
new file mode 100644
index 000000000..dcef6029b
--- /dev/null
+++ b/changes/ce/feat-10933.en.md
@@ -0,0 +1 @@
+Add support for configuring TCP keep-alive in MQTT/TCP and MQTT/SSL listeners
diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon
index 54d866014..c6901f937 100644
--- a/rel/i18n/emqx_schema.hocon
+++ b/rel/i18n/emqx_schema.hocon
@@ -975,6 +975,20 @@ fields_tcp_opts_nodelay.desc:
 fields_tcp_opts_nodelay.label:
 """TCP_NODELAY"""
 
+fields_tcp_opts_keepalive.desc:
+"""
+Enable TCP keepalive for MQTT connections over TCP or SSL.
+The value is three comma separated numbers in the format of 'Idle,Interval,Probes'
+ - Idle: The number of seconds a connection needs to be idle before the server begins to send out keep-alive probes (Linux default 7200).
+ - Interval: The number of seconds between TCP keep-alive probes (Linux default 75).
+ - Probes: The maximum number of TCP keep-alive probes to send before giving up and killing the connection if no response is obtained from the other end (Linux default 9).
+For example "240,30,5" means: EMQX should start sending TCP keepalive probes after the connection is in idle for 240 seconds, and the probes are sent every 30 seconds until a response is received from the MQTT client, if it misses 5 consecutive responses, EMQX should close the connection.
+Default: 'none'
+"""
+
+fields_tcp_opts_keepalive.label:
+"""TCP keepalive options"""
+
 sysmon_top_db_username.desc:
 """Username of the PostgreSQL database"""
 

From cc8631223ed676a8bb89c2a57d14bd228725ff8b Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Wed, 7 Jun 2023 10:26:54 -0300
Subject: [PATCH 23/29] fix(schema): avoid `function_clause` error when
 compacting errors

Fixes https://emqx.atlassian.net/browse/EMQX-10168

The bridge probe API displayed the typecheck errors for the new
timeout duration types correctly, but when an user tried to create the
bridge anyway a `function_clause` error was raised when trying to
compact hocon errors:

```
09:47:19.045 [warning] [exception: :error, path: '/bridges', reason: {:case_clause, {:error, {:config_update_crashed, :function_clause}}}, stacktrace: [{:emqx_bridge_api, :create_or_update_bridge, 4, [file: '/home/thales/dev/emqx/emqx/apps/emqx_bridge/src/emqx_bridge_api.erl', line: 602]}, {:minirest_handler, :apply_callback, 3, [file: '/home/thales/dev/emqx/emqx/deps/minirest/src/minirest_handler.erl', line: 111]}, {:minirest_handler, :handle, 2, [file: '/home/thales/dev/emqx/emqx/deps/minirest/src/minirest_handler.erl', line: 44]}, {:minirest_handler, :init, 2, [file: '/home/thales/dev/emqx/emqx/deps/minirest/src/minirest_handler.erl', line: 27]}, {:cowboy_handler, :execute, 2, [file: '/home/thales/dev/emqx/emqx/deps/cowboy/src/cowboy_handler.erl', line: 41]}, {:cowboy_stream_h, :execute, 3, [file: '/home/thales/dev/emqx/emqx/deps/cowboy/src/cowboy_stream_h.erl', line: 318]}, {:cowboy_stream_h, :request_process, 3, [file: '/home/thales/dev/emqx/emqx/deps/cowboy/src/cowboy_stream_h.erl', line: 302]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 240]}]]
```

This fixes the issue so that both APIs return more friendly error messages.
---
 apps/emqx/src/emqx_hocon.erl                   | 11 ++++++++++-
 apps/emqx/src/emqx_schema.erl                  |  6 +++++-
 apps/emqx/test/emqx_schema_tests.erl           | 18 +++++++++++++++---
 .../test/emqx_resource_schema_tests.erl        |  2 +-
 4 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl
index 08192e9be..e028ebb14 100644
--- a/apps/emqx/src/emqx_hocon.erl
+++ b/apps/emqx/src/emqx_hocon.erl
@@ -100,7 +100,16 @@ no_stacktrace(Map) ->
 %% it's maybe too much when reporting to the user
 -spec compact_errors(any(), Stacktrace :: list()) -> {error, any()}.
 compact_errors({SchemaModule, Errors}, Stacktrace) ->
-    compact_errors(SchemaModule, Errors, Stacktrace).
+    compact_errors(SchemaModule, Errors, Stacktrace);
+compact_errors(ErrorContext0, _Stacktrace) when is_map(ErrorContext0) ->
+    case ErrorContext0 of
+        #{exception := #{schema_module := _Mod, message := _Msg} = Detail} ->
+            Error0 = maps:remove(exception, ErrorContext0),
+            Error = maps:merge(Error0, Detail),
+            {error, Error};
+        _ ->
+            {error, ErrorContext0}
+    end.
 
 compact_errors(SchemaModule, [Error0 | More], _Stacktrace) when is_map(Error0) ->
     Error1 =
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 521293f7a..28404a7da 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -2695,7 +2695,11 @@ do_to_timeout_duration(Str, Fn, Max, Unit) ->
                     Msg = lists:flatten(
                         io_lib:format("timeout value too large (max: ~b ~s)", [Max, Unit])
                     ),
-                    throw(Msg)
+                    throw(#{
+                        schema_module => ?MODULE,
+                        message => Msg,
+                        kind => validation_error
+                    })
             end;
         Err ->
             Err
diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl
index 58f9a94d5..b5fd156ad 100644
--- a/apps/emqx/test/emqx_schema_tests.erl
+++ b/apps/emqx/test/emqx_schema_tests.erl
@@ -886,15 +886,27 @@ timeout_types_test_() ->
             typerefl:from_string(emqx_schema:timeout_duration_s(), <<"4294967000ms">>)
         ),
         ?_assertThrow(
-            "timeout value too large (max: 4294967295 ms)",
+            #{
+                kind := validation_error,
+                message := "timeout value too large (max: 4294967295 ms)",
+                schema_module := emqx_schema
+            },
             typerefl:from_string(emqx_schema:timeout_duration(), <<"4294967296ms">>)
         ),
         ?_assertThrow(
-            "timeout value too large (max: 4294967295 ms)",
+            #{
+                kind := validation_error,
+                message := "timeout value too large (max: 4294967295 ms)",
+                schema_module := emqx_schema
+            },
             typerefl:from_string(emqx_schema:timeout_duration_ms(), <<"4294967296ms">>)
         ),
         ?_assertThrow(
-            "timeout value too large (max: 4294967 s)",
+            #{
+                kind := validation_error,
+                message := "timeout value too large (max: 4294967 s)",
+                schema_module := emqx_schema
+            },
             typerefl:from_string(emqx_schema:timeout_duration_s(), <<"4294967001ms">>)
         )
     ].
diff --git a/apps/emqx_resource/test/emqx_resource_schema_tests.erl b/apps/emqx_resource/test/emqx_resource_schema_tests.erl
index 676a42510..219861a4e 100644
--- a/apps/emqx_resource/test/emqx_resource_schema_tests.erl
+++ b/apps/emqx_resource/test/emqx_resource_schema_tests.erl
@@ -67,7 +67,7 @@ health_check_interval_validator_test_() ->
                 parse_and_check_webhook_bridge(webhook_bridge_health_check_hocon(<<"3_600_000ms">>))
             )},
         ?_assertThrow(
-            #{exception := "timeout value too large" ++ _},
+            #{exception := #{message := "timeout value too large" ++ _}},
             parse_and_check_webhook_bridge(
                 webhook_bridge_health_check_hocon(<<"150000000000000s">>)
             )

From 65483e972ed80b04f8b6d6a659ab1b2203a002d9 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 31 May 2023 11:12:53 +0800
Subject: [PATCH 24/29] refactor: use unified table define

---
 apps/emqx/{src => include}/emqx_cm.hrl        | 10 ++++++
 apps/emqx/include/emqx_router.hrl             | 31 +++++++++++++++++++
 apps/emqx/src/emqx_broker.erl                 |  7 ++---
 apps/emqx/src/emqx_channel.erl                |  3 ++
 apps/emqx/src/emqx_cm.erl                     | 15 +++------
 apps/emqx/src/emqx_cm_registry.erl            | 14 ++++-----
 apps/emqx/src/emqx_router.erl                 |  3 +-
 apps/emqx/src/emqx_router_helper.erl          |  9 +++---
 apps/emqx/src/emqx_ws_connection.erl          |  3 +-
 apps/emqx/src/proto/emqx_cm_proto_v1.erl      |  2 +-
 apps/emqx/src/proto/emqx_cm_proto_v2.erl      |  2 +-
 apps/emqx/test/emqx_cm_SUITE.erl              |  7 +++--
 .../test/emqx_quic_multistreams_SUITE.erl     |  5 +--
 apps/emqx/test/emqx_router_SUITE.erl          |  3 +-
 apps/emqx/test/emqx_router_helper_SUITE.erl   |  6 ++--
 apps/emqx/test/emqx_takeover_SUITE.erl        |  3 +-
 .../test/emqx_eviction_agent_SUITE.erl        |  3 +-
 apps/emqx_gateway/test/test_mqtt_broker.erl   |  3 +-
 apps/emqx_management/src/emqx_mgmt.erl        | 10 +++---
 .../src/emqx_mgmt_api_clients.erl             |  8 ++---
 .../src/emqx_mgmt_api_topics.erl              |  3 +-
 apps/emqx_management/src/emqx_mgmt_cli.erl    | 24 +++++++-------
 .../test/emqx_mgmt_api_topics_SUITE.erl       |  3 +-
 23 files changed, 110 insertions(+), 67 deletions(-)
 rename apps/emqx/{src => include}/emqx_cm.hrl (75%)
 create mode 100644 apps/emqx/include/emqx_router.hrl

diff --git a/apps/emqx/src/emqx_cm.hrl b/apps/emqx/include/emqx_cm.hrl
similarity index 75%
rename from apps/emqx/src/emqx_cm.hrl
rename to apps/emqx/include/emqx_cm.hrl
index 3896ea30a..ae70f131f 100644
--- a/apps/emqx/src/emqx_cm.hrl
+++ b/apps/emqx/include/emqx_cm.hrl
@@ -13,9 +13,19 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
+
 -ifndef(EMQX_CM_HRL).
 -define(EMQX_CM_HRL, true).
 
+%% Tables for channel management.
+-define(CHAN_TAB, emqx_channel).
+-define(CHAN_CONN_TAB, emqx_channel_conn).
+-define(CHAN_INFO_TAB, emqx_channel_info).
+-define(CHAN_LIVE_TAB, emqx_channel_live).
+
+%% Mria/Mnesia Tables for channel management.
+-define(CHAN_REG_TAB, emqx_channel_registry).
+
 -define(T_KICK, 5_000).
 -define(T_GET_INFO, 5_000).
 -define(T_TAKEOVER, 15_000).
diff --git a/apps/emqx/include/emqx_router.hrl b/apps/emqx/include/emqx_router.hrl
new file mode 100644
index 000000000..035ff5455
--- /dev/null
+++ b/apps/emqx/include/emqx_router.hrl
@@ -0,0 +1,31 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-ifndef(EMQX_ROUTER_HRL).
+-define(EMQX_ROUTER_HRL, true).
+
+%% ETS table for message routing
+-define(ROUTE_TAB, emqx_route).
+
+%% Mnesia table for message routing
+-define(ROUTING_NODE, emqx_routing_node).
+
+%% ETS tables for PubSub
+-define(SUBOPTION, emqx_suboption).
+-define(SUBSCRIBER, emqx_subscriber).
+-define(SUBSCRIPTION, emqx_subscription).
+
+-endif.
diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl
index 5f7c4aaf5..390b732b9 100644
--- a/apps/emqx/src/emqx_broker.erl
+++ b/apps/emqx/src/emqx_broker.erl
@@ -19,6 +19,8 @@
 -behaviour(gen_server).
 
 -include("emqx.hrl").
+-include("emqx_router.hrl").
+
 -include("logger.hrl").
 -include("types.hrl").
 -include("emqx_mqtt.hrl").
@@ -80,11 +82,6 @@
 
 -define(BROKER, ?MODULE).
 
-%% ETS tables for PubSub
--define(SUBOPTION, emqx_suboption).
--define(SUBSCRIBER, emqx_subscriber).
--define(SUBSCRIPTION, emqx_subscription).
-
 %% Guards
 -define(IS_SUBID(Id), (is_binary(Id) orelse is_atom(Id))).
 
diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index 5637bb171..75be6b905 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -1866,6 +1866,9 @@ check_pub_caps(
 %%--------------------------------------------------------------------
 %% Check Sub Authorization
 
+%% TODO: not only check topic filter. Qos chould be checked too.
+%% Not implemented yet:
+%% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7]
 check_sub_authzs(TopicFilters, Channel) ->
     check_sub_authzs(TopicFilters, Channel, []).
 
diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl
index db3d48f5d..c193cea44 100644
--- a/apps/emqx/src/emqx_cm.erl
+++ b/apps/emqx/src/emqx_cm.erl
@@ -20,6 +20,7 @@
 -behaviour(gen_server).
 
 -include("emqx.hrl").
+-include("emqx_cm.hrl").
 -include("logger.hrl").
 -include("types.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -118,14 +119,6 @@
     _Stats :: emqx_types:stats()
 }.
 
--include("emqx_cm.hrl").
-
-%% Tables for channel management.
--define(CHAN_TAB, emqx_channel).
--define(CHAN_CONN_TAB, emqx_channel_conn).
--define(CHAN_INFO_TAB, emqx_channel_info).
--define(CHAN_LIVE_TAB, emqx_channel_live).
-
 -define(CHAN_STATS, [
     {?CHAN_TAB, 'channels.count', 'channels.max'},
     {?CHAN_TAB, 'sessions.count', 'sessions.max'},
@@ -669,12 +662,12 @@ lookup_client({username, Username}) ->
     MatchSpec = [
         {{'_', #{clientinfo => #{username => '$1'}}, '_'}, [{'=:=', '$1', Username}], ['$_']}
     ],
-    ets:select(emqx_channel_info, MatchSpec);
+    ets:select(?CHAN_INFO_TAB, MatchSpec);
 lookup_client({clientid, ClientId}) ->
     [
         Rec
-     || Key <- ets:lookup(emqx_channel, ClientId),
-        Rec <- ets:lookup(emqx_channel_info, Key)
+     || Key <- ets:lookup(?CHAN_TAB, ClientId),
+        Rec <- ets:lookup(?CHAN_INFO_TAB, Key)
     ].
 
 %% @private
diff --git a/apps/emqx/src/emqx_cm_registry.erl b/apps/emqx/src/emqx_cm_registry.erl
index 8a5639486..058bb53ec 100644
--- a/apps/emqx/src/emqx_cm_registry.erl
+++ b/apps/emqx/src/emqx_cm_registry.erl
@@ -20,6 +20,7 @@
 -behaviour(gen_server).
 
 -include("emqx.hrl").
+-include("emqx_cm.hrl").
 -include("logger.hrl").
 -include("types.hrl").
 
@@ -50,7 +51,6 @@
 ]).
 
 -define(REGISTRY, ?MODULE).
--define(TAB, emqx_channel_registry).
 -define(LOCK, {?MODULE, cleanup_down}).
 
 -record(channel, {chid, pid}).
@@ -78,7 +78,7 @@ register_channel(ClientId) when is_binary(ClientId) ->
     register_channel({ClientId, self()});
 register_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
     case is_enabled() of
-        true -> mria:dirty_write(?TAB, record(ClientId, ChanPid));
+        true -> mria:dirty_write(?CHAN_REG_TAB, record(ClientId, ChanPid));
         false -> ok
     end.
 
@@ -91,14 +91,14 @@ unregister_channel(ClientId) when is_binary(ClientId) ->
     unregister_channel({ClientId, self()});
 unregister_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
     case is_enabled() of
-        true -> mria:dirty_delete_object(?TAB, record(ClientId, ChanPid));
+        true -> mria:dirty_delete_object(?CHAN_REG_TAB, record(ClientId, ChanPid));
         false -> ok
     end.
 
 %% @doc Lookup the global channels.
 -spec lookup_channels(emqx_types:clientid()) -> list(pid()).
 lookup_channels(ClientId) ->
-    [ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(?TAB, ClientId)].
+    [ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(?CHAN_REG_TAB, ClientId)].
 
 record(ClientId, ChanPid) ->
     #channel{chid = ClientId, pid = ChanPid}.
@@ -109,7 +109,7 @@ record(ClientId, ChanPid) ->
 
 init([]) ->
     mria_config:set_dirty_shard(?CM_SHARD, true),
-    ok = mria:create_table(?TAB, [
+    ok = mria:create_table(?CHAN_REG_TAB, [
         {type, bag},
         {rlog_shard, ?CM_SHARD},
         {storage, ram_copies},
@@ -166,7 +166,7 @@ cleanup_channels(Node) ->
 
 do_cleanup_channels(Node) ->
     Pat = [{#channel{pid = '$1', _ = '_'}, [{'==', {node, '$1'}, Node}], ['$_']}],
-    lists:foreach(fun delete_channel/1, mnesia:select(?TAB, Pat, write)).
+    lists:foreach(fun delete_channel/1, mnesia:select(?CHAN_REG_TAB, Pat, write)).
 
 delete_channel(Chan) ->
-    mnesia:delete_object(?TAB, Chan, write).
+    mnesia:delete_object(?CHAN_REG_TAB, Chan, write).
diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl
index 42430af5d..95b6136a7 100644
--- a/apps/emqx/src/emqx_router.erl
+++ b/apps/emqx/src/emqx_router.erl
@@ -22,6 +22,7 @@
 -include("logger.hrl").
 -include("types.hrl").
 -include_lib("mria/include/mria.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).
@@ -69,8 +70,6 @@
 
 -type dest() :: node() | {group(), node()}.
 
--define(ROUTE_TAB, emqx_route).
-
 %%--------------------------------------------------------------------
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl
index 4bff98072..78cf62d6c 100644
--- a/apps/emqx/src/emqx_router_helper.erl
+++ b/apps/emqx/src/emqx_router_helper.erl
@@ -19,6 +19,7 @@
 -behaviour(gen_server).
 
 -include("emqx.hrl").
+-include("emqx_router.hrl").
 -include("logger.hrl").
 -include("types.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -54,8 +55,6 @@
 
 -record(routing_node, {name, const = unused}).
 
--define(ROUTE, emqx_route).
--define(ROUTING_NODE, emqx_routing_node).
 -define(LOCK, {?MODULE, cleanup_routes}).
 
 -dialyzer({nowarn_function, [cleanup_routes/1]}).
@@ -185,7 +184,7 @@ code_change(_OldVsn, State, _Extra) ->
 %%--------------------------------------------------------------------
 
 stats_fun() ->
-    case ets:info(?ROUTE, size) of
+    case ets:info(?ROUTE_TAB, size) of
         undefined ->
             ok;
         Size ->
@@ -198,6 +197,6 @@ cleanup_routes(Node) ->
         #route{_ = '_', dest = {'_', Node}}
     ],
     [
-        mnesia:delete_object(?ROUTE, Route, write)
-     || Pat <- Patterns, Route <- mnesia:match_object(?ROUTE, Pat, write)
+        mnesia:delete_object(?ROUTE_TAB, Route, write)
+     || Pat <- Patterns, Route <- mnesia:match_object(?ROUTE_TAB, Pat, write)
     ].
diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl
index 00fe545eb..6cc86e5f4 100644
--- a/apps/emqx/src/emqx_ws_connection.erl
+++ b/apps/emqx/src/emqx_ws_connection.erl
@@ -18,6 +18,7 @@
 -module(emqx_ws_connection).
 
 -include("emqx.hrl").
+-include("emqx_cm.hrl").
 -include("emqx_mqtt.hrl").
 -include("logger.hrl").
 -include("types.hrl").
@@ -1034,7 +1035,7 @@ check_max_connection(Type, Listener) ->
             allow;
         Max ->
             MatchSpec = [{{'_', emqx_ws_connection}, [], [true]}],
-            Curr = ets:select_count(emqx_channel_conn, MatchSpec),
+            Curr = ets:select_count(?CHAN_CONN_TAB, MatchSpec),
             case Curr >= Max of
                 false ->
                     allow;
diff --git a/apps/emqx/src/proto/emqx_cm_proto_v1.erl b/apps/emqx/src/proto/emqx_cm_proto_v1.erl
index d81469df4..3c176d586 100644
--- a/apps/emqx/src/proto/emqx_cm_proto_v1.erl
+++ b/apps/emqx/src/proto/emqx_cm_proto_v1.erl
@@ -33,7 +33,7 @@
 ]).
 
 -include("bpapi.hrl").
--include("src/emqx_cm.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 
 introduced_in() ->
     "5.0.0".
diff --git a/apps/emqx/src/proto/emqx_cm_proto_v2.erl b/apps/emqx/src/proto/emqx_cm_proto_v2.erl
index 4208df97f..9458ba1be 100644
--- a/apps/emqx/src/proto/emqx_cm_proto_v2.erl
+++ b/apps/emqx/src/proto/emqx_cm_proto_v2.erl
@@ -34,7 +34,7 @@
 ]).
 
 -include("bpapi.hrl").
--include("src/emqx_cm.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 
 introduced_in() ->
     "5.0.0".
diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl
index 5589f5bd8..6cb58be46 100644
--- a/apps/emqx/test/emqx_cm_SUITE.erl
+++ b/apps/emqx/test/emqx_cm_SUITE.erl
@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
@@ -200,10 +201,10 @@ t_open_session_race_condition(_) ->
         end,
     Winner = WaitForDowns(Pids),
 
-    ?assertMatch([_], ets:lookup(emqx_channel, ClientId)),
+    ?assertMatch([_], ets:lookup(?CHAN_TAB, ClientId)),
     ?assertEqual([Winner], emqx_cm:lookup_channels(ClientId)),
-    ?assertMatch([_], ets:lookup(emqx_channel_conn, {ClientId, Winner})),
-    ?assertMatch([_], ets:lookup(emqx_channel_registry, ClientId)),
+    ?assertMatch([_], ets:lookup(?CHAN_CONN_TAB, {ClientId, Winner})),
+    ?assertMatch([_], ets:lookup(?CHAN_REG_TAB, ClientId)),
 
     exit(Winner, kill),
     receive
diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 1f4ccab88..1b45cb669 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -23,6 +23,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("quicer/include/quicer.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
@@ -1465,7 +1466,7 @@ t_multi_streams_emqx_ctrl_kill(Config) ->
     ),
 
     ClientId = proplists:get_value(clientid, emqtt:info(C)),
-    [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId),
+    [{ClientId, TransPid}] = ets:lookup(?CHAN_TAB, ClientId),
     exit(TransPid, kill),
 
     %% Client should be closed
@@ -1518,7 +1519,7 @@ t_multi_streams_emqx_ctrl_exit_normal(Config) ->
     ),
 
     ClientId = proplists:get_value(clientid, emqtt:info(C)),
-    [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId),
+    [{ClientId, TransPid}] = ets:lookup(?CHAN_TAB, ClientId),
 
     emqx_connection:stop(TransPid),
     %% Client exit normal.
diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl
index 2db0acf82..067f11634 100644
--- a/apps/emqx/test/emqx_router_SUITE.erl
+++ b/apps/emqx/test/emqx_router_SUITE.erl
@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
@@ -127,5 +128,5 @@ t_unexpected(_) ->
 clear_tables() ->
     lists:foreach(
         fun mnesia:clear_table/1,
-        [emqx_route, emqx_trie, emqx_trie_node]
+        [?ROUTE_TAB, ?TRIE, emqx_trie_node]
     ).
diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl
index 9fc3bd97b..c0796288e 100644
--- a/apps/emqx/test/emqx_router_helper_SUITE.erl
+++ b/apps/emqx/test/emqx_router_helper_SUITE.erl
@@ -20,11 +20,11 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -define(ROUTER_HELPER, emqx_router_helper).
--define(ROUTE_TAB, emqx_route).
 
 all() -> emqx_common_test_helpers:all(?MODULE).
 
@@ -82,9 +82,9 @@ t_monitor(_) ->
     emqx_router_helper:monitor(undefined).
 
 t_mnesia(_) ->
-    ?ROUTER_HELPER ! {mnesia_table_event, {delete, {emqx_routing_node, node()}, undefined}},
+    ?ROUTER_HELPER ! {mnesia_table_event, {delete, {?ROUTING_NODE, node()}, undefined}},
     ?ROUTER_HELPER ! {mnesia_table_event, testing},
-    ?ROUTER_HELPER ! {mnesia_table_event, {write, {emqx_routing_node, node()}, undefined}},
+    ?ROUTER_HELPER ! {mnesia_table_event, {write, {?ROUTING_NODE, node()}, undefined}},
     ?ROUTER_HELPER ! {membership, testing},
     ?ROUTER_HELPER ! {membership, {mnesia, down, node()}},
     ct:sleep(200).
diff --git a/apps/emqx/test/emqx_takeover_SUITE.erl b/apps/emqx/test/emqx_takeover_SUITE.erl
index d5b36c2c3..0b0ab7121 100644
--- a/apps/emqx/test/emqx_takeover_SUITE.erl
+++ b/apps/emqx/test/emqx_takeover_SUITE.erl
@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -117,7 +118,7 @@ load_meck(ClientId) ->
     [ChanPid] = emqx_cm:lookup_channels(ClientId),
     ChanInfo = #{conninfo := ConnInfo} = emqx_cm:get_chan_info(ClientId),
     NChanInfo = ChanInfo#{conninfo := ConnInfo#{conn_mod := fake_conn_mod}},
-    true = ets:update_element(emqx_channel_info, {ClientId, ChanPid}, {2, NChanInfo}).
+    true = ets:update_element(?CHAN_INFO_TAB, {ClientId, ChanPid}, {2, NChanInfo}).
 
 unload_meck(_ClientId) ->
     meck:unload(fake_conn_mod).
diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
index 883407a94..65df26387 100644
--- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
+++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl
@@ -11,6 +11,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx/include/asserts.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 
 -import(
     emqx_eviction_agent_test_helpers,
@@ -295,7 +296,7 @@ t_session_serialization(_Config) ->
     ?assertMatch(
         #{data := [#{clientid := <<"client_with_session">>}]},
         emqx_mgmt_api:cluster_query(
-            emqx_channel_info,
+            ?CHAN_INFO_TAB,
             #{},
             [],
             fun emqx_mgmt_api_clients:qs2ms/2,
diff --git a/apps/emqx_gateway/test/test_mqtt_broker.erl b/apps/emqx_gateway/test/test_mqtt_broker.erl
index 497f81f76..74717dc1f 100644
--- a/apps/emqx_gateway/test/test_mqtt_broker.erl
+++ b/apps/emqx_gateway/test/test_mqtt_broker.erl
@@ -24,6 +24,7 @@
 -record(state, {subscriber}).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 
 -include_lib("emqx/include/emqx_mqtt.hrl").
 
@@ -50,7 +51,7 @@ unsubscribe(Topic) ->
     gen_server:call(?MODULE, {unsubscribe, Topic}).
 
 get_subscrbied_topics() ->
-    [Topic || {_Client, Topic} <- ets:tab2list(emqx_subscription)].
+    [Topic || {_Client, Topic} <- ets:tab2list(?SUBSCRIPTION)].
 
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 24a4c3fe4..8f40190bd 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -17,6 +17,8 @@
 -module(emqx_mgmt).
 
 -include("emqx_mgmt.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
+
 -elvis([{elvis_style, invalid_dynamic_call, disable}]).
 -elvis([{elvis_style, god_modules, disable}]).
 
@@ -139,7 +141,7 @@ node_info() ->
         max_fds => proplists:get_value(
             max_fds, lists:usort(lists:flatten(erlang:system_info(check_io)))
         ),
-        connections => ets:info(emqx_channel, size),
+        connections => ets:info(?CHAN_TAB, size),
         node_status => 'running',
         uptime => proplists:get_value(uptime, BrokerInfo),
         version => iolist_to_binary(proplists:get_value(version, BrokerInfo)),
@@ -487,7 +489,7 @@ subscribe([], _ClientId, _TopicTables) ->
 -spec do_subscribe(emqx_types:clientid(), emqx_types:topic_filters()) ->
     {subscribe, _} | {error, atom()}.
 do_subscribe(ClientId, TopicTables) ->
-    case ets:lookup(emqx_channel, ClientId) of
+    case ets:lookup(?CHAN_TAB, ClientId) of
         [] -> {error, channel_not_found};
         [{_, Pid}] -> Pid ! {subscribe, TopicTables}
     end.
@@ -514,7 +516,7 @@ unsubscribe([], _ClientId, _Topic) ->
 -spec do_unsubscribe(emqx_types:clientid(), emqx_types:topic()) ->
     {unsubscribe, _} | {error, _}.
 do_unsubscribe(ClientId, Topic) ->
-    case ets:lookup(emqx_channel, ClientId) of
+    case ets:lookup(?CHAN_TAB, ClientId) of
         [] -> {error, channel_not_found};
         [{_, Pid}] -> Pid ! {unsubscribe, [emqx_topic:parse(Topic)]}
     end.
@@ -537,7 +539,7 @@ unsubscribe_batch([], _ClientId, _Topics) ->
 -spec do_unsubscribe_batch(emqx_types:clientid(), [emqx_types:topic()]) ->
     {unsubscribe_batch, _} | {error, _}.
 do_unsubscribe_batch(ClientId, Topics) ->
-    case ets:lookup(emqx_channel, ClientId) of
+    case ets:lookup(?CHAN_TAB, ClientId) of
         [] -> {error, channel_not_found};
         [{_, Pid}] -> Pid ! {unsubscribe, [emqx_topic:parse(Topic) || Topic <- Topics]}
     end.
diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index 5f2446aad..ec236c06c 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -20,6 +20,7 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
 -include_lib("emqx/include/logger.hrl").
@@ -57,7 +58,6 @@
 %% for batch operation
 -export([do_subscribe/3]).
 
--define(CLIENT_QTAB, emqx_channel_info).
 -define(TAGS, [<<"Clients">>]).
 
 -define(CLIENT_QSCHEMA, [
@@ -666,7 +666,7 @@ list_clients(QString) ->
         case maps:get(<<"node">>, QString, undefined) of
             undefined ->
                 emqx_mgmt_api:cluster_query(
-                    ?CLIENT_QTAB,
+                    ?CHAN_INFO_TAB,
                     QString,
                     ?CLIENT_QSCHEMA,
                     fun ?MODULE:qs2ms/2,
@@ -678,7 +678,7 @@ list_clients(QString) ->
                         QStringWithoutNode = maps:without([<<"node">>], QString),
                         emqx_mgmt_api:node_query(
                             Node1,
-                            ?CLIENT_QTAB,
+                            ?CHAN_INFO_TAB,
                             QStringWithoutNode,
                             ?CLIENT_QSCHEMA,
                             fun ?MODULE:qs2ms/2,
@@ -753,7 +753,7 @@ subscribe_batch(#{clientid := ClientID, topics := Topics}) ->
     %% We use emqx_channel instead of emqx_channel_info (used by the emqx_mgmt:lookup_client/2),
     %% as the emqx_channel_info table will only be populated after the hook `client.connected`
     %% has returned. So if one want to subscribe topics in this hook, it will fail.
-    case ets:lookup(emqx_channel, ClientID) of
+    case ets:lookup(?CHAN_TAB, ClientID) of
         [] ->
             {404, ?CLIENTID_NOT_FOUND};
         _ ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl
index f6e790bf4..b6294ecbf 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl
@@ -17,6 +17,7 @@
 -module(emqx_mgmt_api_topics).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
@@ -111,7 +112,7 @@ do_list(Params) ->
     case
         emqx_mgmt_api:node_query(
             node(),
-            emqx_route,
+            ?ROUTE_TAB,
             Params,
             ?TOPICS_QUERY_SCHEMA,
             fun ?MODULE:qs2ms/2,
diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl
index 27de11d0f..3ebf8e314 100644
--- a/apps/emqx_management/src/emqx_mgmt_cli.erl
+++ b/apps/emqx_management/src/emqx_mgmt_cli.erl
@@ -17,6 +17,8 @@
 -module(emqx_mgmt_cli).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx/include/logger.hrl").
 
@@ -168,7 +170,7 @@ sort_map_list_field(Field, Map) ->
 %% @doc Query clients
 
 clients(["list"]) ->
-    dump(emqx_channel, client);
+    dump(?CHAN_TAB, client);
 clients(["show", ClientId]) ->
     if_client(ClientId, fun print/1);
 clients(["kick", ClientId]) ->
@@ -182,7 +184,7 @@ clients(_) ->
     ]).
 
 if_client(ClientId, Fun) ->
-    case ets:lookup(emqx_channel, (bin(ClientId))) of
+    case ets:lookup(?CHAN_TAB, (bin(ClientId))) of
         [] -> emqx_ctl:print("Not Found.~n");
         [Channel] -> Fun({client, Channel})
     end.
@@ -191,9 +193,9 @@ if_client(ClientId, Fun) ->
 %% @doc Topics Command
 
 topics(["list"]) ->
-    dump(emqx_route, emqx_topic);
+    dump(?ROUTE_TAB, emqx_topic);
 topics(["show", Topic]) ->
-    Routes = ets:lookup(emqx_route, bin(Topic)),
+    Routes = ets:lookup(?ROUTE_TAB, bin(Topic)),
     [print({emqx_topic, Route}) || Route <- Routes];
 topics(_) ->
     emqx_ctl:usage([
@@ -204,23 +206,23 @@ topics(_) ->
 subscriptions(["list"]) ->
     lists:foreach(
         fun(Suboption) ->
-            print({emqx_suboption, Suboption})
+            print({?SUBOPTION, Suboption})
         end,
-        ets:tab2list(emqx_suboption)
+        ets:tab2list(?SUBOPTION)
     );
 subscriptions(["show", ClientId]) ->
     case ets:lookup(emqx_subid, bin(ClientId)) of
         [] ->
             emqx_ctl:print("Not Found.~n");
         [{_, Pid}] ->
-            case ets:match_object(emqx_suboption, {{'_', Pid}, '_'}) of
+            case ets:match_object(?SUBOPTION, {{'_', Pid}, '_'}) of
                 [] -> emqx_ctl:print("Not Found.~n");
-                Suboption -> [print({emqx_suboption, Sub}) || Sub <- Suboption]
+                Suboption -> [print({?SUBOPTION, Sub}) || Sub <- Suboption]
             end
     end;
 subscriptions(["add", ClientId, Topic, QoS]) ->
     if_valid_qos(QoS, fun(IntQos) ->
-        case ets:lookup(emqx_channel, bin(ClientId)) of
+        case ets:lookup(?CHAN_TAB, bin(ClientId)) of
             [] ->
                 emqx_ctl:print("Error: Channel not found!");
             [{_, Pid}] ->
@@ -230,7 +232,7 @@ subscriptions(["add", ClientId, Topic, QoS]) ->
         end
     end);
 subscriptions(["del", ClientId, Topic]) ->
-    case ets:lookup(emqx_channel, bin(ClientId)) of
+    case ets:lookup(?CHAN_TAB, bin(ClientId)) of
         [] ->
             emqx_ctl:print("Error: Channel not found!");
         [{_, Pid}] ->
@@ -841,7 +843,7 @@ print({emqx_topic, #route{topic = Topic, dest = {_, Node}}}) ->
     emqx_ctl:print("~ts -> ~ts~n", [Topic, Node]);
 print({emqx_topic, #route{topic = Topic, dest = Node}}) ->
     emqx_ctl:print("~ts -> ~ts~n", [Topic, Node]);
-print({emqx_suboption, {{Topic, Pid}, Options}}) when is_pid(Pid) ->
+print({?SUBOPTION, {{Topic, Pid}, Options}}) when is_pid(Pid) ->
     SubId = maps:get(subid, Options),
     QoS = maps:get(qos, Options, 0),
     NL = maps:get(nl, Options, 0),
diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
index e617c6dcb..a2f546267 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
@@ -18,11 +18,10 @@
 -compile(export_all).
 -compile(nowarn_export_all).
 
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
--define(ROUTE_TAB, emqx_route).
-
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 

From ea81a924f1702ab8da58a51aeb31396dea8c6238 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 1 Jun 2023 14:34:25 +0800
Subject: [PATCH 25/29] refactor: move topic() types def in `emqx_types.erl`

---
 apps/emqx/include/emqx_mqtt.hrl               |  7 +++++++
 apps/emqx/src/emqx_broker.erl                 |  4 ++--
 apps/emqx/src/emqx_frame.erl                  |  2 ++
 apps/emqx/src/emqx_mqueue.erl                 |  3 +--
 apps/emqx/src/emqx_session_router.erl         | 12 +++++------
 apps/emqx/src/emqx_shared_sub.erl             |  4 ++--
 apps/emqx/src/emqx_topic.erl                  | 21 +++++++------------
 apps/emqx/src/emqx_trie.erl                   |  6 +++---
 apps/emqx/src/emqx_types.erl                  | 15 ++++++++++---
 apps/emqx_authz/src/emqx_authz_mnesia.erl     |  2 +-
 .../src/emqx_bridge_mqtt_egress.erl           |  2 +-
 .../src/emqx_bridge_mqtt_ingress.erl          |  2 +-
 .../emqx_retainer/src/emqx_retainer_index.erl | 14 ++++++-------
 13 files changed, 52 insertions(+), 42 deletions(-)

diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl
index e4f9111da..8a40f614f 100644
--- a/apps/emqx/include/emqx_mqtt.hrl
+++ b/apps/emqx/include/emqx_mqtt.hrl
@@ -48,6 +48,13 @@
     {?MQTT_PROTO_V5, <<"MQTT">>}
 ]).
 
+%%--------------------------------------------------------------------
+%% MQTT Topic and TopitFilter byte length
+%%--------------------------------------------------------------------
+
+%% MQTT-3.1.1 and MQTT-5.0 [MQTT-4.7.3-3]
+-define(MAX_TOPIC_LEN, 65535).
+
 %%--------------------------------------------------------------------
 %% MQTT QoS Levels
 %%--------------------------------------------------------------------
diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl
index 390b732b9..1bbe4dc82 100644
--- a/apps/emqx/src/emqx_broker.erl
+++ b/apps/emqx/src/emqx_broker.erl
@@ -103,10 +103,10 @@ start_link(Pool, Id) ->
 create_tabs() ->
     TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}],
 
-    %% SubOption: {Topic, SubPid} -> SubOption
+    %% SubOption: {TopicFilter, SubPid} -> SubOption
     ok = emqx_utils_ets:new(?SUBOPTION, [ordered_set | TabOpts]),
 
-    %% Subscription: SubPid -> Topic1, Topic2, Topic3, ...
+    %% Subscription: SubPid -> TopicFilter1, TopicFilter2, TopicFilter3, ...
     %% duplicate_bag: o(1) insert
     ok = emqx_utils_ets:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]),
 
diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl
index 5999e93f3..8620f834f 100644
--- a/apps/emqx/src/emqx_frame.erl
+++ b/apps/emqx/src/emqx_frame.erl
@@ -300,6 +300,7 @@ parse_connect2(
     ConnPacket = #mqtt_packet_connect{
         proto_name = ProtoName,
         proto_ver = ProtoVer,
+        %% For bridge mode, non-standard implementation
         is_bridge = (BridgeTag =:= 8),
         clean_start = bool(CleanStart),
         will_flag = bool(WillFlag),
@@ -762,6 +763,7 @@ serialize_variable(
     #mqtt_packet_connect{
         proto_name = ProtoName,
         proto_ver = ProtoVer,
+        %% For bridge mode, non-standard implementation
         is_bridge = IsBridge,
         clean_start = CleanStart,
         will_flag = WillFlag,
diff --git a/apps/emqx/src/emqx_mqueue.erl b/apps/emqx/src/emqx_mqueue.erl
index fb43941ee..0ef7d56e5 100644
--- a/apps/emqx/src/emqx_mqueue.erl
+++ b/apps/emqx/src/emqx_mqueue.erl
@@ -75,11 +75,10 @@
 
 -export_type([mqueue/0, options/0]).
 
--type topic() :: emqx_types:topic().
 -type priority() :: infinity | integer().
 -type pq() :: emqx_pqueue:q().
 -type count() :: non_neg_integer().
--type p_table() :: ?NO_PRIORITY_TABLE | #{topic() := priority()}.
+-type p_table() :: ?NO_PRIORITY_TABLE | #{emqx_types:topic() := priority()}.
 -type options() :: #{
     max_len := count(),
     priorities => p_table(),
diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl
index 0435ddca3..47e4bdf74 100644
--- a/apps/emqx/src/emqx_session_router.erl
+++ b/apps/emqx/src/emqx_session_router.erl
@@ -57,9 +57,7 @@
     code_change/3
 ]).
 
--type group() :: binary().
-
--type dest() :: node() | {group(), node()}.
+-type dest() :: node() | {emqx_types:group(), node()}.
 
 -define(ROUTE_RAM_TAB, emqx_session_route_ram).
 -define(ROUTE_DISC_TAB, emqx_session_route_disc).
@@ -114,7 +112,7 @@ start_link(Pool, Id) ->
 %% Route APIs
 %%--------------------------------------------------------------------
 
--spec do_add_route(emqx_topic:topic(), dest()) -> ok | {error, term()}.
+-spec do_add_route(emqx_types:topic(), dest()) -> ok | {error, term()}.
 do_add_route(Topic, SessionID) when is_binary(Topic) ->
     Route = #route{topic = Topic, dest = SessionID},
     case lists:member(Route, lookup_routes(Topic)) of
@@ -135,7 +133,7 @@ do_add_route(Topic, SessionID) when is_binary(Topic) ->
     end.
 
 %% @doc Match routes
--spec match_routes(emqx_topic:topic()) -> [emqx_types:route()].
+-spec match_routes(emqx_types:topic()) -> [emqx_types:route()].
 match_routes(Topic) when is_binary(Topic) ->
     case match_trie(Topic) of
         [] -> lookup_routes(Topic);
@@ -153,7 +151,7 @@ match_trie(Topic) ->
 delete_routes(SessionID, Subscriptions) ->
     cast(pick(SessionID), {delete_routes, SessionID, Subscriptions}).
 
--spec do_delete_route(emqx_topic:topic(), dest()) -> ok | {error, term()}.
+-spec do_delete_route(emqx_types:topic(), dest()) -> ok | {error, term()}.
 do_delete_route(Topic, SessionID) ->
     Route = #route{topic = Topic, dest = SessionID},
     case emqx_topic:wildcard(Topic) of
@@ -165,7 +163,7 @@ do_delete_route(Topic, SessionID) ->
     end.
 
 %% @doc Print routes to a topic
--spec print_routes(emqx_topic:topic()) -> ok.
+-spec print_routes(emqx_types:topic()) -> ok.
 print_routes(Topic) ->
     lists:foreach(
         fun(#route{topic = To, dest = SessionID}) ->
diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl
index bc13a06c6..3a370ddba 100644
--- a/apps/emqx/src/emqx_shared_sub.erl
+++ b/apps/emqx/src/emqx_shared_sub.erl
@@ -97,7 +97,7 @@
 -define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
 -define(SUBSCRIBER_DOWN, noproc).
 
--type redispatch_to() :: ?REDISPATCH_TO(emqx_topic:group(), emqx_topic:topic()).
+-type redispatch_to() :: ?REDISPATCH_TO(emqx_types:group(), emqx_types:topic()).
 
 -record(state, {pmon}).
 
@@ -156,7 +156,7 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) ->
             end
     end.
 
--spec strategy(emqx_topic:group()) -> strategy().
+-spec strategy(emqx_types:group()) -> strategy().
 strategy(Group) ->
     try
         emqx:get_config([
diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl
index 6434154ed..f667455b4 100644
--- a/apps/emqx/src/emqx_topic.erl
+++ b/apps/emqx/src/emqx_topic.erl
@@ -16,6 +16,8 @@
 
 -module(emqx_topic).
 
+-include("emqx_mqtt.hrl").
+
 %% APIs
 -export([
     match/2,
@@ -33,18 +35,9 @@
     parse/2
 ]).
 
--export_type([
-    group/0,
-    topic/0,
-    word/0
-]).
-
--type group() :: binary().
--type topic() :: binary().
--type word() :: '' | '+' | '#' | binary().
--type words() :: list(word()).
-
--define(MAX_TOPIC_LEN, 65535).
+-type topic() :: emqx_types:topic().
+-type word() :: emqx_types:word().
+-type words() :: emqx_types:words().
 
 %%--------------------------------------------------------------------
 %% APIs
@@ -142,6 +135,7 @@ prepend(Parent0, W) ->
         _ -> <<Parent/binary, $/, (bin(W))/binary>>
     end.
 
+-spec bin(word()) -> binary().
 bin('') -> <<>>;
 bin('+') -> <<"+">>;
 bin('#') -> <<"#">>;
@@ -163,6 +157,7 @@ tokens(Topic) ->
 words(Topic) when is_binary(Topic) ->
     [word(W) || W <- tokens(Topic)].
 
+-spec word(binary()) -> word().
 word(<<>>) -> '';
 word(<<"+">>) -> '+';
 word(<<"#">>) -> '#';
@@ -185,7 +180,7 @@ feed_var(Var, Val, [Var | Words], Acc) ->
 feed_var(Var, Val, [W | Words], Acc) ->
     feed_var(Var, Val, Words, [W | Acc]).
 
--spec join(list(binary())) -> binary().
+-spec join(list(word())) -> binary().
 join([]) ->
     <<>>;
 join([W]) ->
diff --git a/apps/emqx/src/emqx_trie.erl b/apps/emqx/src/emqx_trie.erl
index b4ff49bb3..229a0e3f4 100644
--- a/apps/emqx/src/emqx_trie.erl
+++ b/apps/emqx/src/emqx_trie.erl
@@ -114,7 +114,7 @@ create_session_trie(Type) ->
 insert(Topic) when is_binary(Topic) ->
     insert(Topic, ?TRIE).
 
--spec insert_session(emqx_topic:topic()) -> ok.
+-spec insert_session(emqx_types:topic()) -> ok.
 insert_session(Topic) when is_binary(Topic) ->
     insert(Topic, session_trie()).
 
@@ -132,7 +132,7 @@ delete(Topic) when is_binary(Topic) ->
     delete(Topic, ?TRIE).
 
 %% @doc Delete a topic filter from the trie.
--spec delete_session(emqx_topic:topic()) -> ok.
+-spec delete_session(emqx_types:topic()) -> ok.
 delete_session(Topic) when is_binary(Topic) ->
     delete(Topic, session_trie()).
 
@@ -148,7 +148,7 @@ delete(Topic, Trie) when is_binary(Topic) ->
 match(Topic) when is_binary(Topic) ->
     match(Topic, ?TRIE).
 
--spec match_session(emqx_topic:topic()) -> list(emqx_topic:topic()).
+-spec match_session(emqx_types:topic()) -> list(emqx_types:topic()).
 match_session(Topic) when is_binary(Topic) ->
     match(Topic, session_trie()).
 
diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl
index a7ec0cec4..c6a567318 100644
--- a/apps/emqx/src/emqx_types.erl
+++ b/apps/emqx/src/emqx_types.erl
@@ -29,10 +29,16 @@
 -export_type([
     zone/0,
     pubsub/0,
-    topic/0,
     subid/0
 ]).
 
+-export_type([
+    group/0,
+    topic/0,
+    word/0,
+    words/0
+]).
+
 -export_type([
     socktype/0,
     sockstate/0,
@@ -122,9 +128,13 @@
 
 -type zone() :: atom().
 -type pubsub() :: publish | subscribe.
--type topic() :: emqx_topic:topic().
 -type subid() :: binary() | atom().
 
+-type group() :: binary() | undefined.
+-type topic() :: binary().
+-type word() :: '' | '+' | '#' | binary().
+-type words() :: list(word()).
+
 -type socktype() :: tcp | udp | ssl | proxy | atom().
 -type sockstate() :: idle | running | blocked | closed.
 -type conninfo() :: #{
@@ -230,7 +240,6 @@
     | {share, topic(), deliver_result()}
 ].
 -type route() :: #route{}.
--type group() :: emqx_topic:group().
 -type route_entry() :: {topic(), node()} | {topic, group()}.
 -type command() :: #command{}.
 
diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl
index 6b9c367d1..58df08653 100644
--- a/apps/emqx_authz/src/emqx_authz_mnesia.erl
+++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl
@@ -33,7 +33,7 @@
 -type clientid() :: {clientid, binary()}.
 -type who() :: username() | clientid() | all.
 
--type rule() :: {emqx_authz_rule:permission(), emqx_authz_rule:action(), emqx_topic:topic()}.
+-type rule() :: {emqx_authz_rule:permission(), emqx_authz_rule:action(), emqx_types:topic()}.
 -type rules() :: [rule()].
 
 -record(emqx_acl, {
diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl
index 673a30726..2d50d92ce 100644
--- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl
+++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl
@@ -49,7 +49,7 @@
 
 -type egress() :: #{
     local => #{
-        topic => emqx_topic:topic()
+        topic => emqx_types:topic()
     },
     remote := emqx_bridge_mqtt_msg:msgvars()
 }.
diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl
index 91ec27e74..c0b2da908 100644
--- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl
+++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl
@@ -43,7 +43,7 @@
 -type ingress() :: #{
     server := string(),
     remote := #{
-        topic := emqx_topic:topic(),
+        topic := emqx_types:topic(),
         qos => emqx_types:qos()
     },
     local := emqx_bridge_mqtt_msg:msgvars(),
diff --git a/apps/emqx_retainer/src/emqx_retainer_index.erl b/apps/emqx_retainer/src/emqx_retainer_index.erl
index 55fe7ce3f..d1e5425ed 100644
--- a/apps/emqx_retainer/src/emqx_retainer_index.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_index.erl
@@ -31,7 +31,7 @@
 -type index() :: list(pos_integer()).
 
 %% @doc Index key is a term that can be effectively searched in the index table.
--type index_key() :: {index(), {emqx_topic:words(), emqx_topic:words()}}.
+-type index_key() :: {index(), {emqx_types:words(), emqx_types:words()}}.
 
 -type match_pattern_part() :: term().
 
@@ -42,7 +42,7 @@
 %% @doc Given words of a concrete topic (`Tokens') and a list of `Indices',
 %% constructs index keys for the topic and each of the indices.
 %% `Fun' is called with each of these keys.
--spec foreach_index_key(fun((index_key()) -> any()), list(index()), emqx_topic:words()) -> ok.
+-spec foreach_index_key(fun((index_key()) -> any()), list(index()), emqx_types:words()) -> ok.
 foreach_index_key(_Fun, [], _Tokens) ->
     ok;
 foreach_index_key(Fun, [Index | Indices], Tokens) ->
@@ -59,7 +59,7 @@ foreach_index_key(Fun, [Index | Indices], Tokens) ->
 %% returns `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' term.
 %%
 %% @see foreach_index_key/3
--spec to_index_key(index(), emqx_topic:words()) -> index_key().
+-spec to_index_key(index(), emqx_types:words()) -> index_key().
 to_index_key(Index, Tokens) ->
     {Index, split_index_tokens(Index, Tokens, 1, [], [])}.
 
@@ -73,7 +73,7 @@ to_index_key(Index, Tokens) ->
 %%
 %% @see foreach_index_key/3
 %% @see to_index_key/2
--spec index_score(index(), emqx_topic:words()) -> non_neg_integer().
+-spec index_score(index(), emqx_types:words()) -> non_neg_integer().
 index_score(Index, Tokens) ->
     index_score(Index, Tokens, 1, 0).
 
@@ -92,7 +92,7 @@ select_index(Tokens, Indices) ->
 %%
 %% E.g. for `[2, 3]' index and <code>['+', <<"b">>, '+', <<"d">>]</code> wildcard topic
 %% returns <code>{[2, 3], {[<<"b">>, '_'], ['_', <<"d">>]}}</code> pattern.
--spec condition(index(), emqx_topic:words()) -> match_pattern_part().
+-spec condition(index(), emqx_types:words()) -> match_pattern_part().
 condition(Index, Tokens) ->
     {Index, condition(Index, Tokens, 1, [], [])}.
 
@@ -100,7 +100,7 @@ condition(Index, Tokens) ->
 %%
 %% E.g. for <code>['+', <<"b">>, '+', <<"d">>, '#']</code> wildcard topic
 %% returns <code>['_', <<"b">>, '_', <<"d">> | '_']</code> pattern.
--spec condition(emqx_topic:words()) -> match_pattern_part().
+-spec condition(emqx_types:words()) -> match_pattern_part().
 condition(Tokens) ->
     Tokens1 = [
         case W =:= '+' of
@@ -118,7 +118,7 @@ condition(Tokens) ->
 %%
 %% E.g given `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' index key
 %% returns `[<<"a">>, <<"b">>, <<"c">>, <<"d">>]' topic.
--spec restore_topic(index_key()) -> emqx_topic:words().
+-spec restore_topic(index_key()) -> emqx_types:words().
 restore_topic({Index, {IndexTokens, OtherTokens}}) ->
     restore_topic(Index, IndexTokens, OtherTokens, 1, []).
 

From 070e410c69473ea08ecda7905ea46dc87267fa67 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 1 Jun 2023 15:24:31 +0800
Subject: [PATCH 26/29] refactor: variable and func rename

---
 apps/emqx/src/emqx_channel.erl | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index 75be6b905..79c9ba528 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -484,13 +484,13 @@ handle_in(
             {ok, Channel}
     end;
 handle_in(
-    Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
+    SubPkt = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
     Channel = #channel{clientinfo = ClientInfo}
 ) ->
-    case emqx_packet:check(Packet) of
+    case emqx_packet:check(SubPkt) of
         ok ->
             TopicFilters0 = parse_topic_filters(TopicFilters),
-            TopicFilters1 = put_subid_in_subopts(Properties, TopicFilters0),
+            TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0),
             TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel),
             HasAuthzDeny = lists:any(
                 fun({_TopicFilter, ReasonCode}) ->
@@ -503,7 +503,10 @@ handle_in(
                 true ->
                     handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel);
                 false ->
-                    TopicFilters2 = [TopicFilter || {TopicFilter, 0} <- TupleTopicFilters0],
+                    TopicFilters2 = [
+                        TopicFilter
+                     || {TopicFilter, ?RC_SUCCESS} <- TupleTopicFilters0
+                    ],
                     TopicFilters3 = run_hooks(
                         'client.subscribe',
                         [ClientInfo, Properties],
@@ -1879,7 +1882,7 @@ check_sub_authzs(
 ) ->
     case emqx_access_control:authorize(ClientInfo, subscribe, Topic) of
         allow ->
-            check_sub_authzs(More, Channel, [{TopicFilter, 0} | Acc]);
+            check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]);
         deny ->
             check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
     end;
@@ -1895,9 +1898,9 @@ check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = ClientInfo}) ->
 %%--------------------------------------------------------------------
 %% Enrich SubId
 
-put_subid_in_subopts(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
+enrich_subopts_subid(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
     [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
-put_subid_in_subopts(_Properties, TopicFilters) ->
+enrich_subopts_subid(_Properties, TopicFilters) ->
     TopicFilters.
 
 %%--------------------------------------------------------------------

From b0e18f5e753ab35c4f605e0ee03d65d8e6b74f5b Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Thu, 1 Jun 2023 17:31:13 +0800
Subject: [PATCH 27/29] fix(shared-sub): validate share topic error send
 DISCONNECT

---
 apps/emqx/include/emqx_mqtt.hrl     |  5 +++++
 apps/emqx/src/emqx_topic.erl        | 34 +++++++++++++++++++++++++++--
 apps/emqx/test/emqx_topic_SUITE.erl | 26 ++++++++++++++++++++--
 3 files changed, 61 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl
index 8a40f614f..2bd27339c 100644
--- a/apps/emqx/include/emqx_mqtt.hrl
+++ b/apps/emqx/include/emqx_mqtt.hrl
@@ -669,6 +669,11 @@ end).
     end
 ).
 
+-define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty).
+-define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty).
+-define(SHARE_RECURSIVELY, '$share_cannot_be_used_as_real_topic_filter').
+-define(SHARE_NAME_INVALID_CHAR, share_subscription_group_name_cannot_include_wildcard).
+
 -define(FRAME_PARSE_ERROR, frame_parse_error).
 -define(FRAME_SERIALIZE_ERROR, frame_serialize_error).
 -define(THROW_FRAME_ERROR(Reason), erlang:throw({?FRAME_PARSE_ERROR, Reason})).
diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl
index f667455b4..c1d91fbf1 100644
--- a/apps/emqx/src/emqx_topic.erl
+++ b/apps/emqx/src/emqx_topic.erl
@@ -90,11 +90,15 @@ validate({Type, Topic}) when Type =:= name; Type =:= filter ->
 
 -spec validate(name | filter, topic()) -> true.
 validate(_, <<>>) ->
+    %% MQTT-5.0 [MQTT-4.7.3-1]
     error(empty_topic);
 validate(_, Topic) when is_binary(Topic) andalso (size(Topic) > ?MAX_TOPIC_LEN) ->
+    %% MQTT-5.0 [MQTT-4.7.3-3]
     error(topic_too_long);
-validate(filter, Topic) when is_binary(Topic) ->
-    validate2(words(Topic));
+validate(filter, SharedFilter = <<"$share/", _Rest/binary>>) ->
+    validate_share(SharedFilter);
+validate(filter, Filter) when is_binary(Filter) ->
+    validate2(words(Filter));
 validate(name, Topic) when is_binary(Topic) ->
     Words = words(Topic),
     validate2(Words) andalso
@@ -122,6 +126,32 @@ validate3(<<C/utf8, _Rest/binary>>) when C == $#; C == $+; C == 0 ->
 validate3(<<_/utf8, Rest/binary>>) ->
     validate3(Rest).
 
+validate_share(<<"$share/", Rest/binary>>) when
+    Rest =:= <<>> orelse Rest =:= <<"/">>
+->
+    %% MQTT-5.0 [MQTT-4.8.2-1]
+    error(?SHARE_EMPTY_FILTER);
+validate_share(<<"$share/", Rest/binary>>) ->
+    case binary:split(Rest, <<"/">>) of
+        %% MQTT-5.0 [MQTT-4.8.2-1]
+        [<<>>, _] ->
+            error(?SHARE_EMPTY_GROUP);
+        %% MQTT-5.0 [MQTT-4.7.3-1]
+        [_, <<>>] ->
+            error(?SHARE_EMPTY_FILTER);
+        [ShareName, Filter] ->
+            validate_share(ShareName, Filter)
+    end.
+
+validate_share(_, <<"$share/", _Rest/binary>>) ->
+    error(?SHARE_RECURSIVELY);
+validate_share(ShareName, Filter) ->
+    case binary:match(ShareName, [<<"+">>, <<"#">>]) of
+        %% MQTT-5.0 [MQTT-4.8.2-2]
+        nomatch -> validate2(words(Filter));
+        _ -> error(?SHARE_NAME_INVALID_CHAR)
+    end.
+
 %% @doc Prepend a topic prefix.
 %% Ensured to have only one / between prefix and suffix.
 prepend(undefined, W) ->
diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl
index cbd7e5a6d..fcbf053ba 100644
--- a/apps/emqx/test/emqx_topic_SUITE.erl
+++ b/apps/emqx/test/emqx_topic_SUITE.erl
@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx/include/emqx_placeholder.hrl").
 
 -import(
@@ -130,14 +131,35 @@ t_validate(_) ->
     true = validate({filter, <<"x">>}),
     true = validate({name, <<"x//y">>}),
     true = validate({filter, <<"sport/tennis/#">>}),
+    %% MQTT-5.0 [MQTT-4.7.3-1]
     ?assertError(empty_topic, validate({name, <<>>})),
+    ?assertError(empty_topic, validate({filter, <<>>})),
     ?assertError(topic_name_error, validate({name, <<"abc/#">>})),
     ?assertError(topic_too_long, validate({name, long_topic()})),
-    ?assertError('topic_invalid_#', validate({filter, <<"abc/#/1">>})),
     ?assertError(topic_invalid_char, validate({filter, <<"abc/#xzy/+">>})),
     ?assertError(topic_invalid_char, validate({filter, <<"abc/xzy/+9827">>})),
     ?assertError(topic_invalid_char, validate({filter, <<"sport/tennis#">>})),
-    ?assertError('topic_invalid_#', validate({filter, <<"sport/tennis/#/ranking">>})).
+    %% MQTT-5.0 [MQTT-4.7.1-1]
+    ?assertError('topic_invalid_#', validate({filter, <<"abc/#/1">>})),
+    ?assertError('topic_invalid_#', validate({filter, <<"sport/tennis/#/ranking">>})),
+    %% MQTT-5.0 [MQTT-4.8.2-1]
+    ?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share/">>})),
+    ?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share//">>})),
+    ?assertError(?SHARE_EMPTY_GROUP, validate({filter, <<"$share//t">>})),
+    ?assertError(?SHARE_EMPTY_GROUP, validate({filter, <<"$share//test">>})),
+    %% MQTT-5.0 [MQTT-4.7.3-1] for shared-sub
+    ?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share/g/">>})),
+    ?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share/g2/">>})),
+    %% MQTT-5.0 [MQTT-4.8.2-2]
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/p+q/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/m+/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/+n/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/x#y/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/x#/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/#y/1">>})),
+    %% share recursively
+    ?assertError(?SHARE_RECURSIVELY, validate({filter, <<"$share/g1/$share/t">>})),
+    true = validate({filter, <<"$share/g1/topic/$share">>}).
 
 t_sigle_level_validate(_) ->
     true = validate({filter, <<"+">>}),

From 4ea348308397aae52990632fcdb8e9c777f37a6d Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 7 Jun 2023 16:30:18 +0800
Subject: [PATCH 28/29] perf(topic): use tail recursive to joining a topic

---
 apps/emqx/src/emqx_topic.erl        | 32 +++++++++++++++--------------
 apps/emqx/test/emqx_topic_SUITE.erl |  5 ++++-
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl
index c1d91fbf1..62dca99c7 100644
--- a/apps/emqx/src/emqx_topic.erl
+++ b/apps/emqx/src/emqx_topic.erl
@@ -39,6 +39,11 @@
 -type word() :: emqx_types:word().
 -type words() :: emqx_types:words().
 
+%% Guards
+-define(MULTI_LEVEL_WILDCARD_NOT_LAST(C, REST),
+    ((C =:= '#' orelse C =:= <<"#">>) andalso REST =/= [])
+).
+
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -110,7 +115,8 @@ validate2([]) ->
 % end with '#'
 validate2(['#']) ->
     true;
-validate2(['#' | Words]) when length(Words) > 0 ->
+%% MQTT-5.0 [MQTT-4.7.1-1]
+validate2([C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
     error('topic_invalid_#');
 validate2(['' | Words]) ->
     validate2(Words);
@@ -213,20 +219,16 @@ feed_var(Var, Val, [W | Words], Acc) ->
 -spec join(list(word())) -> binary().
 join([]) ->
     <<>>;
-join([W]) ->
-    bin(W);
-join(Words) ->
-    {_, Bin} = lists:foldr(
-        fun
-            (W, {true, Tail}) ->
-                {false, <<W/binary, Tail/binary>>};
-            (W, {false, Tail}) ->
-                {false, <<W/binary, "/", Tail/binary>>}
-        end,
-        {true, <<>>},
-        [bin(W) || W <- Words]
-    ),
-    Bin.
+join([Word | Words]) ->
+    do_join(bin(Word), Words).
+
+do_join(TopicAcc, []) ->
+    TopicAcc;
+%% MQTT-5.0 [MQTT-4.7.1-1]
+do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
+    error('topic_invalid_#');
+do_join(TopicAcc, [Word | Words]) ->
+    do_join(<<TopicAcc/binary, "/", (bin(Word))/binary>>, Words).
 
 -spec parse(topic() | {topic(), map()}) -> {topic(), #{share => binary()}}.
 parse(TopicFilter) when is_binary(TopicFilter) ->
diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl
index fcbf053ba..521efe751 100644
--- a/apps/emqx/test/emqx_topic_SUITE.erl
+++ b/apps/emqx/test/emqx_topic_SUITE.erl
@@ -199,7 +199,10 @@ t_join(_) ->
     ?assertEqual(<<"+//#">>, join(['+', '', '#'])),
     ?assertEqual(<<"x/y/z/+">>, join([<<"x">>, <<"y">>, <<"z">>, '+'])),
     ?assertEqual(<<"/ab/cd/ef/">>, join(words(<<"/ab/cd/ef/">>))),
-    ?assertEqual(<<"ab/+/#">>, join(words(<<"ab/+/#">>))).
+    ?assertEqual(<<"ab/+/#">>, join(words(<<"ab/+/#">>))),
+    %% MQTT-5.0 [MQTT-4.7.1-1]
+    ?assertError('topic_invalid_#', join(['+', <<"a">>, '#', <<"b">>, '', '+'])),
+    ?assertError('topic_invalid_#', join(['+', <<"c">>, <<"#">>, <<"d">>, '', '+'])).
 
 t_systop(_) ->
     SysTop1 = iolist_to_binary(["$SYS/brokers/", atom_to_list(node()), "/xyz"]),

From 7d073c0d3def42bb8ab7e8a0a92ebbaa4fd12726 Mon Sep 17 00:00:00 2001
From: JimMoen <LnJimMoen@outlook.com>
Date: Wed, 7 Jun 2023 23:06:47 +0800
Subject: [PATCH 29/29] style: make static_checks happy

---
 rel/i18n/emqx_schema.hocon | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon
index c6901f937..78dd2ecaa 100644
--- a/rel/i18n/emqx_schema.hocon
+++ b/rel/i18n/emqx_schema.hocon
@@ -976,15 +976,13 @@ fields_tcp_opts_nodelay.label:
 """TCP_NODELAY"""
 
 fields_tcp_opts_keepalive.desc:
-"""
-Enable TCP keepalive for MQTT connections over TCP or SSL.
+"""Enable TCP keepalive for MQTT connections over TCP or SSL.
 The value is three comma separated numbers in the format of 'Idle,Interval,Probes'
  - Idle: The number of seconds a connection needs to be idle before the server begins to send out keep-alive probes (Linux default 7200).
  - Interval: The number of seconds between TCP keep-alive probes (Linux default 75).
  - Probes: The maximum number of TCP keep-alive probes to send before giving up and killing the connection if no response is obtained from the other end (Linux default 9).
 For example "240,30,5" means: EMQX should start sending TCP keepalive probes after the connection is in idle for 240 seconds, and the probes are sent every 30 seconds until a response is received from the MQTT client, if it misses 5 consecutive responses, EMQX should close the connection.
-Default: 'none'
-"""
+Default: 'none'"""
 
 fields_tcp_opts_keepalive.label:
 """TCP keepalive options"""