From 4290847b9dedcf9b9a39697b9d04a9ac9180dd85 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Wed, 9 Nov 2022 17:47:47 +0800
Subject: [PATCH 01/69] feat: filter out messages which the source client is
 banned when delivering the retained message

---
 apps/emqx_modules/test/emqx_delayed_SUITE.erl |  3 +-
 .../src/emqx_retainer_dispatcher.erl          | 16 +++++++-
 .../test/emqx_retainer_SUITE.erl              | 41 +++++++++++++++++++
 3 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl
index 5864646ad..3d1576b0b 100644
--- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl
+++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl
@@ -37,8 +37,7 @@
 }).
 
 all() ->
-    [t_banned_delayed].
-%%    emqx_common_test_helpers:all(?MODULE).
+    emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
     ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF, #{
diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl
index f52fd982c..c4df41ca4 100644
--- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl
@@ -20,6 +20,7 @@
 
 -include("emqx_retainer.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 %% API
 -export([
@@ -286,7 +287,20 @@ do_deliver(Msgs, DeliverNum, Pid, Topic, Limiter) ->
     end.
 
 do_deliver([Msg | T], Pid, Topic) ->
-    Pid ! {deliver, Topic, Msg},
+    case emqx_banned:look_up({clientid, Msg#message.from}) of
+        [] ->
+            Pid ! {deliver, Topic, Msg},
+            ok;
+        _ ->
+            ?tp(
+                notice,
+                ignore_retained_message_deliver,
+                #{
+                    reason => "client is banned",
+                    clienid => Msg#message.from
+                }
+            )
+    end,
     do_deliver(T, Pid, Topic);
 do_deliver([], _, _) ->
     ok.
diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl
index 09e6c4bb4..86eaa4255 100644
--- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl
+++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl
@@ -639,6 +639,47 @@ test_disable_then_start(_Config) ->
     ?assertNotEqual([], gproc_pool:active_workers(emqx_retainer_dispatcher)),
     ok.
 
+t_deliver_when_banned(_) ->
+    ClientId = <<"c1">>,
+
+    {ok, C1} = emqtt:start_link([{clientid, ClientId}, {clean_start, true}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(C1),
+
+    lists:foreach(
+        fun(I) ->
+            Topic = erlang:list_to_binary(io_lib:format("retained/~p", [I])),
+            emqtt:publish(
+                C1,
+                Topic,
+                <<"this is a retained message">>,
+                [{qos, 0}, {retain, true}]
+            )
+        end,
+        lists:seq(1, 3)
+    ),
+
+    Now = erlang:system_time(second),
+    Who = {clientid, ClientId},
+    emqx_banned:create(#{
+        who => Who,
+        by => <<"test">>,
+        reason => <<"test">>,
+        at => Now,
+        until => Now + 120
+    }),
+
+    timer:sleep(100),
+    snabbkaffe:start_trace(),
+    {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/+">>, [{qos, 0}, {rh, 0}]),
+    timer:sleep(500),
+
+    Trace = snabbkaffe:collect_trace(),
+    ?assertEqual(3, length(?of_kind(ignore_retained_message_deliver, Trace))),
+    snabbkaffe:stop(),
+    emqx_banned:delete(Who),
+    {ok, #{}, [0]} = emqtt:unsubscribe(C1, <<"retained/+">>),
+    ok = emqtt:disconnect(C1).
+
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------

From cd2cf15677a2c0d328fab0e0f464dcd2ab3252df Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Wed, 9 Nov 2022 18:18:59 +0800
Subject: [PATCH 02/69] chore: update changes

---
 changes/v5.0.11-en.md | 3 +++
 changes/v5.0.11-zh.md | 6 +++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index 9fbc2225f..e020590b2 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -2,5 +2,8 @@
 
 ## Enhancements
 
+- Improve the integration of the `banned` and the `retain` feature [#9326](https://github.com/emqx/emqx/pull/9326).
+  The retained messages that its source client is banned will be filtered out when they are delivered.
+
 ## Bug fixes
 
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index cea0f10fb..c8a143d94 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -2,4 +2,8 @@
 
 ## 增强
 
-## 修复
+- 增强 `封禁` 和 `保留消息` 这两个功能的集成性 [#9332](https://github.com/emqx/emqx/pull/9332)。
+  现在投递保留消息前，会先过滤掉来源客户端被封禁了的那些消息。
+
+## Bug fixes
+

From a66a4ffa49cab76b99cf3f331fd7bd10d3dfb8a3 Mon Sep 17 00:00:00 2001
From: raoxiaoli <xiaoli.rao@emqx.io>
Date: Mon, 7 Nov 2022 16:27:45 +0800
Subject: [PATCH 03/69] fix(helm): fix existingName does not take effect

when ssl.enabled and ssl.useExisting are true, ssl.existingName does not take effect
---
 changes/v5.0.10-en.md                                |  2 ++
 changes/v5.0.10-zh.md                                |  2 ++
 .../emqx-enterprise/templates/StatefulSet.yaml       |  2 +-
 deploy/charts/emqx-enterprise/templates/_helpers.tpl | 12 ++++++++++++
 deploy/charts/emqx/templates/StatefulSet.yaml        |  2 +-
 deploy/charts/emqx/templates/_helpers.tpl            | 12 ++++++++++++
 6 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md
index fe3aa46e6..0f9583946 100644
--- a/changes/v5.0.10-en.md
+++ b/changes/v5.0.10-en.md
@@ -32,3 +32,5 @@
   the encoding (to JSON) of the event will fail.
 
 - Fix bad HTTP response status code for `/gateways` API, when Gateway name is unknown, it should return `404` instead of `400` [#9268](https://github.com/emqx/emqx/pull/9268).
+
+- Fix `ssl.existingName` option of  helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307).
diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md
index 01c49b49e..564943d73 100644
--- a/changes/v5.0.10-zh.md
+++ b/changes/v5.0.10-zh.md
@@ -30,3 +30,5 @@
   `$events/message_dropped`, 如果消息事件是共享订阅产生的，在编码（到 JSON 格式）过程中会失败。
 
 - 修复 HTTP API `/gateways` 的返回状态码，未知 Gateway 名字应返回 `404` 而不是 `400` [#9268](https://github.com/emqx/emqx/pull/9268)。
+
+- 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。
diff --git a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml
index 4b970e183..5f259dc35 100644
--- a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml
@@ -56,7 +56,7 @@ spec:
       {{- if .Values.ssl.enabled }}
       - name: ssl-cert
         secret:
-          secretName: {{ include "emqx.fullname" . }}-tls
+          secretName: {{ include "emqx.ssl.secretName" . }}
       {{- end }}
       {{- if not .Values.persistence.enabled }}
       - name: emqx-data
diff --git a/deploy/charts/emqx-enterprise/templates/_helpers.tpl b/deploy/charts/emqx-enterprise/templates/_helpers.tpl
index ad01d5862..e55740b01 100644
--- a/deploy/charts/emqx-enterprise/templates/_helpers.tpl
+++ b/deploy/charts/emqx-enterprise/templates/_helpers.tpl
@@ -30,3 +30,15 @@ Create chart name and version as used by the chart label.
 {{- define "emqx.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+
+{{/*
+Get ssl secret name .
+*/}}
+{{- define "emqx.ssl.secretName" -}}
+{{- if and .Values.ssl.useExisting .Values.ssl.existingName -}}
+    {{ .Values.ssl.existingName }}
+{{- else -}}
+    {{ include "emqx.fullname" . }}-tls
+{{- end -}}
+{{- end -}}
diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml
index 4b970e183..5f259dc35 100644
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@@ -56,7 +56,7 @@ spec:
       {{- if .Values.ssl.enabled }}
       - name: ssl-cert
         secret:
-          secretName: {{ include "emqx.fullname" . }}-tls
+          secretName: {{ include "emqx.ssl.secretName" . }}
       {{- end }}
       {{- if not .Values.persistence.enabled }}
       - name: emqx-data
diff --git a/deploy/charts/emqx/templates/_helpers.tpl b/deploy/charts/emqx/templates/_helpers.tpl
index ad01d5862..e55740b01 100644
--- a/deploy/charts/emqx/templates/_helpers.tpl
+++ b/deploy/charts/emqx/templates/_helpers.tpl
@@ -30,3 +30,15 @@ Create chart name and version as used by the chart label.
 {{- define "emqx.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+
+{{/*
+Get ssl secret name .
+*/}}
+{{- define "emqx.ssl.secretName" -}}
+{{- if and .Values.ssl.useExisting .Values.ssl.existingName -}}
+    {{ .Values.ssl.existingName }}
+{{- else -}}
+    {{ include "emqx.fullname" . }}-tls
+{{- end -}}
+{{- end -}}

From 03db5ba8bf273484c046e4cf733f3b0e3007eec9 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 11 Nov 2022 08:43:28 +0100
Subject: [PATCH 04/69] test: change CT_READABLE default value to true

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ea617a248..246a2d522 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ PKG_PROFILES := emqx-pkg emqx-enterprise-pkg
 PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
 
 CT_NODE_NAME ?= 'test@127.0.0.1'
-CT_READABLE ?= false
+CT_READABLE ?= true
 
 export REBAR_GIT_CLONE_OPTIONS += --depth=1
 

From d636f283834bb859095e8edb9950e2e77711f860 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Fri, 11 Nov 2022 16:57:39 +0800
Subject: [PATCH 05/69] chore: fix the bad change logs introduced in #9307

---
 changes/v5.0.10-en.md | 3 ---
 changes/v5.0.10-zh.md | 3 ---
 changes/v5.0.11-en.md | 2 ++
 changes/v5.0.11-zh.md | 3 +++
 4 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md
index 124e07063..42c3f6610 100644
--- a/changes/v5.0.10-en.md
+++ b/changes/v5.0.10-en.md
@@ -40,9 +40,6 @@
 
 - Fix bad HTTP response status code for `/gateways` API, when Gateway name is unknown, it should return `404` instead of `400` [#9268](https://github.com/emqx/emqx/pull/9268).
 
-
-- Fix `ssl.existingName` option of  helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307).
-
 - Fix incorrect topic authorize checking of delayed messages [#9290](https://github.com/emqx/emqx/pull/9290).
   Now will determine the actual topic of the delayed messages, e.g. `$delayed/1/t/foo` will be treated as `t/foo` in authorize checks.
 
diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md
index 89f345bcf..bbaa758b3 100644
--- a/changes/v5.0.10-zh.md
+++ b/changes/v5.0.10-zh.md
@@ -38,9 +38,6 @@
 
 - 修复 HTTP API `/gateways` 的返回状态码，未知 Gateway 名字应返回 `404` 而不是 `400` [#9268](https://github.com/emqx/emqx/pull/9268)。
 
-
-- 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。
-
 - 修复延迟消息的主题授权判断不正确的问题 [#9290](https://github.com/emqx/emqx/pull/9290)。
   现在将会对延迟消息中的真实主题进行授权判断，比如，`$delayed/1/t/foo` 会被当作 `t/foo` 进行判断。
 
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index 9fbc2225f..a1618acf1 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -4,3 +4,5 @@
 
 ## Bug fixes
 
+- Fix `ssl.existingName` option of  helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307).
+
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index cea0f10fb..348106111 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -3,3 +3,6 @@
 ## 增强
 
 ## 修复
+
+- 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。
+

From 6b0de714bc9b98bad1b0b4db998e881f60e76c85 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 10 Nov 2022 10:23:26 +0800
Subject: [PATCH 06/69] chore: bump retainer version

---
 apps/emqx_retainer/src/emqx_retainer.app.src        | 2 +-
 apps/emqx_retainer/src/emqx_retainer_dispatcher.erl | 2 +-
 changes/v5.0.11-en.md                               | 4 ++--
 changes/v5.0.11-zh.md                               | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src
index c49794cfe..844277ba6 100644
--- a/apps/emqx_retainer/src/emqx_retainer.app.src
+++ b/apps/emqx_retainer/src/emqx_retainer.app.src
@@ -2,7 +2,7 @@
 {application, emqx_retainer, [
     {description, "EMQX Retainer"},
     % strict semver, bump manually!
-    {vsn, "5.0.6"},
+    {vsn, "5.0.7"},
     {modules, []},
     {registered, [emqx_retainer_sup]},
     {applications, [kernel, stdlib, emqx]},
diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl
index c4df41ca4..abecbbeb1 100644
--- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl
@@ -297,7 +297,7 @@ do_deliver([Msg | T], Pid, Topic) ->
                 ignore_retained_message_deliver,
                 #{
                     reason => "client is banned",
-                    clienid => Msg#message.from
+                    clientid => Msg#message.from
                 }
             )
     end,
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index e020590b2..e9f005949 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -2,8 +2,8 @@
 
 ## Enhancements
 
-- Improve the integration of the `banned` and the `retain` feature [#9326](https://github.com/emqx/emqx/pull/9326).
-  The retained messages that its source client is banned will be filtered out when they are delivered.
+- Security enhancement for retained messages [#9326](https://github.com/emqx/emqx/pull/9326).
+  The retained messages will not be published if the publisher client is banned.
 
 ## Bug fixes
 
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index c8a143d94..edf3418e4 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -2,7 +2,7 @@
 
 ## 增强
 
-- 增强 `封禁` 和 `保留消息` 这两个功能的集成性 [#9332](https://github.com/emqx/emqx/pull/9332)。
+- 增强 `保留消息` 的安全性 [#9332](https://github.com/emqx/emqx/pull/9332)。
   现在投递保留消息前，会先过滤掉来源客户端被封禁了的那些消息。
 
 ## Bug fixes

From 12ba8312466cbaea369ec35d9ac2eb7463a025f6 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 9 Nov 2022 15:43:47 +0100
Subject: [PATCH 07/69] fix(emqx_authn_api): return 404 for status of unknown
 authenticator

This also makes sure we call the same code everytime we access an authenticator.
Moreover we return a 500 in case a remote call fails due to technical issues.
---
 apps/emqx_authn/src/emqx_authn.app.src        |  2 +-
 apps/emqx_authn/src/emqx_authn_api.erl        | 49 ++++++++++++++-----
 apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 33 +++++++++++++
 changes/v5.0.11-en.md                         |  1 +
 changes/v5.0.11-zh.md                         |  3 +-
 5 files changed, 73 insertions(+), 15 deletions(-)

diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src
index 992e1452b..9f66a978c 100644
--- a/apps/emqx_authn/src/emqx_authn.app.src
+++ b/apps/emqx_authn/src/emqx_authn.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authn, [
     {description, "EMQX Authentication"},
-    {vsn, "0.1.8"},
+    {vsn, "0.1.9"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]},
diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl
index 6b5a8c5cd..f6627b3be 100644
--- a/apps/emqx_authn/src/emqx_authn_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_api.erl
@@ -30,6 +30,7 @@
 -define(BAD_REQUEST, 'BAD_REQUEST').
 -define(NOT_FOUND, 'NOT_FOUND').
 -define(ALREADY_EXISTS, 'ALREADY_EXISTS').
+-define(INTERNAL_ERROR, 'INTERNAL_ERROR').
 
 % Swagger
 
@@ -224,7 +225,8 @@ schema("/authentication/:id/status") ->
                     hoconsc:ref(emqx_authn_schema, "metrics_status_fields"),
                     status_metrics_example()
                 ),
-                400 => error_codes([?BAD_REQUEST], <<"Bad Request">>)
+                404 => error_codes([?NOT_FOUND], <<"Not Found">>),
+                500 => error_codes([?INTERNAL_ERROR], <<"Internal Service Error">>)
             }
         }
     };
@@ -576,7 +578,11 @@ authenticator(delete, #{bindings := #{id := AuthenticatorID}}) ->
     delete_authenticator([authentication], ?GLOBAL, AuthenticatorID).
 
 authenticator_status(get, #{bindings := #{id := AuthenticatorID}}) ->
-    lookup_from_all_nodes(?GLOBAL, AuthenticatorID).
+    with_authenticator(
+        AuthenticatorID,
+        [authentication],
+        fun(_) -> lookup_from_all_nodes(?GLOBAL, AuthenticatorID) end
+    ).
 
 listener_authenticators(post, #{bindings := #{listener_id := ListenerID}, body := Config}) ->
     with_listener(
@@ -647,8 +653,12 @@ listener_authenticator_status(
 ) ->
     with_listener(
         ListenerID,
-        fun(_, _, ChainName) ->
-            lookup_from_all_nodes(ChainName, AuthenticatorID)
+        fun(Type, Name, ChainName) ->
+            with_authenticator(
+                AuthenticatorID,
+                [listeners, Type, Name, authentication],
+                fun(_) -> lookup_from_all_nodes(ChainName, AuthenticatorID) end
+            )
         end
     ).
 
@@ -774,6 +784,18 @@ listener_authenticator_user(delete, #{
 %% Internal functions
 %%------------------------------------------------------------------------------
 
+with_authenticator(AuthenticatorID, ConfKeyPath, Fun) ->
+    case find_authenticator_config(AuthenticatorID, ConfKeyPath) of
+        {ok, AuthenticatorConfig} ->
+            Fun(AuthenticatorConfig);
+        {error, Reason} ->
+            serialize_error(Reason)
+    end.
+
+find_authenticator_config(AuthenticatorID, ConfKeyPath) ->
+    AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath),
+    find_config(AuthenticatorID, AuthenticatorsConfig).
+
 with_listener(ListenerID, Fun) ->
     case find_listener(ListenerID) of
         {ok, {BType, BName}} ->
@@ -836,13 +858,13 @@ list_authenticators(ConfKeyPath) ->
     {200, NAuthenticators}.
 
 list_authenticator(_, ConfKeyPath, AuthenticatorID) ->
-    AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath),
-    case find_config(AuthenticatorID, AuthenticatorsConfig) of
-        {ok, AuthenticatorConfig} ->
-            {200, maps:put(id, AuthenticatorID, convert_certs(AuthenticatorConfig))};
-        {error, Reason} ->
-            serialize_error(Reason)
-    end.
+    with_authenticator(
+        AuthenticatorID,
+        ConfKeyPath,
+        fun(AuthenticatorConfig) ->
+            {200, maps:put(id, AuthenticatorID, convert_certs(AuthenticatorConfig))}
+        end
+    ).
 
 resource_provider() ->
     [
@@ -877,7 +899,8 @@ lookup_from_local_node(ChainName, AuthenticatorID) ->
 
 lookup_from_all_nodes(ChainName, AuthenticatorID) ->
     Nodes = mria_mnesia:running_nodes(),
-    case is_ok(emqx_authn_proto_v1:lookup_from_all_nodes(Nodes, ChainName, AuthenticatorID)) of
+    LookupResult = emqx_authn_proto_v1:lookup_from_all_nodes(Nodes, ChainName, AuthenticatorID),
+    case is_ok(LookupResult) of
         {ok, ResList} ->
             {StatusMap, MetricsMap, ResourceMetricsMap, ErrorMap} = make_result_map(ResList),
             AggregateStatus = aggregate_status(maps:values(StatusMap)),
@@ -901,7 +924,7 @@ lookup_from_all_nodes(ChainName, AuthenticatorID) ->
                 node_error => HelpFun(maps:map(Fun, ErrorMap), reason)
             }};
         {error, ErrL} ->
-            {400, #{
+            {500, #{
                 code => <<"INTERNAL_ERROR">>,
                 message => list_to_binary(io_lib:format("~p", [ErrL]))
             }}
diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
index 9d999c820..64247f2bc 100644
--- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl
@@ -39,6 +39,9 @@ all() ->
 groups() ->
     [].
 
+init_per_testcase(t_authenticator_fail, Config) ->
+    meck:expect(emqx_authn_proto_v1, lookup_from_all_nodes, 3, [{error, {exception, badarg}}]),
+    init_per_testcase(default, Config);
 init_per_testcase(_, Config) ->
     {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
     emqx_authn_test_lib:delete_authenticators(
@@ -54,6 +57,12 @@ init_per_testcase(_, Config) ->
     {atomic, ok} = mria:clear_table(emqx_authn_mnesia),
     Config.
 
+end_per_testcase(t_authenticator_fail, Config) ->
+    meck:unload(emqx_authn_proto_v1),
+    Config;
+end_per_testcase(_, Config) ->
+    Config.
+
 init_per_suite(Config) ->
     emqx_config:erase(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY),
     _ = application:load(emqx_conf),
@@ -90,6 +99,21 @@ t_authenticators(_) ->
 t_authenticator(_) ->
     test_authenticator([]).
 
+t_authenticator_fail(_) ->
+    ValidConfig0 = emqx_authn_test_lib:http_example(),
+    {ok, 200, _} = request(
+        post,
+        uri([?CONF_NS]),
+        ValidConfig0
+    ),
+    ?assertMatch(
+        {ok, 500, _},
+        request(
+            get,
+            uri([?CONF_NS, "password_based:http", "status"])
+        )
+    ).
+
 t_authenticator_users(_) ->
     test_authenticator_users([]).
 
@@ -247,6 +271,15 @@ test_authenticator(PathPrefix) ->
         <<"connected">>,
         LookFun([<<"status">>])
     ),
+
+    ?assertMatch(
+        {ok, 404, _},
+        request(
+            get,
+            uri(PathPrefix ++ [?CONF_NS, "unknown_auth_chain", "status"])
+        )
+    ),
+
     {ok, 404, _} = request(
         get,
         uri(PathPrefix ++ [?CONF_NS, "password_based:redis"])
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index e9f005949..b73bb4247 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -7,3 +7,4 @@
 
 ## Bug fixes
 
+- Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index edf3418e4..959061f6a 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -5,5 +5,6 @@
 - 增强 `保留消息` 的安全性 [#9332](https://github.com/emqx/emqx/pull/9332)。
   现在投递保留消息前，会先过滤掉来源客户端被封禁了的那些消息。
 
-## Bug fixes
+## 修复
 
+- 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。

From b5a84b8b0f17eeafda3e98ca03515642cfb59964 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Thu, 10 Nov 2022 17:35:09 +0100
Subject: [PATCH 08/69] test: increase timeout creating listeners

---
 apps/emqx/test/emqx_trace_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/test/emqx_trace_SUITE.erl b/apps/emqx/test/emqx_trace_SUITE.erl
index 76e8606fd..0c55687d0 100644
--- a/apps/emqx/test/emqx_trace_SUITE.erl
+++ b/apps/emqx/test/emqx_trace_SUITE.erl
@@ -40,7 +40,7 @@ init_per_suite(Config) ->
         ?wait_async_action(
             emqx_common_test_helpers:start_apps([]),
             #{?snk_kind := listener_started, bind := 1883},
-            timer:seconds(10)
+            timer:seconds(100)
         ),
         fun(Trace) ->
             %% more than one listener

From 2f4b2ba405dc3f1e5018bb35a89c6ff4dbf1211d Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Fri, 11 Nov 2022 16:52:29 +0800
Subject: [PATCH 09/69] fix(mgmt_api): Convert only what is needed when parsing
 subscription information

---
 apps/emqx_management/src/emqx_mgmt_api_clients.erl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index beff0d53e..f4fe0387f 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -584,13 +584,13 @@ authz_cache(delete, #{bindings := Bindings}) ->
     clean_authz_cache(Bindings).
 
 subscribe(post, #{bindings := #{clientid := ClientID}, body := TopicInfo}) ->
-    Opts = emqx_map_lib:unsafe_atom_key_map(TopicInfo),
+    Opts = to_topic_info(TopicInfo),
     subscribe(Opts#{clientid => ClientID}).
 
 subscribe_batch(post, #{bindings := #{clientid := ClientID}, body := TopicInfos}) ->
     Topics =
         [
-            emqx_map_lib:unsafe_atom_key_map(TopicInfo)
+            to_topic_info(TopicInfo)
          || TopicInfo <- TopicInfos
         ],
     subscribe_batch(#{clientid => ClientID, topics => Topics}).
@@ -973,3 +973,7 @@ format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) ->
         result => AuthzResult,
         updated_time => Timestamp
     }.
+
+to_topic_info(Data) ->
+    M = maps:with([<<"topic">>, <<"qos">>, <<"nl">>, <<"rap">>, <<"rh">>], Data),
+    emqx_map_lib:safe_atom_key_map(M).

From d9e7d365804fa9a7fc43a8f12469651232f5f6e2 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Mon, 14 Nov 2022 09:45:38 +0800
Subject: [PATCH 10/69] chore: bump version && update changes

---
 apps/emqx_management/src/emqx_management.app.src | 2 +-
 changes/v5.0.11-en.md                            | 2 ++
 changes/v5.0.11-zh.md                            | 2 ++
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index b91b6a8b1..ab726cbb2 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.0.7"},
+    {vsn, "5.0.8"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index b73bb4247..f35d11719 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -5,6 +5,8 @@
 - Security enhancement for retained messages [#9326](https://github.com/emqx/emqx/pull/9326).
   The retained messages will not be published if the publisher client is banned.
 
+- Security enhancement for the `subscribe` API [#9355](https://github.com/emqx/emqx/pull/9355).
+
 ## Bug fixes
 
 - Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 959061f6a..c16c3193a 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -5,6 +5,8 @@
 - 增强 `保留消息` 的安全性 [#9332](https://github.com/emqx/emqx/pull/9332)。
   现在投递保留消息前，会先过滤掉来源客户端被封禁了的那些消息。
 
+- 增强订阅 API 的安全性 [#9355](https://github.com/emqx/emqx/pull/9355)。
+
 ## 修复
 
 - 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。

From c079760b0afa55d4443d86cb7a5b6f1f70d43ea5 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Tue, 15 Nov 2022 15:41:01 +0800
Subject: [PATCH 11/69] fix(JWT): make the `exp` to be optional claim

---
 .../src/simple_authn/emqx_authn_jwt.erl       |  4 +-
 apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl | 44 +++++++++++++++++++
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
index 0509cfd62..8c43332ca 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl
@@ -365,11 +365,11 @@ verify(JWT, JWKs, VerifyClaims, AclClaimName) ->
 acl(Claims, AclClaimName) ->
     Acl =
         case Claims of
-            #{<<"exp">> := Expire, AclClaimName := Rules} ->
+            #{AclClaimName := Rules} ->
                 #{
                     acl => #{
                         rules => Rules,
-                        expire => Expire
+                        expire => maps:get(<<"exp">>, Claims, undefined)
                     }
                 };
             _ ->
diff --git a/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl b/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl
index 4c36b0fa6..16600a0ac 100644
--- a/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl
@@ -305,6 +305,50 @@ t_check_expire(_Config) ->
 
     ok = emqtt:disconnect(C).
 
+t_check_no_expire(_Config) ->
+    Payload = #{
+        <<"username">> => <<"username">>,
+        <<"acl">> => #{<<"sub">> => [<<"a/b">>]}
+    },
+
+    JWT = generate_jws(Payload),
+
+    {ok, C} = emqtt:start_link(
+        [
+            {clean_start, true},
+            {proto_ver, v5},
+            {clientid, <<"clientid">>},
+            {username, <<"username">>},
+            {password, JWT}
+        ]
+    ),
+    {ok, _} = emqtt:connect(C),
+    ?assertMatch(
+        {ok, #{}, [0]},
+        emqtt:subscribe(C, <<"a/b">>, 0)
+    ),
+
+    ?assertMatch(
+        {ok, #{}, [0]},
+        emqtt:unsubscribe(C, <<"a/b">>)
+    ),
+
+    ok = emqtt:disconnect(C).
+
+t_check_undefined_expire(_Config) ->
+    Acl = #{expire => undefined, rules => #{<<"sub">> => [<<"a/b">>]}},
+    Client = #{acl => Acl},
+
+    ?assertMatch(
+        {matched, allow},
+        emqx_authz_client_info:authorize(Client, subscribe, <<"a/b">>, undefined)
+    ),
+
+    ?assertMatch(
+        {matched, deny},
+        emqx_authz_client_info:authorize(Client, subscribe, <<"a/bar">>, undefined)
+    ).
+
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------

From 0b9f4e70cfc106f6f40729588094878499b11fae Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Tue, 15 Nov 2022 16:10:44 +0800
Subject: [PATCH 12/69] chore: update changes

---
 changes/v5.0.11-en.md | 2 ++
 changes/v5.0.11-zh.md | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index f35d11719..e90415f69 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -10,3 +10,5 @@
 ## Bug fixes
 
 - Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
+
+- Fix that JWT ACL rules are only applied if an `exp` claim is set [#9368](https://github.com/emqx/emqx/pull/9368).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index c16c3193a..1cfcc722e 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -10,3 +10,5 @@
 ## 修复
 
 - 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。
+
+- 修复 JWT ACL 规则只在设置了超期时间时才生效的问题 [#9368](https://github.com/emqx/emqx/pull/9368)。

From 25c32dd2f5c808ff95dea6ed38845450a8639ffe Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.ru>
Date: Tue, 15 Nov 2022 20:57:12 +0300
Subject: [PATCH 13/69] chore(acl): fix acl test

---
 apps/emqx/src/emqx.app.src                   |  2 +-
 apps/emqx/src/emqx_access_control.erl        |  2 +-
 apps/emqx/test/emqx_access_control_SUITE.erl | 30 +++++++++-----------
 3 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src
index 5d2d8eb2c..3d1fe32d3 100644
--- a/apps/emqx/src/emqx.app.src
+++ b/apps/emqx/src/emqx.app.src
@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.10"},
+    {vsn, "5.0.11"},
     {modules, []},
     {registered, []},
     {applications, [
diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl
index 66d45b29a..d99699a9a 100644
--- a/apps/emqx/src/emqx_access_control.erl
+++ b/apps/emqx/src/emqx_access_control.erl
@@ -56,7 +56,7 @@ authorize(ClientInfo, PubSub, <<"$delayed/", Data/binary>> = RawTopic) ->
             authorize(ClientInfo, PubSub, Topic);
         _ ->
             ?SLOG(warning, #{
-                msg => "invalid_dealyed_topic_format",
+                msg => "invalid_delayed_topic_format",
                 expected_example => "$delayed/1/t/foo",
                 got => RawTopic
             }),
diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl
index ee594ec0a..7b6b4f463 100644
--- a/apps/emqx/test/emqx_access_control_SUITE.erl
+++ b/apps/emqx/test/emqx_access_control_SUITE.erl
@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("emqx/include/emqx_hooks.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
 all() -> emqx_common_test_helpers:all(?MODULE).
@@ -32,12 +33,12 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     emqx_common_test_helpers:stop_apps([]).
 
-end_per_testcase(t_delayed_authorize, Config) ->
-    meck:unload(emqx_access_control),
-    Config;
-end_per_testcase(_, Config) ->
+init_per_testcase(_, Config) ->
     Config.
 
+end_per_testcase(_, _Config) ->
+    ok = emqx_hooks:del('client.authorize', {?MODULE, authz_stub}).
+
 t_authenticate(_) ->
     ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())).
 
@@ -46,31 +47,26 @@ t_authorize(_) ->
     ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish, <<"t">>)).
 
 t_delayed_authorize(_) ->
-    RawTopic = "$dealyed/1/foo/2",
-    InvalidTopic = "$dealyed/1/foo/3",
-    Topic = "foo/2",
+    RawTopic = <<"$delayed/1/foo/2">>,
+    InvalidTopic = <<"$delayed/1/foo/3">>,
+    Topic = <<"foo/2">>,
 
-    ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
-    ok = meck:expect(
-        emqx_access_control,
-        do_authorize,
-        fun
-            (_, _, Topic) -> allow;
-            (_, _, _) -> deny
-        end
-    ),
+    ok = emqx_hooks:put('client.authorize', {?MODULE, authz_stub, [Topic]}, ?HP_AUTHZ),
 
     Publish1 = ?PUBLISH_PACKET(?QOS_0, RawTopic, 1, <<"payload">>),
     ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish1, RawTopic)),
 
     Publish2 = ?PUBLISH_PACKET(?QOS_0, InvalidTopic, 1, <<"payload">>),
-    ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)),
+    ?assertEqual(deny, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)),
     ok.
 
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------
 
+authz_stub(_Client, _PubSub, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}};
+authz_stub(_Client, _PubSub, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}.
+
 clientinfo() -> clientinfo(#{}).
 clientinfo(InitProps) ->
     maps:merge(

From 085074ac41314ad7ae38710e7218d8b7bffb9bdd Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Tue, 15 Nov 2022 14:08:58 +0800
Subject: [PATCH 14/69] feat(banned): kick session when it is banned by
 clientid

---
 apps/emqx/src/emqx_banned.erl                 | 23 ++++++++++++++--
 apps/emqx/test/emqx_banned_SUITE.erl          | 26 +++++++++++++++++--
 .../test/emqx_retainer_SUITE.erl              | 17 ++++++------
 3 files changed, 53 insertions(+), 13 deletions(-)

diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl
index cf81c735b..0639557f3 100644
--- a/apps/emqx/src/emqx_banned.erl
+++ b/apps/emqx/src/emqx_banned.erl
@@ -21,6 +21,7 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 -include("types.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).
@@ -180,7 +181,7 @@ create(#{
 create(Banned = #banned{who = Who}) ->
     case look_up(Who) of
         [] ->
-            mria:dirty_write(?BANNED_TAB, Banned),
+            insert_banned(Banned),
             {ok, Banned};
         [OldBanned = #banned{until = Until}] ->
             %% Don't support shorten or extend the until time by overwrite.
@@ -190,7 +191,7 @@ create(Banned = #banned{who = Who}) ->
                     {error, {already_exist, OldBanned}};
                 %% overwrite expired one is ok.
                 false ->
-                    mria:dirty_write(?BANNED_TAB, Banned),
+                    insert_banned(Banned),
                     {ok, Banned}
             end
     end.
@@ -266,3 +267,21 @@ expire_banned_items(Now) ->
         ok,
         ?BANNED_TAB
     ).
+
+insert_banned(Banned) ->
+    mria:dirty_write(?BANNED_TAB, Banned),
+    on_banned(Banned).
+
+on_banned(#banned{who = {clientid, ClientId}}) ->
+    %% kick the session if the client is banned by clientid
+    ?tp(
+        warning,
+        kick_session_due_to_banned,
+        #{
+            clientid => ClientId
+        }
+    ),
+    emqx_cm:kick_session(ClientId),
+    ok;
+on_banned(_) ->
+    ok.
diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl
index d8827721f..afbf9a579 100644
--- a/apps/emqx/test/emqx_banned_SUITE.erl
+++ b/apps/emqx/test/emqx_banned_SUITE.erl
@@ -21,18 +21,20 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    application:load(emqx),
+    emqx_common_test_helpers:start_apps([]),
     ok = ekka:start(),
     Config.
 
 end_per_suite(_Config) ->
     ekka:stop(),
     mria:stop(),
-    mria_mnesia:delete_schema().
+    mria_mnesia:delete_schema(),
+    emqx_common_test_helpers:stop_apps([]).
 
 t_add_delete(_) ->
     Banned = #banned{
@@ -111,3 +113,23 @@ t_unused(_) ->
     %% expiry timer
     timer:sleep(500),
     ok = emqx_banned:stop().
+
+t_kick(_) ->
+    ClientId = <<"client">>,
+    snabbkaffe:start_trace(),
+
+    Now = erlang:system_time(second),
+    Who = {clientid, ClientId},
+
+    emqx_banned:create(#{
+        who => Who,
+        by => <<"test">>,
+        reason => <<"test">>,
+        at => Now,
+        until => Now + 120
+    }),
+
+    Trace = snabbkaffe:collect_trace(),
+    snabbkaffe:stop(),
+    emqx_banned:delete(Who),
+    ?assertEqual(1, length(?of_kind(kick_session_due_to_banned, Trace))).
diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl
index 86eaa4255..f3e46aed9 100644
--- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl
+++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl
@@ -640,26 +640,25 @@ test_disable_then_start(_Config) ->
     ok.
 
 t_deliver_when_banned(_) ->
-    ClientId = <<"c1">>,
+    Client1 = <<"c1">>,
+    Client2 = <<"c2">>,
 
-    {ok, C1} = emqtt:start_link([{clientid, ClientId}, {clean_start, true}, {proto_ver, v5}]),
+    {ok, C1} = emqtt:start_link([{clientid, Client1}, {clean_start, true}, {proto_ver, v5}]),
     {ok, _} = emqtt:connect(C1),
 
     lists:foreach(
         fun(I) ->
             Topic = erlang:list_to_binary(io_lib:format("retained/~p", [I])),
-            emqtt:publish(
-                C1,
-                Topic,
-                <<"this is a retained message">>,
-                [{qos, 0}, {retain, true}]
-            )
+            Msg = emqx_message:make(Client2, 0, Topic, <<"this is a retained message">>),
+            Msg2 = emqx_message:set_flag(retain, Msg),
+            emqx:publish(Msg2)
         end,
         lists:seq(1, 3)
     ),
 
     Now = erlang:system_time(second),
-    Who = {clientid, ClientId},
+    Who = {clientid, Client2},
+
     emqx_banned:create(#{
         who => Who,
         by => <<"test">>,

From b1889fa203685a7a9999cc4f9bc7c843a74e7765 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Tue, 15 Nov 2022 14:48:35 +0800
Subject: [PATCH 15/69] chore: bump emqx version && update changes

---
 apps/emqx/src/emqx.app.src | 2 +-
 changes/v5.0.11-en.md      | 3 +++
 changes/v5.0.11-zh.md      | 3 +++
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src
index 5d2d8eb2c..3d1fe32d3 100644
--- a/apps/emqx/src/emqx.app.src
+++ b/apps/emqx/src/emqx.app.src
@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.10"},
+    {vsn, "5.0.11"},
     {modules, []},
     {registered, []},
     {applications, [
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index f35d11719..652879cc8 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -7,6 +7,9 @@
 
 - Security enhancement for the `subscribe` API [#9355](https://github.com/emqx/emqx/pull/9355).
 
+- Enhance the `banned` feature [#9367](https://github.com/emqx/emqx/pull/9367).
+  Now the corresponding session will be kicked when client is banned by `clientid`.
+
 ## Bug fixes
 
 - Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index c16c3193a..9ec1e0414 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -7,6 +7,9 @@
 
 - 增强订阅 API 的安全性 [#9355](https://github.com/emqx/emqx/pull/9355)。
 
+- 增加 `封禁` 功能 [#9367](https://github.com/emqx/emqx/pull/9367)。
+  现在客户端通过 `clientid` 被封禁时将会踢掉对应的会话。
+
 ## 修复
 
 - 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。

From d54597e0d454edb3fd5ad11e12a490a7b0fe8951 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Wed, 16 Nov 2022 11:55:02 +0100
Subject: [PATCH 16/69] ci: fix version for push-helm-action

---
 .github/workflows/release.yaml | 57 ++++++++++++----------------------
 1 file changed, 20 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 05dcb62e3..7c5bf94e4 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -15,22 +15,34 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
-      - name: Get packages
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+      - name: Detect profile
+        id: profile
         run: |
           REF=${{ github.ref_name }}
           case "$REF" in
             v*)
-              s3dir='emqx-ce'
+              echo "profile=emqx" >> $GITHUB_OUTPUT
+              echo "s3dir=emqx-ce" >> $GITHUB_OUTPUT
+              echo "version=$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT
               ;;
             e*)
-              s3dir='emqx-ee'
+              echo "profile=emqx-enterprise" >> $GITHUB_OUTPUT
+              echo "s3dir=emqx-ee" >> $GITHUB_OUTPUT
+              echo "version=$(./pkg-vsn.sh emqx-enterprise)" >> $GITHUB_OUTPUT
               ;;
             *)
               echo "tag $REF is not supported"
               exit 1
               ;;
           esac
-          aws s3 cp --recursive s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} packages
+      - name: Get packages
+        run: |
+          BUCKET=${{ secrets.AWS_S3_BUCKET }}
+          OUTPUT_DIR=${{ steps.profile.outputs.s3dir }}
+          aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages
           cd packages
           DEFAULT_BEAM_PLATFORM='otp24.3.4.2-1'
           # all packages including full-name and default-name are uploaded to s3
@@ -58,18 +70,8 @@ jobs:
                -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
                ${{ secrets.EMQX_IO_RELEASE_API }}
       - name: update homebrew packages
-        if: github.event_name == 'release'
+        if: github.event_name == 'release' && startsWith(github.ref_name, 'v')
         run: |
-          REF=${{ github.ref_name }}
-          case "$REF" in
-            v*)
-              BOOL_FLAG_NAME="emqx_ce"
-              ;;
-            e*)
-              echo "Not updating homebrew for enterprise eidition"
-              exit 0
-              ;;
-          esac
           if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
               curl --silent --show-error \
                 -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
@@ -78,30 +80,11 @@ jobs:
                 -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \
                 "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
           fi
-
-  upload-helm:
-    runs-on: ubuntu-20.04
-    if: github.event_name == 'release'
-    strategy:
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.ref }}
       - uses: emqx/push-helm-action@v1
-        if: startsWith(github.ref_name, 'v')
+        if: github.event_name == 'release'
         with:
-          charts_dir: "${{ github.workspace }}/deploy/charts/emqx"
-          version: ${{ github.ref_name }}
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws_region: "us-west-2"
-          aws_bucket_name: "repos-emqx-io"
-      - uses: emqx/push-helm-action@v1
-        if: startsWith(github.ref_name, 'e')
-        with:
-          charts_dir: "${{ github.workspace }}/deploy/charts/emqx-enterprise"
-          version: ${{ github.ref_name }}
+          charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}"
+          version: ${{ steps.profile.outputs.version }}
           aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_region: "us-west-2"

From 70c52f43cc82edb56a46cb8013fd6119c9b77b16 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Wed, 16 Nov 2022 14:28:47 +0100
Subject: [PATCH 17/69] ci: enable manual trigger on release workflow

---
 .github/workflows/release.yaml | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 7c5bf94e4..3a4f32499 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -3,6 +3,15 @@ on:
   release:
     types:
       - published
+  workflow_dispatch:
+    inputs:
+      tag:
+        type: string
+        required: true
+      publish_release_artefacts:
+        type: boolean
+        required: true
+        default: false
 
 jobs:
   upload:
@@ -17,25 +26,26 @@ jobs:
           aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.ref }}
+          ref: ${{ github.event.inputs.tag }}
       - name: Detect profile
         id: profile
         run: |
-          REF=${{ github.ref_name }}
+          if git describe --tags --match '[v|e]*' --exact; then
+            REF=$(git describe --tags --match '[v|e]*' --exact)
+          else
+            echo "Only release tags matching '[v|e]*' are supported"
+            exit 1
+          fi
           case "$REF" in
             v*)
               echo "profile=emqx" >> $GITHUB_OUTPUT
-              echo "s3dir=emqx-ce" >> $GITHUB_OUTPUT
               echo "version=$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT
+              echo "s3dir=emqx-ce" >> $GITHUB_OUTPUT
               ;;
             e*)
               echo "profile=emqx-enterprise" >> $GITHUB_OUTPUT
-              echo "s3dir=emqx-ee" >> $GITHUB_OUTPUT
               echo "version=$(./pkg-vsn.sh emqx-enterprise)" >> $GITHUB_OUTPUT
-              ;;
-            *)
-              echo "tag $REF is not supported"
-              exit 1
+              echo "s3dir=emqx-ee" >> $GITHUB_OUTPUT
               ;;
           esac
       - name: Get packages
@@ -59,7 +69,7 @@ jobs:
         with:
           asset_paths: '["packages/*"]'
       - name: update to emqx.io
-        if: github.event_name == 'release'
+        if: github.event_name == 'release' || inputs.publish_release_artefacts
         run: |
           set -e -x -u
           curl -w %{http_code} \
@@ -70,7 +80,7 @@ jobs:
                -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
                ${{ secrets.EMQX_IO_RELEASE_API }}
       - name: update homebrew packages
-        if: github.event_name == 'release' && startsWith(github.ref_name, 'v')
+        if: steps.profile.outputs.profile == 'emqx' && (github.event_name == 'release' || inputs.publish_release_artefacts)
         run: |
           if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
               curl --silent --show-error \
@@ -81,7 +91,7 @@ jobs:
                 "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
           fi
       - uses: emqx/push-helm-action@v1
-        if: github.event_name == 'release'
+        if: github.event_name == 'release' || inputs.publish_release_artefacts
         with:
           charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}"
           version: ${{ steps.profile.outputs.version }}

From 07347e3f9e7af92f20b2c7ae400f733ad357c21f Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Fri, 11 Nov 2022 13:29:50 +0100
Subject: [PATCH 18/69] style: fix typo in comment

---
 apps/emqx_gateway/src/emqx_gateway_api.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl
index dc34c03a8..6a9744d75 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api.erl
@@ -34,7 +34,7 @@
     ]
 ).
 
-%% minirest/dashbaord_swagger behaviour callbacks
+%% minirest/dashboard_swagger behaviour callbacks
 -export([
     api_spec/0,
     paths/0,

From 2914523117fe8ef0e29cd0fce4cdeaa461649711 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Fri, 11 Nov 2022 13:30:06 +0100
Subject: [PATCH 19/69] style: remove unused import

---
 apps/emqx_gateway/src/emqx_gateway_api_authn.erl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
index f337563ee..71d0e393e 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
@@ -30,8 +30,7 @@
     [
         return_http_error/2,
         with_gateway/2,
-        with_authn/2,
-        checks/2
+        with_authn/2
     ]
 ).
 

From 9cc5920b0d853b20e3b2e3c8bd1c59c35879a947 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Fri, 11 Nov 2022 15:16:13 +0100
Subject: [PATCH 20/69] refactor: only PUT, no POST for '/gateways'

---
 .../i18n/emqx_gateway_api_i18n.conf           |   8 +
 apps/emqx_gateway/src/emqx_gateway.app.src    |   2 +-
 apps/emqx_gateway/src/emqx_gateway_api.erl    | 247 ++++++++++--------
 .../test/emqx_gateway_api_SUITE.erl           | 191 ++++++++------
 .../test/emqx_gateway_test_utils.erl          |   2 +-
 apps/emqx_gateway/test/emqx_stomp_SUITE.erl   |   6 +-
 changes/v5.0.11-en.md                         |   5 +
 changes/v5.0.11-zh.md                         |   3 +
 8 files changed, 270 insertions(+), 194 deletions(-)

diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf
index 34e9b8567..197e6a5ed 100644
--- a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf
+++ b/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf
@@ -57,6 +57,14 @@ It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto`
         }
     }
 
+    gateway_enable_in_path {
+        desc {
+            en: """Whether or not gateway is enabled"""
+
+            zh: """是否开启此网关"""
+        }
+    }
+
     gateway_status {
         desc {
             en: """Gateway status"""
diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src
index 491d0242a..24cb76630 100644
--- a/apps/emqx_gateway/src/emqx_gateway.app.src
+++ b/apps/emqx_gateway/src/emqx_gateway.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway, [
     {description, "The Gateway management application"},
-    {vsn, "0.1.7"},
+    {vsn, "0.1.8"},
     {registered, []},
     {mod, {emqx_gateway_app, []}},
     {applications, [kernel, stdlib, grpc, emqx, emqx_authn]},
diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl
index 6a9744d75..e06748034 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api.erl
@@ -19,8 +19,6 @@
 -include("emqx_gateway_http.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include_lib("emqx/include/emqx_placeholder.hrl").
--include_lib("emqx/include/emqx_authentication.hrl").
 
 -behaviour(minirest_api).
 
@@ -49,8 +47,9 @@
 
 %% http handlers
 -export([
+    gateways/2,
     gateway/2,
-    gateway_insta/2
+    gateway_enable/2
 ]).
 
 -define(KNOWN_GATEWAY_STATUSES, [<<"running">>, <<"stopped">>, <<"unloaded">>]).
@@ -66,13 +65,14 @@ api_spec() ->
 paths() ->
     emqx_gateway_utils:make_deprecated_paths([
         "/gateways",
-        "/gateways/:name"
+        "/gateways/:name",
+        "/gateways/:name/enable/:enable"
     ]).
 
 %%--------------------------------------------------------------------
 %% http handlers
 
-gateway(get, Request) ->
+gateways(get, Request) ->
     Params = maps:get(query_string, Request, #{}),
     Status = maps:get(<<"status">>, Params, <<"all">>),
     case lists:member(Status, [<<"all">> | ?KNOWN_GATEWAY_STATUSES]) of
@@ -89,84 +89,85 @@ gateway(get, Request) ->
                     lists:join(", ", ?KNOWN_GATEWAY_STATUSES)
                 ]
             )
-    end;
-gateway(post, Request) ->
-    Body = maps:get(body, Request, #{}),
-    try
-        Name0 = maps:get(<<"name">>, Body),
-        GwName = binary_to_existing_atom(Name0),
-        case emqx_gateway_registry:lookup(GwName) of
-            undefined ->
-                error(badarg);
-            _ ->
-                GwConf = maps:without([<<"name">>], Body),
-                case emqx_gateway_conf:load_gateway(GwName, GwConf) of
-                    {ok, NGwConf} ->
-                        {201, NGwConf};
-                    {error, Reason} ->
-                        emqx_gateway_http:reason2resp(Reason)
-                end
-        end
-    catch
-        error:{badkey, K} ->
-            return_http_error(400, [K, " is required"]);
-        error:{badconf, _} = Reason1 ->
-            emqx_gateway_http:reason2resp(Reason1);
-        error:badarg ->
-            return_http_error(404, "Bad gateway name")
     end.
 
-gateway_insta(delete, #{bindings := #{name := Name0}}) ->
-    with_gateway(Name0, fun(GwName, _) ->
-        case emqx_gateway_conf:unload_gateway(GwName) of
-            ok ->
+gateway(get, #{bindings := #{name := Name}}) ->
+    try
+        GwName = gw_name(Name),
+        case emqx_gateway:lookup(GwName) of
+            undefined ->
+                {200, #{name => GwName, status => unloaded}};
+            Gateway ->
+                GwConf = emqx_gateway_conf:gateway(Name),
+                GwInfo0 = emqx_gateway_utils:unix_ts_to_rfc3339(
+                    [created_at, started_at, stopped_at],
+                    Gateway
+                ),
+                GwInfo1 = maps:with(
+                    [
+                        name,
+                        status,
+                        created_at,
+                        started_at,
+                        stopped_at
+                    ],
+                    GwInfo0
+                ),
+                {200, maps:merge(GwConf, GwInfo1)}
+        end
+    catch
+        throw:not_found ->
+            return_http_error(404, <<"NOT FOUND">>)
+    end;
+gateway(put, #{
+    body := GwConf0,
+    bindings := #{name := Name}
+}) ->
+    GwConf = maps:without([<<"name">>], GwConf0),
+    try
+        GwName = gw_name(Name),
+        LoadOrUpdateF =
+            case emqx_gateway:lookup(GwName) of
+                undefined ->
+                    fun emqx_gateway_conf:load_gateway/2;
+                _ ->
+                    fun emqx_gateway_conf:update_gateway/2
+            end,
+        case LoadOrUpdateF(GwName, GwConf) of
+            {ok, _} ->
                 {204};
             {error, Reason} ->
                 emqx_gateway_http:reason2resp(Reason)
         end
-    end);
-gateway_insta(get, #{bindings := #{name := Name0}}) ->
-    try binary_to_existing_atom(Name0) of
-        GwName ->
-            case emqx_gateway:lookup(GwName) of
-                undefined ->
-                    {200, #{name => GwName, status => unloaded}};
-                Gateway ->
-                    GwConf = emqx_gateway_conf:gateway(Name0),
-                    GwInfo0 = emqx_gateway_utils:unix_ts_to_rfc3339(
-                        [created_at, started_at, stopped_at],
-                        Gateway
-                    ),
-                    GwInfo1 = maps:with(
-                        [
-                            name,
-                            status,
-                            created_at,
-                            started_at,
-                            stopped_at
-                        ],
-                        GwInfo0
-                    ),
-                    {200, maps:merge(GwConf, GwInfo1)}
-            end
     catch
-        error:badarg ->
-            return_http_error(404, "Bad gateway name")
-    end;
-gateway_insta(put, #{
-    body := GwConf0,
-    bindings := #{name := Name0}
-}) ->
-    with_gateway(Name0, fun(GwName, _) ->
-        %% XXX: Clear the unused fields
-        GwConf = maps:without([<<"name">>], GwConf0),
-        case emqx_gateway_conf:update_gateway(GwName, GwConf) of
-            {ok, Gateway} ->
-                {200, Gateway};
-            {error, Reason} ->
-                emqx_gateway_http:reason2resp(Reason)
+        error:{badconf, _} = Reason1 ->
+            emqx_gateway_http:reason2resp(Reason1);
+        throw:not_found ->
+            return_http_error(404, <<"NOT FOUND">>)
+    end.
+
+gateway_enable(put, #{bindings := #{name := Name, enable := Enable}}) ->
+    try
+        GwName = gw_name(Name),
+        case emqx_gateway:lookup(GwName) of
+            undefined ->
+                return_http_error(404, <<"NOT FOUND">>);
+            _Gateway ->
+                {ok, _} = emqx_gateway_conf:update_gateway(GwName, #{<<"enable">> => Enable}),
+                {204}
         end
-    end).
+    catch
+        throw:not_found ->
+            return_http_error(404, <<"NOT FOUND">>)
+    end.
+
+-spec gw_name(binary()) -> stomp | coap | lwm2m | mqttsn | exproto | no_return().
+gw_name(<<"stomp">>) -> stomp;
+gw_name(<<"coap">>) -> coap;
+gw_name(<<"lwm2m">>) -> lwm2m;
+gw_name(<<"mqttsn">>) -> mqttsn;
+gw_name(<<"exproto">>) -> exproto;
+gw_name(_Else) -> throw(not_found).
 
 %%--------------------------------------------------------------------
 %% Swagger defines
@@ -174,7 +175,7 @@ gateway_insta(put, #{
 
 schema("/gateways") ->
     #{
-        'operationId' => gateway,
+        'operationId' => gateways,
         get =>
             #{
                 tags => ?TAGS,
@@ -182,29 +183,20 @@ schema("/gateways") ->
                 summary => <<"List All Gateways">>,
                 parameters => params_gateway_status_in_qs(),
                 responses =>
-                    ?STANDARD_RESP(
-                        #{
-                            200 => emqx_dashboard_swagger:schema_with_example(
-                                hoconsc:array(ref(gateway_overview)),
-                                examples_gateway_overview()
-                            )
-                        }
-                    )
-            },
-        post =>
-            #{
-                tags => ?TAGS,
-                desc => ?DESC(enable_gateway),
-                summary => <<"Enable a Gateway">>,
-                %% TODO: distinguish create & response swagger schema
-                'requestBody' => schema_gateways_conf(),
-                responses =>
-                    ?STANDARD_RESP(#{201 => schema_gateways_conf()})
+                    #{
+                        200 => emqx_dashboard_swagger:schema_with_example(
+                            hoconsc:array(ref(gateway_overview)),
+                            examples_gateway_overview()
+                        ),
+                        400 => emqx_dashboard_swagger:error_codes(
+                            [?BAD_REQUEST], <<"Bad request">>
+                        )
+                    }
             }
     };
 schema("/gateways/:name") ->
     #{
-        'operationId' => gateway_insta,
+        'operationId' => gateway,
         get =>
             #{
                 tags => ?TAGS,
@@ -212,26 +204,41 @@ schema("/gateways/:name") ->
                 summary => <<"Get the Gateway">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
-                    ?STANDARD_RESP(#{200 => schema_gateways_conf()})
-            },
-        delete =>
-            #{
-                tags => ?TAGS,
-                desc => ?DESC(delete_gateway),
-                summary => <<"Unload the gateway">>,
-                parameters => params_gateway_name_in_path(),
-                responses =>
-                    ?STANDARD_RESP(#{204 => <<"Deleted">>})
+                    #{
+                        200 => schema_gateways_conf(),
+                        404 => emqx_dashboard_swagger:error_codes(
+                            [?NOT_FOUND, ?RESOURCE_NOT_FOUND], <<"Not Found">>
+                        )
+                    }
             },
         put =>
             #{
                 tags => ?TAGS,
                 desc => ?DESC(update_gateway),
-                summary => <<"Update the gateway confs">>,
+                % [FIXME] add proper desc
+                summary => <<"Load or update the gateway confs">>,
                 parameters => params_gateway_name_in_path(),
-                'requestBody' => schema_update_gateways_conf(),
+                'requestBody' => schema_load_or_update_gateways_conf(),
                 responses =>
-                    ?STANDARD_RESP(#{200 => schema_gateways_conf()})
+                    ?STANDARD_RESP(#{204 => <<"Gateway configuration updated">>})
+            }
+    };
+schema("/gateways/:name/enable/:enable") ->
+    #{
+        'operationId' => gateway_enable,
+        put =>
+            #{
+                tags => ?TAGS,
+                desc => ?DESC(update_gateway),
+                summary => <<"Enable or disable gateway">>,
+                parameters => params_gateway_name_in_path() ++ params_gateway_enable_in_path(),
+                responses =>
+                    #{
+                        204 => <<"Gateway configuration updated">>,
+                        404 => emqx_dashboard_swagger:error_codes(
+                            [?NOT_FOUND, ?RESOURCE_NOT_FOUND], <<"Not Found">>
+                        )
+                    }
             }
     };
 schema(Path) ->
@@ -268,6 +275,18 @@ params_gateway_status_in_qs() ->
             )}
     ].
 
+params_gateway_enable_in_path() ->
+    [
+        {enable,
+            mk(
+                boolean(),
+                #{
+                    in => path,
+                    desc => ?DESC(gateway_enable_in_path),
+                    example => true
+                }
+            )}
+    ].
 %%--------------------------------------------------------------------
 %% schemas
 
@@ -377,8 +396,6 @@ fields(Gw) when
 ->
     [{name, mk(Gw, #{desc => ?DESC(gateway_name)})}] ++
         convert_listener_struct(emqx_gateway_schema:fields(Gw));
-fields(update_disable_enable_only) ->
-    [{enable, mk(boolean(), #{desc => <<"Enable/Disable the gateway">>})}];
 fields(Gw) when
     Gw == update_stomp;
     Gw == update_mqttsn;
@@ -431,15 +448,19 @@ fields(Listener) when
 fields(gateway_stats) ->
     [{key, mk(binary(), #{})}].
 
-schema_update_gateways_conf() ->
+schema_load_or_update_gateways_conf() ->
     emqx_dashboard_swagger:schema_with_examples(
         hoconsc:union([
+            ref(?MODULE, stomp),
+            ref(?MODULE, mqttsn),
+            ref(?MODULE, coap),
+            ref(?MODULE, lwm2m),
+            ref(?MODULE, exproto),
             ref(?MODULE, update_stomp),
             ref(?MODULE, update_mqttsn),
             ref(?MODULE, update_coap),
             ref(?MODULE, update_lwm2m),
-            ref(?MODULE, update_exproto),
-            ref(?MODULE, update_disable_enable_only)
+            ref(?MODULE, update_exproto)
         ]),
         examples_update_gateway_confs()
     ).
diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
index c4a6758a3..7d58d14b6 100644
--- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
@@ -23,7 +23,7 @@
     emqx_gateway_test_utils,
     [
         assert_confs/2,
-        assert_feilds_apperence/2,
+        assert_fields_exist/2,
         request/2,
         request/3,
         ssl_server_opts/0,
@@ -32,6 +32,7 @@
 ).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 %% this parses to #{}, will not cause config cleanup
 %% so we will need call emqx_config:erase
@@ -55,32 +56,68 @@ end_per_suite(Conf) ->
     emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]),
     Conf.
 
+init_per_testcase(t_gateway_fail, Config) ->
+    meck:expect(
+        emqx_gateway_conf,
+        update_gateway,
+        fun
+            (stomp, V) -> {error, {badconf, #{key => gw, value => V, reason => test_error}}};
+            (coap, V) -> error({badconf, #{key => gw, value => V, reason => test_crash}})
+        end
+    ),
+    Config;
+init_per_testcase(_, Config) ->
+    Config.
+
+end_per_testcase(TestCase, Config) ->
+    case TestCase of
+        t_gateway_fail -> meck:unload(emqx_gateway_conf);
+        _ -> ok
+    end,
+    [emqx_gateway_conf:unload_gateway(GwName) || GwName <- [stomp, mqttsn, coap, lwm2m, exproto]],
+    Config.
+
 %%--------------------------------------------------------------------
 %% Cases
 %%--------------------------------------------------------------------
 
-t_gateway(_) ->
+t_gateways(_) ->
     {200, Gateways} = request(get, "/gateways"),
     lists:foreach(fun assert_gw_unloaded/1, Gateways),
     {200, UnloadedGateways} = request(get, "/gateways?status=unloaded"),
     lists:foreach(fun assert_gw_unloaded/1, UnloadedGateways),
     {200, NoRunningGateways} = request(get, "/gateways?status=running"),
     ?assertEqual([], NoRunningGateways),
-    {404, GwNotFoundReq} = request(get, "/gateways/unknown_gateway"),
-    assert_not_found(GwNotFoundReq),
     {400, BadReqInvalidStatus} = request(get, "/gateways?status=invalid_status"),
     assert_bad_request(BadReqInvalidStatus),
     {400, BadReqUCStatus} = request(get, "/gateways?status=UNLOADED"),
     assert_bad_request(BadReqUCStatus),
-    {201, _} = request(post, "/gateways", #{name => <<"stomp">>}),
-    {200, StompGw1} = request(get, "/gateways/stomp"),
-    assert_feilds_apperence(
+    ok.
+
+t_gateway(_) ->
+    {404, GwNotFoundReq1} = request(get, "/gateways/not_a_known_atom"),
+    assert_not_found(GwNotFoundReq1),
+    {404, GwNotFoundReq2} = request(get, "/gateways/undefined"),
+    assert_not_found(GwNotFoundReq2),
+    {204, _} = request(put, "/gateways/stomp", #{}),
+    {200, StompGw} = request(get, "/gateways/stomp"),
+    assert_fields_exist(
         [name, status, enable, created_at, started_at],
-        StompGw1
+        StompGw
     ),
-    {204, _} = request(delete, "/gateways/stomp"),
-    {200, StompGw2} = request(get, "/gateways/stomp"),
-    assert_gw_unloaded(StompGw2),
+    {204, _} = request(put, "/gateways/stomp", #{enable => true}),
+    {200, #{enable := true}} = request(get, "/gateway/stomp"),
+    {204, _} = request(put, "/gateways/stomp", #{enable => false}),
+    {200, #{enable := false}} = request(get, "/gateway/stomp"),
+    {404, _} = request(put, "/gateways/undefined", #{}),
+    {400, _} = request(put, "/gateways/stomp", #{bad_key => "foo"}),
+    ok.
+
+t_gateway_fail(_) ->
+    {204, _} = request(put, "/gateways/stomp", #{}),
+    {400, _} = request(put, "/gateways/stomp", #{}),
+    {204, _} = request(put, "/gateways/coap", #{}),
+    {400, _} = request(put, "/gateways/coap", #{}),
     ok.
 
 t_deprecated_gateway(_) ->
@@ -88,21 +125,30 @@ t_deprecated_gateway(_) ->
     lists:foreach(fun assert_gw_unloaded/1, Gateways),
     {404, NotFoundReq} = request(get, "/gateway/uname_gateway"),
     assert_not_found(NotFoundReq),
-    {201, _} = request(post, "/gateway", #{name => <<"stomp">>}),
-    {200, StompGw1} = request(get, "/gateway/stomp"),
-    assert_feilds_apperence(
+    {204, _} = request(put, "/gateway/stomp", #{}),
+    {200, StompGw} = request(get, "/gateway/stomp"),
+    assert_fields_exist(
         [name, status, enable, created_at, started_at],
-        StompGw1
+        StompGw
     ),
-    {204, _} = request(delete, "/gateway/stomp"),
-    {200, StompGw2} = request(get, "/gateway/stomp"),
-    assert_gw_unloaded(StompGw2),
+    ok.
+
+t_gateway_enable(_) ->
+    {204, _} = request(put, "/gateways/stomp", #{}),
+    {200, #{enable := Enable}} = request(get, "/gateway/stomp"),
+    NotEnable = not Enable,
+    {204, _} = request(put, "/gateways/stomp/enable/" ++ atom_to_list(NotEnable), undefined),
+    {200, #{enable := NotEnable}} = request(get, "/gateway/stomp"),
+    {204, _} = request(put, "/gateways/stomp/enable/" ++ atom_to_list(Enable), undefined),
+    {200, #{enable := Enable}} = request(get, "/gateway/stomp"),
+    {404, _} = request(put, "/gateways/undefined/enable/true", undefined),
+    {404, _} = request(put, "/gateways/not_a_known_atom/enable/true", undefined),
+    {404, _} = request(put, "/gateways/coap/enable/true", undefined),
     ok.
 
 t_gateway_stomp(_) ->
     {200, Gw} = request(get, "/gateways/stomp"),
     assert_gw_unloaded(Gw),
-    %% post
     GwConf = #{
         name => <<"stomp">>,
         frame => #{
@@ -114,20 +160,18 @@ t_gateway_stomp(_) ->
             #{name => <<"def">>, type => <<"tcp">>, bind => <<"61613">>}
         ]
     },
-    {201, _} = request(post, "/gateways", GwConf),
+    {204, _} = request(put, "/gateways/stomp", GwConf),
     {200, ConfResp} = request(get, "/gateways/stomp"),
     assert_confs(GwConf, ConfResp),
-    %% put
     GwConf2 = emqx_map_lib:deep_merge(GwConf, #{frame => #{max_headers => 10}}),
-    {200, _} = request(put, "/gateways/stomp", maps:without([name, listeners], GwConf2)),
+    {204, _} = request(put, "/gateways/stomp", maps:without([name, listeners], GwConf2)),
     {200, ConfResp2} = request(get, "/gateways/stomp"),
     assert_confs(GwConf2, ConfResp2),
-    {204, _} = request(delete, "/gateways/stomp").
+    ok.
 
 t_gateway_mqttsn(_) ->
     {200, Gw} = request(get, "/gateways/mqttsn"),
     assert_gw_unloaded(Gw),
-    %% post
     GwConf = #{
         name => <<"mqttsn">>,
         gateway_id => 1,
@@ -138,20 +182,18 @@ t_gateway_mqttsn(_) ->
             #{name => <<"def">>, type => <<"udp">>, bind => <<"1884">>}
         ]
     },
-    {201, _} = request(post, "/gateways", GwConf),
+    {204, _} = request(put, "/gateways/mqttsn", GwConf),
     {200, ConfResp} = request(get, "/gateways/mqttsn"),
     assert_confs(GwConf, ConfResp),
-    %% put
     GwConf2 = emqx_map_lib:deep_merge(GwConf, #{predefined => []}),
-    {200, _} = request(put, "/gateways/mqttsn", maps:without([name, listeners], GwConf2)),
+    {204, _} = request(put, "/gateways/mqttsn", maps:without([name, listeners], GwConf2)),
     {200, ConfResp2} = request(get, "/gateways/mqttsn"),
     assert_confs(GwConf2, ConfResp2),
-    {204, _} = request(delete, "/gateways/mqttsn").
+    ok.
 
 t_gateway_coap(_) ->
     {200, Gw} = request(get, "/gateways/coap"),
     assert_gw_unloaded(Gw),
-    %% post
     GwConf = #{
         name => <<"coap">>,
         heartbeat => <<"60s">>,
@@ -160,20 +202,18 @@ t_gateway_coap(_) ->
             #{name => <<"def">>, type => <<"udp">>, bind => <<"5683">>}
         ]
     },
-    {201, _} = request(post, "/gateways", GwConf),
+    {204, _} = request(put, "/gateways/coap", GwConf),
     {200, ConfResp} = request(get, "/gateways/coap"),
     assert_confs(GwConf, ConfResp),
-    %% put
     GwConf2 = emqx_map_lib:deep_merge(GwConf, #{heartbeat => <<"10s">>}),
-    {200, _} = request(put, "/gateways/coap", maps:without([name, listeners], GwConf2)),
+    {204, _} = request(put, "/gateways/coap", maps:without([name, listeners], GwConf2)),
     {200, ConfResp2} = request(get, "/gateways/coap"),
     assert_confs(GwConf2, ConfResp2),
-    {204, _} = request(delete, "/gateways/coap").
+    ok.
 
 t_gateway_lwm2m(_) ->
     {200, Gw} = request(get, "/gateways/lwm2m"),
     assert_gw_unloaded(Gw),
-    %% post
     GwConf = #{
         name => <<"lwm2m">>,
         xml_dir => <<"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml">>,
@@ -192,20 +232,18 @@ t_gateway_lwm2m(_) ->
             #{name => <<"def">>, type => <<"udp">>, bind => <<"5783">>}
         ]
     },
-    {201, _} = request(post, "/gateways", GwConf),
+    {204, _} = request(put, "/gateways/lwm2m", GwConf),
     {200, ConfResp} = request(get, "/gateways/lwm2m"),
     assert_confs(GwConf, ConfResp),
-    %% put
     GwConf2 = emqx_map_lib:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}),
-    {200, _} = request(put, "/gateways/lwm2m", maps:without([name, listeners], GwConf2)),
+    {204, _} = request(put, "/gateways/lwm2m", maps:without([name, listeners], GwConf2)),
     {200, ConfResp2} = request(get, "/gateways/lwm2m"),
     assert_confs(GwConf2, ConfResp2),
-    {204, _} = request(delete, "/gateways/lwm2m").
+    ok.
 
 t_gateway_exproto(_) ->
     {200, Gw} = request(get, "/gateways/exproto"),
     assert_gw_unloaded(Gw),
-    %% post
     GwConf = #{
         name => <<"exproto">>,
         server => #{bind => <<"9100">>},
@@ -214,15 +252,14 @@ t_gateway_exproto(_) ->
             #{name => <<"def">>, type => <<"tcp">>, bind => <<"7993">>}
         ]
     },
-    {201, _} = request(post, "/gateways", GwConf),
+    {204, _} = request(put, "/gateways/exproto", GwConf),
     {200, ConfResp} = request(get, "/gateways/exproto"),
     assert_confs(GwConf, ConfResp),
-    %% put
     GwConf2 = emqx_map_lib:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}),
-    {200, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
+    {204, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
     {200, ConfResp2} = request(get, "/gateways/exproto"),
     assert_confs(GwConf2, ConfResp2),
-    {204, _} = request(delete, "/gateways/exproto").
+    ok.
 
 t_gateway_exproto_with_ssl(_) ->
     {200, Gw} = request(get, "/gateways/exproto"),
@@ -230,7 +267,6 @@ t_gateway_exproto_with_ssl(_) ->
 
     SslSvrOpts = ssl_server_opts(),
     SslCliOpts = ssl_client_opts(),
-    %% post
     GwConf = #{
         name => <<"exproto">>,
         server => #{
@@ -245,27 +281,22 @@ t_gateway_exproto_with_ssl(_) ->
             #{name => <<"def">>, type => <<"tcp">>, bind => <<"7993">>}
         ]
     },
-    {201, _} = request(post, "/gateways", GwConf),
+    {204, _} = request(put, "/gateways/exproto", GwConf),
     {200, ConfResp} = request(get, "/gateways/exproto"),
     assert_confs(GwConf, ConfResp),
-    %% put
     GwConf2 = emqx_map_lib:deep_merge(GwConf, #{
         server => #{
             bind => <<"9200">>,
             ssl_options => SslCliOpts
         }
     }),
-    {200, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
+    {204, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)),
     {200, ConfResp2} = request(get, "/gateways/exproto"),
     assert_confs(GwConf2, ConfResp2),
-    {204, _} = request(delete, "/gateways/exproto").
+    ok.
 
 t_authn(_) ->
-    GwConf = #{name => <<"stomp">>},
-    {201, _} = request(post, "/gateways", GwConf),
-    ct:sleep(500),
-    {204, _} = request(get, "/gateways/stomp/authentication"),
-
+    init_gw("stomp"),
     AuthConf = #{
         mechanism => <<"password_based">>,
         backend => <<"built_in_database">>,
@@ -283,22 +314,18 @@ t_authn(_) ->
 
     {204, _} = request(delete, "/gateways/stomp/authentication"),
     {204, _} = request(get, "/gateways/stomp/authentication"),
-    {204, _} = request(delete, "/gateways/stomp").
+    ok.
 
 t_authn_data_mgmt(_) ->
-    GwConf = #{name => <<"stomp">>},
-    {201, _} = request(post, "/gateways", GwConf),
-    ct:sleep(500),
-    {204, _} = request(get, "/gateways/stomp/authentication"),
-
+    init_gw("stomp"),
     AuthConf = #{
         mechanism => <<"password_based">>,
         backend => <<"built_in_database">>,
         user_id_type => <<"clientid">>
     },
     {201, _} = request(post, "/gateways/stomp/authentication", AuthConf),
-    ct:sleep(500),
-    {200, ConfResp} = request(get, "/gateways/stomp/authentication"),
+    {200, ConfResp} =
+        ?retry(10, 10, {200, _} = request(get, "/gateways/stomp/authentication")),
     assert_confs(AuthConf, ConfResp),
 
     User1 = #{
@@ -358,11 +385,10 @@ t_authn_data_mgmt(_) ->
 
     {204, _} = request(delete, "/gateways/stomp/authentication"),
     {204, _} = request(get, "/gateways/stomp/authentication"),
-    {204, _} = request(delete, "/gateways/stomp").
+    ok.
 
 t_listeners_tcp(_) ->
-    GwConf = #{name => <<"stomp">>},
-    {201, _} = request(post, "/gateways", GwConf),
+    {204, _} = request(put, "/gateways/stomp", #{}),
     {404, _} = request(get, "/gateways/stomp/listeners"),
     LisConf = #{
         name => <<"def">>,
@@ -387,7 +413,7 @@ t_listeners_tcp(_) ->
 
     {204, _} = request(delete, "/gateways/stomp/listeners/stomp:tcp:def"),
     {404, _} = request(get, "/gateways/stomp/listeners/stomp:tcp:def"),
-    {204, _} = request(delete, "/gateways/stomp").
+    ok.
 
 t_listeners_authn(_) ->
     GwConf = #{
@@ -400,9 +426,7 @@ t_listeners_authn(_) ->
             }
         ]
     },
-    {201, _} = request(post, "/gateways", GwConf),
-    ct:sleep(500),
-    {200, ConfResp} = request(get, "/gateways/stomp"),
+    ConfResp = init_gw("stomp", GwConf),
     assert_confs(GwConf, ConfResp),
 
     AuthConf = #{
@@ -424,7 +448,7 @@ t_listeners_authn(_) ->
     {204, _} = request(delete, Path),
     %% FIXME: 204?
     {204, _} = request(get, Path),
-    {204, _} = request(delete, "/gateways/stomp").
+    ok.
 
 t_listeners_authn_data_mgmt(_) ->
     GwConf = #{
@@ -437,7 +461,7 @@ t_listeners_authn_data_mgmt(_) ->
             }
         ]
     },
-    {201, _} = request(post, "/gateways", GwConf),
+    {204, _} = request(put, "/gateways/stomp", GwConf),
     {200, ConfResp} = request(get, "/gateways/stomp"),
     assert_confs(GwConf, ConfResp),
 
@@ -514,13 +538,10 @@ t_listeners_authn_data_mgmt(_) ->
         {filename, "user-credentials.csv", CSVData}
     ]),
 
-    {204, _} = request(delete, "/gateways/stomp").
+    ok.
 
 t_authn_fuzzy_search(_) ->
-    GwConf = #{name => <<"stomp">>},
-    {201, _} = request(post, "/gateways", GwConf),
-    {204, _} = request(get, "/gateways/stomp/authentication"),
-
+    init_gw("stomp"),
     AuthConf = #{
         mechanism => <<"password_based">>,
         backend => <<"built_in_database">>,
@@ -561,7 +582,25 @@ t_authn_fuzzy_search(_) ->
 
     {204, _} = request(delete, "/gateways/stomp/authentication"),
     {204, _} = request(get, "/gateways/stomp/authentication"),
-    {204, _} = request(delete, "/gateways/stomp").
+    ok.
+
+%%--------------------------------------------------------------------
+%% Helpers
+
+init_gw(GwName) ->
+    init_gw(GwName, #{}).
+
+init_gw(GwName, GwConf) ->
+    {204, _} = request(put, "/gateways/" ++ GwName, GwConf),
+    ?retry(
+        10,
+        10,
+        begin
+            {200, #{status := Status} = RespConf} = request(get, "/gateways/" ++ GwName),
+            false = (Status == <<"unloaded">>),
+            RespConf
+        end
+    ).
 
 %%--------------------------------------------------------------------
 %% Asserts
diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl
index 0fed97517..4682aa02e 100644
--- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl
+++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl
@@ -94,7 +94,7 @@ maybe_unconvert_listeners(Conf) when is_map(Conf) ->
 maybe_unconvert_listeners(Conf) ->
     Conf.
 
-assert_feilds_apperence(Ks, Map) ->
+assert_fields_exist(Ks, Map) ->
     lists:foreach(
         fun(K) ->
             _ = maps:get(K, Map)
diff --git a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl
index 7299159ab..ca0dfe3c1 100644
--- a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl
@@ -25,7 +25,7 @@
 -import(
     emqx_gateway_test_utils,
     [
-        assert_feilds_apperence/2,
+        assert_fields_exist/2,
         request/2,
         request/3
     ]
@@ -730,7 +730,7 @@ t_rest_clienit_info(_) ->
                 binary_to_list(ClientId),
         {200, StompClient1} = request(get, ClientPath),
         ?assertEqual(StompClient, StompClient1),
-        assert_feilds_apperence(
+        assert_fields_exist(
             [
                 proto_name,
                 awaiting_rel_max,
@@ -787,7 +787,7 @@ t_rest_clienit_info(_) ->
 
         {200, Subs} = request(get, ClientPath ++ "/subscriptions"),
         ?assertEqual(1, length(Subs)),
-        assert_feilds_apperence([topic, qos], lists:nth(1, Subs)),
+        assert_fields_exist([topic, qos], lists:nth(1, Subs)),
 
         {201, _} = request(
             post,
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index 55226faf1..15188d97e 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -10,6 +10,11 @@
 - Enhance the `banned` feature [#9367](https://github.com/emqx/emqx/pull/9367).
   Now the corresponding session will be kicked when client is banned by `clientid`.
 
+- Redesign `/gateways` API [9364](https://github.com/emqx/emqx/pull/9364).
+  Use `PUT /gateways/{name}` instead of `POST /gateways`, gateway gets 'loaded'
+  automatically if needed. Use `PUT /gateways/{name}/enable/{true|false}` to
+  enable or disable gateway. No more `DELETE /gateways/{name}`.
+
 ## Bug fixes
 
 - Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index a84b2b25c..4c74a7081 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -10,6 +10,9 @@
 - 增加 `封禁` 功能 [#9367](https://github.com/emqx/emqx/pull/9367)。
   现在客户端通过 `clientid` 被封禁时将会踢掉对应的会话。
 
+- 重新设计了 /gateways API [9364](https://github.com/emqx/emqx/pull/9364)。
+  使用 PUT /gateways/{name} 代替了 POST /gateways，现在网关将在需要时自动加载，然后删除了 DELETE /gateways/{name}，之后可以使用 PUT /gateways/{name}/enable/{true|false} 来开启或禁用网关。
+
 ## 修复
 
 - 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。

From 9d1d273a1bd0f7db1efd8bdbcafd814c3db515d2 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Wed, 16 Nov 2022 15:59:40 +0100
Subject: [PATCH 21/69] chore: sign extra files on macos when packaging ee

---
 scripts/macos-sign-binaries.sh | 37 ++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/scripts/macos-sign-binaries.sh b/scripts/macos-sign-binaries.sh
index 11b6b734d..fed55faef 100755
--- a/scripts/macos-sign-binaries.sh
+++ b/scripts/macos-sign-binaries.sh
@@ -42,14 +42,29 @@ for keychain in ${keychains}; do
 done
 security -v list-keychains -s "${keychain_names[@]}" "${KEYCHAIN}"
 
-# sign
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/asn1-*/priv/lib/asn1rt_nif.so
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/bcrypt-*/priv/bcrypt_nif.so
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/crypto-*/priv/lib/{crypto.so,otp_test_engine.so}
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/jiffy-*/priv/jiffy.so
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/jq-*/priv/{jq_nif1.so,libjq.1.dylib,libonig.4.dylib,erlang_jq_port}
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/rocksdb-*/priv/liberocksdb.so
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
-find "${REL_DIR}/lib/" -name libquicer_nif.so -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \;
+# known runtime executables and binaries
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/lib/jq-*/priv/{jq_nif1.so,libjq.1.dylib,libonig.4.dylib,erlang_jq_port}
+# other files from runtime and dependencies
+for f in \
+        asn1rt_nif.so \
+        bcrypt_nif.so \
+        crc32cer_nif.so \
+        crypto.so \
+        crypto_callback.so \
+        jiffy.so \
+        liberocksdb.so \
+        libquicer_nif.so \
+        odbcserver \
+        otp_test_engine.so \
+        sasl_auth.so \
+        snappyer.so \
+        ; do
+    find "${REL_DIR}"/lib/ -name "$f" -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \;
+done

From 7b93b166dc845a0ce19a8069e5c8b8e972435aeb Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 17 Nov 2022 14:29:03 +0800
Subject: [PATCH 22/69] test: fix flaky banned case

since this suite has changed its start method to the `emqx_common_test_helpers:start_apps/1`, the `emqx_banned` server will never stop (its restart type is permanent), thus there is no need to manually start or stops it.
---
 apps/emqx/test/emqx_banned_SUITE.erl | 36 +++++++++++++++++-----------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl
index afbf9a579..a0ff5b5cc 100644
--- a/apps/emqx/test/emqx_banned_SUITE.erl
+++ b/apps/emqx/test/emqx_banned_SUITE.erl
@@ -97,22 +97,30 @@ t_check(_) ->
     ?assertEqual(0, emqx_banned:info(size)).
 
 t_unused(_) ->
-    catch emqx_banned:stop(),
-    {ok, Banned} = emqx_banned:start_link(),
-    {ok, _} = emqx_banned:create(#banned{
-        who = {clientid, <<"BannedClient1">>},
-        until = erlang:system_time(second)
-    }),
-    {ok, _} = emqx_banned:create(#banned{
-        who = {clientid, <<"BannedClient2">>},
-        until = erlang:system_time(second) - 1
-    }),
-    ?assertEqual(ignored, gen_server:call(Banned, unexpected_req)),
-    ?assertEqual(ok, gen_server:cast(Banned, unexpected_msg)),
-    ?assertEqual(ok, Banned ! ok),
+    Who1 = {clientid, <<"BannedClient1">>},
+    Who2 = {clientid, <<"BannedClient2">>},
+
+    ?assertMatch(
+        {ok, _},
+        emqx_banned:create(#banned{
+            who = Who1,
+            until = erlang:system_time(second)
+        })
+    ),
+    ?assertMatch(
+        {ok, _},
+        emqx_banned:create(#banned{
+            who = Who2,
+            until = erlang:system_time(second) - 1
+        })
+    ),
+    ?assertEqual(ignored, gen_server:call(emqx_banned, unexpected_req)),
+    ?assertEqual(ok, gen_server:cast(emqx_banned, unexpected_msg)),
     %% expiry timer
     timer:sleep(500),
-    ok = emqx_banned:stop().
+
+    ok = emqx_banned:delete(Who1),
+    ok = emqx_banned:delete(Who2).
 
 t_kick(_) ->
     ClientId = <<"client">>,

From c430218e26852cca72ef6806719a74bfcc5735f0 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 17 Nov 2022 16:26:50 +0800
Subject: [PATCH 23/69] chore: make remsh node name away from the atom DOS
 attack

The remsh node name is generated to be unique, this may cause atom leakage, so we need to change the generation rule to limit the total of these names
---
 bin/emqx     | 2 +-
 bin/nodetool | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index 811bb2981..89ebed7c6 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -396,7 +396,7 @@ remsh() {
 
 # Generate a random id
 relx_gen_id() {
-    od -t x -N 4 /dev/urandom | head -n1 | awk '{print $2}'
+    od -t u -N 4 /dev/urandom | head -n1 | awk '{print $2 % 1000}'
 }
 
 call_nodetool() {
diff --git a/bin/nodetool b/bin/nodetool
index 4af7aae02..b4f0a0183 100755
--- a/bin/nodetool
+++ b/bin/nodetool
@@ -226,9 +226,14 @@ nodename(Name) ->
 
 this_node_name(longnames, Name) ->
     [Node, Host] = re:split(Name, "@", [{return, list}, unicode]),
-    list_to_atom(lists:concat(["remsh_maint_", Node, os:getpid(), "@", Host]));
+    list_to_atom(lists:concat(["remsh_maint_", Node, node_name_suffix_id(), "@", Host]));
 this_node_name(shortnames, Name) ->
-    list_to_atom(lists:concat(["remsh_maint_", Name, os:getpid()])).
+    list_to_atom(lists:concat(["remsh_maint_", Name, node_name_suffix_id()])).
+
+%% use the reversed value that from pid mod 1000 as the node name suffix
+node_name_suffix_id() ->
+    Pid = os:getpid(),
+    string:slice(string:reverse(Pid), 0, 3).
 
 %% For windows???
 create_mnesia_dir(DataDir, NodeName) ->

From d36de9ac5026fa1e09ce62585c846f5fd2031997 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Mon, 14 Nov 2022 23:00:47 +0800
Subject: [PATCH 24/69] refactor: emqx_statsd hot update

---
 .../i18n/emqx_statsd_schema_i18n.conf         |   6 +
 apps/emqx_statsd/include/emqx_statsd.hrl      |   5 +-
 apps/emqx_statsd/src/emqx_statsd.app.src      |   4 +-
 apps/emqx_statsd/src/emqx_statsd.erl          | 116 ++++++------------
 apps/emqx_statsd/src/emqx_statsd_api.erl      |   9 +-
 apps/emqx_statsd/src/emqx_statsd_app.erl      |  11 +-
 apps/emqx_statsd/src/emqx_statsd_config.erl   |  54 ++++++++
 apps/emqx_statsd/src/emqx_statsd_schema.erl   |  35 +++++-
 apps/emqx_statsd/src/emqx_statsd_sup.erl      |  21 ++--
 apps/emqx_statsd/test/emqx_statsd_SUITE.erl   |  90 ++++++++++++--
 changes/v5.0.11-en.md                         |   2 +
 changes/v5.0.11-zh.md                         |   2 +
 12 files changed, 238 insertions(+), 117 deletions(-)
 create mode 100644 apps/emqx_statsd/src/emqx_statsd_config.erl

diff --git a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf b/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf
index 9c6eb5afb..46d654a46 100644
--- a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf
+++ b/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf
@@ -45,6 +45,12 @@ emqx_statsd_schema {
       zh: """指标的推送间隔。"""
     }
   }
+  tags {
+    desc {
+      en: """The tags for metrics."""
+      zh: """指标的标签。"""
+    }
+  }
 
   enable {
     desc {
diff --git a/apps/emqx_statsd/include/emqx_statsd.hrl b/apps/emqx_statsd/include/emqx_statsd.hrl
index 52f8774c0..92d856670 100644
--- a/apps/emqx_statsd/include/emqx_statsd.hrl
+++ b/apps/emqx_statsd/include/emqx_statsd.hrl
@@ -1,5 +1,2 @@
 -define(APP, emqx_statsd).
--define(DEFAULT_SAMPLE_TIME_INTERVAL, 10000).
--define(DEFAULT_FLUSH_TIME_INTERVAL, 10000).
--define(DEFAULT_HOST, "127.0.0.1").
--define(DEFAULT_PORT, 8125).
+-define(STATSD, [statsd]).
diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src
index 76b04204b..d8ec71da0 100644
--- a/apps/emqx_statsd/src/emqx_statsd.app.src
+++ b/apps/emqx_statsd/src/emqx_statsd.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_statsd, [
-    {description, "An OTP application"},
-    {vsn, "5.0.2"},
+    {description, "EMQ X Statsd"},
+    {vsn, "5.0.3"},
     {registered, []},
     {mod, {emqx_statsd_app, []}},
     {applications, [
diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl
index 8154c9027..4b0a98cd3 100644
--- a/apps/emqx_statsd/src/emqx_statsd.erl
+++ b/apps/emqx_statsd/src/emqx_statsd.erl
@@ -28,18 +28,17 @@
 -include_lib("emqx/include/logger.hrl").
 
 -export([
-    update/1,
     start/0,
     stop/0,
     restart/0,
-    %% for rpc
+    %% for rpc: remove after 5.1.x
     do_start/0,
     do_stop/0,
     do_restart/0
 ]).
 
 %% Interface
--export([start_link/1]).
+-export([start_link/0]).
 
 %% Internal Exports
 -export([
@@ -51,40 +50,15 @@
     terminate/2
 ]).
 
--record(state, {
-    timer :: reference() | undefined,
-    sample_time_interval :: pos_integer(),
-    flush_time_interval :: pos_integer(),
-    estatsd_pid :: pid()
-}).
-
-update(Config) ->
-    case
-        emqx_conf:update(
-            [statsd],
-            Config,
-            #{rawconf_with_defaults => true, override_to => cluster}
-        )
-    of
-        {ok, #{raw_config := NewConfigRows}} ->
-            ok = stop(),
-            case maps:get(<<"enable">>, Config, true) of
-                true ->
-                    ok = restart();
-                false ->
-                    ok = stop()
-            end,
-            {ok, NewConfigRows};
-        {error, Reason} ->
-            {error, Reason}
-    end.
+-define(SAMPLE_TIMEOUT, sample_timeout).
 
+%% Remove after 5.1.x
 start() -> check_multicall_result(emqx_statsd_proto_v1:start(mria_mnesia:running_nodes())).
 stop() -> check_multicall_result(emqx_statsd_proto_v1:stop(mria_mnesia:running_nodes())).
 restart() -> check_multicall_result(emqx_statsd_proto_v1:restart(mria_mnesia:running_nodes())).
 
 do_start() ->
-    emqx_statsd_sup:ensure_child_started(?APP, emqx_conf:get([statsd], #{})).
+    emqx_statsd_sup:ensure_child_started(?APP).
 
 do_stop() ->
     emqx_statsd_sup:ensure_child_stopped(?APP).
@@ -94,59 +68,51 @@ do_restart() ->
     ok = do_start(),
     ok.
 
-start_link(Opts) ->
-    gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []).
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
-init([Opts]) ->
+init([]) ->
     process_flag(trap_exit, true),
-    Tags = tags(maps:get(tags, Opts, #{})),
-    {Host, Port} = maps:get(server, Opts, {?DEFAULT_HOST, ?DEFAULT_PORT}),
-    Opts1 = maps:without(
-        [
-            sample_time_interval,
-            flush_time_interval
-        ],
-        Opts#{
-            tags => Tags,
-            host => Host,
-            port => Port,
-            prefix => <<"emqx">>
-        }
-    ),
-    {ok, Pid} = estatsd:start_link(maps:to_list(Opts1)),
-    SampleTimeInterval = maps:get(sample_time_interval, Opts, ?DEFAULT_FLUSH_TIME_INTERVAL),
-    FlushTimeInterval = maps:get(flush_time_interval, Opts, ?DEFAULT_FLUSH_TIME_INTERVAL),
+    #{
+        tags := TagsRaw,
+        server := {Host, Port},
+        sample_time_interval := SampleTimeInterval,
+        flush_time_interval := FlushTimeInterval
+    } = emqx_conf:get([statsd]),
+    Tags = maps:fold(fun(K, V, Acc) -> [{to_bin(K), to_bin(V)} | Acc] end, [], TagsRaw),
+    Opts = [{tags, Tags}, {host, Host}, {port, Port}, {prefix, <<"emqx">>}],
+    {ok, Pid} = estatsd:start_link(Opts),
     {ok,
-        ensure_timer(#state{
-            sample_time_interval = SampleTimeInterval,
-            flush_time_interval = FlushTimeInterval,
-            estatsd_pid = Pid
+        ensure_timer(#{
+            sample_time_interval => SampleTimeInterval,
+            flush_time_interval => FlushTimeInterval,
+            estatsd_pid => Pid
         })}.
 
 handle_call(_Req, _From, State) ->
-    {noreply, State}.
+    {reply, ignore, State}.
 
 handle_cast(_Msg, State) ->
     {noreply, State}.
 
 handle_info(
-    {timeout, Ref, sample_timeout},
-    State = #state{
-        sample_time_interval = SampleTimeInterval,
-        flush_time_interval = FlushTimeInterval,
-        estatsd_pid = Pid,
-        timer = Ref
+    {timeout, Ref, ?SAMPLE_TIMEOUT},
+    State = #{
+        sample_time_interval := SampleTimeInterval,
+        flush_time_interval := FlushTimeInterval,
+        estatsd_pid := Pid,
+        timer := Ref
     }
 ) ->
     Metrics = emqx_metrics:all() ++ emqx_stats:getstats() ++ emqx_vm_data(),
     SampleRate = SampleTimeInterval / FlushTimeInterval,
     StatsdMetrics = [
-        {gauge, trans_metrics_name(Name), Value, SampleRate, []}
+        {gauge, Name, Value, SampleRate, []}
      || {Name, Value} <- Metrics
     ],
-    estatsd:submit(Pid, StatsdMetrics),
-    {noreply, ensure_timer(State)};
-handle_info({'EXIT', Pid, Error}, State = #state{estatsd_pid = Pid}) ->
+    ok = estatsd:submit(Pid, StatsdMetrics),
+    {noreply, ensure_timer(State), hibernate};
+handle_info({'EXIT', Pid, Error}, State = #{estatsd_pid := Pid}) ->
     {stop, {shutdown, Error}, State};
 handle_info(_Msg, State) ->
     {noreply, State}.
@@ -154,16 +120,13 @@ handle_info(_Msg, State) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-terminate(_Reason, #state{estatsd_pid = Pid}) ->
+terminate(_Reason, #{estatsd_pid := Pid}) ->
     estatsd:stop(Pid),
     ok.
 
 %%------------------------------------------------------------------------------
 %% Internal function
 %%------------------------------------------------------------------------------
-trans_metrics_name(Name) ->
-    Name0 = atom_to_binary(Name, utf8),
-    binary_to_atom(<<"emqx.", Name0/binary>>, utf8).
 
 emqx_vm_data() ->
     Idle =
@@ -179,12 +142,8 @@ emqx_vm_data() ->
         {cpu_use, 100 - Idle}
     ] ++ emqx_vm:mem_info().
 
-tags(Map) ->
-    Tags = maps:to_list(Map),
-    [{atom_to_binary(Key, utf8), Value} || {Key, Value} <- Tags].
-
-ensure_timer(State = #state{sample_time_interval = SampleTimeInterval}) ->
-    State#state{timer = emqx_misc:start_timer(SampleTimeInterval, sample_timeout)}.
+ensure_timer(State = #{sample_time_interval := SampleTimeInterval}) ->
+    State#{timer => emqx_misc:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}.
 
 check_multicall_result({Results, []}) ->
     case
@@ -201,3 +160,8 @@ check_multicall_result({Results, []}) ->
     end;
 check_multicall_result({_, _}) ->
     error(multicall_failed).
+
+to_bin(B) when is_binary(B) -> B;
+to_bin(I) when is_integer(I) -> integer_to_binary(I);
+to_bin(L) when is_list(L) -> list_to_binary(L);
+to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8).
diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl
index 2f2e42303..6007a3327 100644
--- a/apps/emqx_statsd/src/emqx_statsd_api.erl
+++ b/apps/emqx_statsd/src/emqx_statsd_api.erl
@@ -77,15 +77,16 @@ statsd_config_schema() ->
 statsd_example() ->
     #{
         enable => true,
-        flush_time_interval => "32s",
-        sample_time_interval => "32s",
-        server => "127.0.0.1:8125"
+        flush_time_interval => "30s",
+        sample_time_interval => "30s",
+        server => "127.0.0.1:8125",
+        tags => #{}
     }.
 
 statsd(get, _Params) ->
     {200, emqx:get_raw_config([<<"statsd">>], #{})};
 statsd(put, #{body := Body}) ->
-    case emqx_statsd:update(Body) of
+    case emqx_statsd_config:update(Body) of
         {ok, NewConfig} ->
             {200, NewConfig};
         {error, Reason} ->
diff --git a/apps/emqx_statsd/src/emqx_statsd_app.erl b/apps/emqx_statsd/src/emqx_statsd_app.erl
index 4b34006ac..b885772e0 100644
--- a/apps/emqx_statsd/src/emqx_statsd_app.erl
+++ b/apps/emqx_statsd/src/emqx_statsd_app.erl
@@ -27,15 +27,8 @@
 
 start(_StartType, _StartArgs) ->
     {ok, Sup} = emqx_statsd_sup:start_link(),
-    maybe_enable_statsd(),
+    emqx_statsd_config:add_handler(),
     {ok, Sup}.
 stop(_) ->
+    emqx_statsd_config:remove_handler(),
     ok.
-
-maybe_enable_statsd() ->
-    case emqx_conf:get([statsd, enable], false) of
-        true ->
-            emqx_statsd_sup:ensure_child_started(?APP, emqx_conf:get([statsd], #{}));
-        false ->
-            ok
-    end.
diff --git a/apps/emqx_statsd/src/emqx_statsd_config.erl b/apps/emqx_statsd/src/emqx_statsd_config.erl
new file mode 100644
index 000000000..4ec71ed32
--- /dev/null
+++ b/apps/emqx_statsd/src/emqx_statsd_config.erl
@@ -0,0 +1,54 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_statsd_config).
+
+-behaviour(emqx_config_handler).
+
+-include("emqx_statsd.hrl").
+
+-export([add_handler/0, remove_handler/0]).
+-export([post_config_update/5]).
+-export([update/1]).
+
+update(Config) ->
+    case
+        emqx_conf:update(
+            ?STATSD,
+            Config,
+            #{rawconf_with_defaults => true, override_to => cluster}
+        )
+    of
+        {ok, #{raw_config := NewConfigRows}} ->
+            {ok, NewConfigRows};
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+add_handler() ->
+    ok = emqx_config_handler:add_handler(?STATSD, ?MODULE),
+    ok.
+
+remove_handler() ->
+    ok = emqx_config_handler:remove_handler(?STATSD),
+    ok.
+
+post_config_update(?STATSD, _Req, #{enable := true}, _Old, _AppEnvs) ->
+    emqx_statsd_sup:ensure_child_stopped(?APP),
+    emqx_statsd_sup:ensure_child_started(?APP);
+post_config_update(?STATSD, _Req, #{enable := false}, _Old, _AppEnvs) ->
+    emqx_statsd_sup:ensure_child_stopped(?APP);
+post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) ->
+    ok.
diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl
index 9efde5afc..3fb51f3bd 100644
--- a/apps/emqx_statsd/src/emqx_statsd_schema.erl
+++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl
@@ -25,7 +25,8 @@
     namespace/0,
     roots/0,
     fields/1,
-    desc/1
+    desc/1,
+    validations/0
 ]).
 
 namespace() -> "statsd".
@@ -45,7 +46,8 @@ fields("statsd") ->
             )},
         {server, fun server/1},
         {sample_time_interval, fun sample_interval/1},
-        {flush_time_interval, fun flush_interval/1}
+        {flush_time_interval, fun flush_interval/1},
+        {tags, fun tags/1}
     ].
 
 desc("statsd") -> ?DESC(statsd);
@@ -59,12 +61,37 @@ server(_) -> undefined.
 
 sample_interval(type) -> emqx_schema:duration_ms();
 sample_interval(required) -> true;
-sample_interval(default) -> "10s";
+sample_interval(default) -> "30s";
 sample_interval(desc) -> ?DESC(?FUNCTION_NAME);
 sample_interval(_) -> undefined.
 
 flush_interval(type) -> emqx_schema:duration_ms();
 flush_interval(required) -> true;
-flush_interval(default) -> "10s";
+flush_interval(default) -> "30s";
 flush_interval(desc) -> ?DESC(?FUNCTION_NAME);
 flush_interval(_) -> undefined.
+
+tags(type) -> map();
+tags(required) -> false;
+tags(default) -> #{};
+tags(desc) -> ?DESC(?FUNCTION_NAME);
+tags(_) -> undefined.
+
+validations() ->
+    [
+        {check_interval, fun check_interval/1}
+    ].
+
+check_interval(Conf) ->
+    case hocon_maps:get("statsd.sample_time_interval", Conf) of
+        undefined ->
+            ok;
+        Sample ->
+            Flush = hocon_maps:get("statsd.flush_time_interval", Conf),
+            case Sample =< Flush of
+                true ->
+                    true;
+                false ->
+                    {bad_interval, #{sample_time_interval => Sample, flush_time_interval => Flush}}
+            end
+    end.
diff --git a/apps/emqx_statsd/src/emqx_statsd_sup.erl b/apps/emqx_statsd/src/emqx_statsd_sup.erl
index 851dbf8cc..f14242113 100644
--- a/apps/emqx_statsd/src/emqx_statsd_sup.erl
+++ b/apps/emqx_statsd/src/emqx_statsd_sup.erl
@@ -10,7 +10,6 @@
 -export([
     start_link/0,
     ensure_child_started/1,
-    ensure_child_started/2,
     ensure_child_stopped/1
 ]).
 
@@ -19,7 +18,7 @@
 %% Helper macro for declaring children of supervisor
 -define(CHILD(Mod, Opts), #{
     id => Mod,
-    start => {Mod, start_link, [Opts]},
+    start => {Mod, start_link, Opts},
     restart => permanent,
     shutdown => 5000,
     type => worker,
@@ -29,13 +28,9 @@
 start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
--spec ensure_child_started(supervisor:child_spec()) -> ok.
-ensure_child_started(ChildSpec) when is_map(ChildSpec) ->
-    assert_started(supervisor:start_child(?MODULE, ChildSpec)).
-
--spec ensure_child_started(atom(), map()) -> ok.
-ensure_child_started(Mod, Opts) when is_atom(Mod) andalso is_map(Opts) ->
-    assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Opts))).
+-spec ensure_child_started(atom()) -> ok.
+ensure_child_started(Mod) when is_atom(Mod) ->
+    assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, []))).
 
 %% @doc Stop the child worker process.
 -spec ensure_child_stopped(any()) -> ok.
@@ -50,13 +45,17 @@ ensure_child_stopped(ChildId) ->
     end.
 
 init([]) ->
-    {ok, {{one_for_one, 10, 3600}, []}}.
+    Children =
+        case emqx_conf:get([statsd, enable], false) of
+            true -> [?CHILD(emqx_statsd, [])];
+            false -> []
+        end,
+    {ok, {{one_for_one, 100, 3600}, Children}}.
 
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
 
 assert_started({ok, _Pid}) -> ok;
-assert_started({ok, _Pid, _Info}) -> ok;
 assert_started({error, {already_started, _Pid}}) -> ok;
 assert_started({error, Reason}) -> erlang:error(Reason).
diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl
index 08c78dd07..2b5074f48 100644
--- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl
+++ b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl
@@ -5,28 +5,104 @@
 
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-import(emqx_dashboard_api_test_helpers, [request/3, uri/1]).
+
+-define(BASE_CONF, <<
+    "\n"
+    "statsd {\n"
+    "enable = true\n"
+    "flush_time_interval = 4s\n"
+    "sample_time_interval = 4s\n"
+    "server = \"127.0.0.1:8126\"\n"
+    "tags {\"t1\" = \"good\", test = 100}\n"
+    "}\n"
+>>).
 
 init_per_suite(Config) ->
-    emqx_common_test_helpers:start_apps([emqx_statsd]),
+    emqx_common_test_helpers:start_apps(
+        [emqx_conf, emqx_dashboard, emqx_statsd],
+        fun set_special_configs/1
+    ),
+    ok = emqx_common_test_helpers:load_config(emqx_statsd_schema, ?BASE_CONF, #{
+        raw_with_default => true
+    }),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_statsd]).
+    emqx_common_test_helpers:stop_apps([emqx_statsd, emqx_dashboard, emqx_conf]).
+
+set_special_configs(emqx_dashboard) ->
+    emqx_dashboard_api_test_helpers:set_default_config();
+set_special_configs(_) ->
+    ok.
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 t_statsd(_) ->
-    {ok, Socket} = gen_udp:open(8125),
+    {ok, Socket} = gen_udp:open(8126, [{active, true}]),
     receive
-        {udp, _Socket, _Host, _Port, Bin} ->
-            ?assert(length(Bin) > 50)
-    after 11 * 1000 ->
-        ?assert(true, failed)
+        {udp, Socket1, Host, Port, Data} ->
+            ct:pal("receive:~p~n", [{Socket, Socket1, Host, Port}]),
+            ?assert(length(Data) > 50),
+            ?assert(nomatch =/= string:find(Data, "\nemqx.cpu_use:"))
+    after 10 * 1000 ->
+        error(timeout)
     end,
     gen_udp:close(Socket).
 
 t_management(_) ->
     ?assertMatch(ok, emqx_statsd:start()),
+    ?assertMatch(ok, emqx_statsd:start()),
+    ?assertMatch(ok, emqx_statsd:stop()),
     ?assertMatch(ok, emqx_statsd:stop()),
     ?assertMatch(ok, emqx_statsd:restart()).
+
+t_rest_http(_) ->
+    {ok, Res0} = request(get),
+    ?assertEqual(
+        #{
+            <<"enable">> => true,
+            <<"flush_time_interval">> => <<"4s">>,
+            <<"sample_time_interval">> => <<"4s">>,
+            <<"server">> => <<"127.0.0.1:8126">>,
+            <<"tags">> => #{<<"t1">> => <<"good">>, <<"test">> => 100}
+        },
+        Res0
+    ),
+    {ok, Res1} = request(put, #{enable => false}),
+    ?assertMatch(#{<<"enable">> := false}, Res1),
+    ?assertEqual(maps:remove(<<"enable">>, Res0), maps:remove(<<"enable">>, Res1)),
+    {ok, Res2} = request(get),
+    ?assertEqual(Res1, Res2),
+    ?assertEqual(
+        error, request(put, #{sample_time_interval => "11s", flush_time_interval => "10s"})
+    ),
+    {ok, _} = request(put, #{enable => true}),
+    ok.
+
+t_kill_exit(_) ->
+    {ok, _} = request(put, #{enable => true}),
+    Pid = erlang:whereis(emqx_statsd),
+    ?assertEqual(ignore, gen_server:call(Pid, whatever)),
+    ?assertEqual(ok, gen_server:cast(Pid, whatever)),
+    ?assertEqual(Pid, erlang:whereis(emqx_statsd)),
+    #{estatsd_pid := Estatsd} = sys:get_state(emqx_statsd),
+    ?assert(erlang:exit(Estatsd, kill)),
+    ?assertEqual(false, is_process_alive(Estatsd)),
+    ct:sleep(150),
+    Pid1 = erlang:whereis(emqx_statsd),
+    ?assertNotEqual(Pid, Pid1),
+    #{estatsd_pid := Estatsd1} = sys:get_state(emqx_statsd),
+    ?assertNotEqual(Estatsd, Estatsd1),
+    ok.
+
+request(Method) -> request(Method, []).
+
+request(Method, Body) ->
+    case request(Method, uri(["statsd"]), Body) of
+        {ok, 200, Res} ->
+            {ok, emqx_json:decode(Res, [return_maps])};
+        {ok, _Status, _} ->
+            error
+    end.
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index 15188d97e..9e9506e00 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -15,6 +15,8 @@
   automatically if needed. Use `PUT /gateways/{name}/enable/{true|false}` to
   enable or disable gateway. No more `DELETE /gateways/{name}`.
 
+- Support `statsd {tags: {"user-defined-tag" = "tag-value"}` configure and improve stability of `emqx_statsd` [#9363](http://github.com/emqx/emqx/pull/9363).
+
 ## Bug fixes
 
 - Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 4c74a7081..845588eae 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -13,6 +13,8 @@
 - 重新设计了 /gateways API [9364](https://github.com/emqx/emqx/pull/9364)。
   使用 PUT /gateways/{name} 代替了 POST /gateways，现在网关将在需要时自动加载，然后删除了 DELETE /gateways/{name}，之后可以使用 PUT /gateways/{name}/enable/{true|false} 来开启或禁用网关。
 
+- 支持 `statsd {tags: {"user-defined-tag" = "tag-value"}` 配置，并提升 `emqx_statsd` 的稳定性 [#9363](http://github.com/emqx/emqx/pull/9363)。
+
 ## 修复
 
 - 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。

From 2e72256690d5f4aa9fa6a863c98c5bb49d3854f6 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Fri, 18 Nov 2022 11:52:18 +0800
Subject: [PATCH 25/69] chore: update changes

---
 changes/v5.0.11-en.md | 2 ++
 changes/v5.0.11-zh.md | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index 15188d97e..3178e4ee0 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -15,6 +15,8 @@
   automatically if needed. Use `PUT /gateways/{name}/enable/{true|false}` to
   enable or disable gateway. No more `DELETE /gateways/{name}`.
 
+- Improve node name generation rules to avoid potential atom table overflow risk [#9387](https://github.com/emqx/emqx/pull/9387).
+
 ## Bug fixes
 
 - Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 4c74a7081..10fc7cd1d 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -13,6 +13,8 @@
 - 重新设计了 /gateways API [9364](https://github.com/emqx/emqx/pull/9364)。
   使用 PUT /gateways/{name} 代替了 POST /gateways，现在网关将在需要时自动加载，然后删除了 DELETE /gateways/{name}，之后可以使用 PUT /gateways/{name}/enable/{true|false} 来开启或禁用网关。
 
+- 改进了节点名称生成规则，以避免潜在的原子表溢出风险 [#9387](https://github.com/emqx/emqx/pull/9387)。
+
 ## 修复
 
 - 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。

From 65820eb943a04b3e63edc7817d746f10a89c2087 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Fri, 18 Nov 2022 15:07:23 +0800
Subject: [PATCH 26/69] fix: can't get default raw config

---
 apps/emqx_management/src/emqx_mgmt_api_configs.erl   |  4 ++--
 .../test/emqx_mgmt_api_configs_SUITE.erl             | 12 ++++++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl
index db582c612..648ab9568 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl
@@ -268,7 +268,7 @@ config(put, #{body := Body}, Req) ->
 global_zone_configs(get, _Params, _Req) ->
     Paths = global_zone_roots(),
     Zones = lists:foldl(
-        fun(Path, Acc) -> Acc#{Path => get_config_with_default([Path])} end,
+        fun(Path, Acc) -> maps:merge(Acc, get_config_with_default(Path)) end,
         #{},
         Paths
     ),
@@ -343,7 +343,7 @@ get_full_config() ->
     ).
 
 get_config_with_default(Path) ->
-    emqx_config:fill_defaults(emqx:get_raw_config(Path)).
+    emqx_config:fill_defaults(#{Path => emqx:get_raw_config([Path])}).
 
 conf_path_from_querystr(Req) ->
     case proplists:get_value(<<"conf_path">>, cowboy_req:parse_qs(Req)) of
diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl
index 83f68c5fe..adea70af6 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl
@@ -133,6 +133,18 @@ t_global_zone(_Config) ->
 
     BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3),
     ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)),
+
+    %% Remove max_qos_allowed from raw config, but we still get default value(2).
+    Mqtt0 = emqx_conf:get_raw([<<"mqtt">>]),
+    ?assertEqual(1, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt0)),
+    Mqtt1 = maps:remove(<<"max_qos_allowed">>, Mqtt0),
+    ok = emqx_config:put_raw([<<"mqtt">>], Mqtt1),
+    Mqtt2 = emqx_conf:get_raw([<<"mqtt">>]),
+    ?assertNot(maps:is_key(<<"max_qos_allowed">>, Mqtt2), Mqtt2),
+    {ok, #{<<"mqtt">> := Mqtt3}} = get_global_zone(),
+    %% the default value is 2
+    ?assertEqual(2, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt3)),
+    ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0),
     ok.
 
 get_global_zone() ->

From b2db34708e07e9b57677aaa831720a397b56b9f6 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Fri, 18 Nov 2022 15:24:10 +0800
Subject: [PATCH 27/69] chore: add changelog for fix global_zone api

---
 changes/v5.0.11-en.md | 2 ++
 changes/v5.0.11-zh.md | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index 15188d97e..6ce0091f8 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -20,3 +20,5 @@
 - Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
 
 - Fix that JWT ACL rules are only applied if an `exp` claim is set [#9368](https://github.com/emqx/emqx/pull/9368).
+
+- Fix that `/configs/global_zone` API cannot get the default value of the configuration [#9392](https://github.com/emqx/emqx/pull/9392).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 4c74a7081..8072179e3 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -18,3 +18,5 @@
 - 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。
 
 - 修复 JWT ACL 规则只在设置了超期时间时才生效的问题 [#9368](https://github.com/emqx/emqx/pull/9368)。
+
+- 修复 `/configs/global_zone` API 无法正确获取配置的默认值问题 [#9392](https://github.com/emqx/emqx/pull/9392)。

From 1845e472e0d2e2d852b42b901606edd9ffd84836 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Fri, 4 Nov 2022 15:53:36 +0800
Subject: [PATCH 28/69] fix: create trace sometime failed by end_at time has
 already passed

---
 apps/emqx/src/emqx_trace/emqx_trace.erl    | 15 +++++++++------
 apps/emqx/src/emqx_trace/emqx_trace_dl.erl |  4 ++--
 changes/v5.0.10-en.md                      |  2 ++
 changes/v5.0.10-zh.md                      |  2 ++
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl
index 65756fc2f..d86a0af33 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace.erl
@@ -38,7 +38,8 @@
     delete/1,
     clear/0,
     update/2,
-    check/0
+    check/0,
+    now_second/0
 ]).
 
 -export([
@@ -287,7 +288,7 @@ insert_new_trace(Trace) ->
     transaction(fun emqx_trace_dl:insert_new_trace/1, [Trace]).
 
 update_trace(Traces) ->
-    Now = erlang:system_time(second),
+    Now = now_second(),
     {_Waiting, Running, Finished} = classify_by_time(Traces, Now),
     disable_finished(Finished),
     Started = emqx_trace_handler:running(),
@@ -455,7 +456,7 @@ ensure_map(Trace) when is_list(Trace) ->
     ).
 
 fill_default(Trace = #?TRACE{start_at = undefined}) ->
-    fill_default(Trace#?TRACE{start_at = erlang:system_time(second)});
+    fill_default(Trace#?TRACE{start_at = now_second()});
 fill_default(Trace = #?TRACE{end_at = undefined, start_at = StartAt}) ->
     fill_default(Trace#?TRACE{end_at = StartAt + 10 * 60});
 fill_default(Trace) ->
@@ -493,7 +494,7 @@ to_trace(#{start_at := StartAt} = Trace, Rec) ->
     {ok, Sec} = to_system_second(StartAt),
     to_trace(maps:remove(start_at, Trace), Rec#?TRACE{start_at = Sec});
 to_trace(#{end_at := EndAt} = Trace, Rec) ->
-    Now = erlang:system_time(second),
+    Now = now_second(),
     case to_system_second(EndAt) of
         {ok, Sec} when Sec > Now ->
             to_trace(maps:remove(end_at, Trace), Rec#?TRACE{end_at = Sec});
@@ -517,8 +518,7 @@ validate_ip_address(IP) ->
     end.
 
 to_system_second(Sec) ->
-    Now = erlang:system_time(second),
-    {ok, erlang:max(Now, Sec)}.
+    {ok, erlang:max(now_second(), Sec)}.
 
 zip_dir() ->
     filename:join([trace_dir(), "zip"]).
@@ -570,3 +570,6 @@ filter_cli_handler(Names) ->
         end,
         Names
     ).
+
+now_second() ->
+    os:system_time(second).
diff --git a/apps/emqx/src/emqx_trace/emqx_trace_dl.erl b/apps/emqx/src/emqx_trace/emqx_trace_dl.erl
index 3f96e1531..dd546f793 100644
--- a/apps/emqx/src/emqx_trace/emqx_trace_dl.erl
+++ b/apps/emqx/src/emqx_trace/emqx_trace_dl.erl
@@ -30,7 +30,7 @@
 -include("emqx_trace.hrl").
 
 %%================================================================================
-%% API funcions
+%% API functions
 %%================================================================================
 
 %% Introduced in 5.0
@@ -43,7 +43,7 @@ update(Name, Enable) ->
         [#?TRACE{enable = Enable}] ->
             ok;
         [Rec] ->
-            case erlang:system_time(second) >= Rec#?TRACE.end_at of
+            case emqx_trace:now_second() >= Rec#?TRACE.end_at of
                 false -> mnesia:write(?TRACE, Rec#?TRACE{enable = Enable}, write);
                 true -> mnesia:abort(finished)
             end
diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md
index 124e07063..9b04ec4fe 100644
--- a/changes/v5.0.10-en.md
+++ b/changes/v5.0.10-en.md
@@ -26,6 +26,8 @@
 
 ## Bug fixes
 
+- Fix create trace sometime failed by end_at time has already passed. [#9302](https://github.com/emqx/emqx/pull/9302)
+
 - Fix error log message when `mechanism` is missing in authentication config [#8924](https://github.com/emqx/emqx/pull/8924).
 
 - Fix HTTP 500 issue when unknown `status` parameter is used in `/gateway` API call [#9225](https://github.com/emqx/emqx/pull/9225).
diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md
index 89f345bcf..357bcdf42 100644
--- a/changes/v5.0.10-zh.md
+++ b/changes/v5.0.10-zh.md
@@ -25,6 +25,8 @@
 
 ## Bug fixes
 
+- 修复创建追踪日志时偶尔会报`end_at time has already passed`错误，导致创建失败。[#9302](https://github.com/emqx/emqx/pull/9302)
+
 - 优化认认证配置中 `mechanism` 字段缺失情况下的错误日志 [#8924](https://github.com/emqx/emqx/pull/8924)。
 
 - 修复未知 `status` 参数导致 `/gateway` API 发生 HTTP 500 错误的问题 [#9225](https://github.com/emqx/emqx/pull/9225)。

From 6f6ed1b4f80dd848f0819ce54f292b09faa92b74 Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Mon, 21 Nov 2022 10:12:43 +0800
Subject: [PATCH 29/69] chore: update 5.0.11 changelog

---
 changes/v5.0.10-en.md | 2 --
 changes/v5.0.10-zh.md | 2 --
 changes/v5.0.11-en.md | 2 ++
 changes/v5.0.11-zh.md | 2 ++
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/changes/v5.0.10-en.md b/changes/v5.0.10-en.md
index 9b04ec4fe..124e07063 100644
--- a/changes/v5.0.10-en.md
+++ b/changes/v5.0.10-en.md
@@ -26,8 +26,6 @@
 
 ## Bug fixes
 
-- Fix create trace sometime failed by end_at time has already passed. [#9302](https://github.com/emqx/emqx/pull/9302)
-
 - Fix error log message when `mechanism` is missing in authentication config [#8924](https://github.com/emqx/emqx/pull/8924).
 
 - Fix HTTP 500 issue when unknown `status` parameter is used in `/gateway` API call [#9225](https://github.com/emqx/emqx/pull/9225).
diff --git a/changes/v5.0.10-zh.md b/changes/v5.0.10-zh.md
index 357bcdf42..89f345bcf 100644
--- a/changes/v5.0.10-zh.md
+++ b/changes/v5.0.10-zh.md
@@ -25,8 +25,6 @@
 
 ## Bug fixes
 
-- 修复创建追踪日志时偶尔会报`end_at time has already passed`错误，导致创建失败。[#9302](https://github.com/emqx/emqx/pull/9302)
-
 - 优化认认证配置中 `mechanism` 字段缺失情况下的错误日志 [#8924](https://github.com/emqx/emqx/pull/8924)。
 
 - 修复未知 `status` 参数导致 `/gateway` API 发生 HTTP 500 错误的问题 [#9225](https://github.com/emqx/emqx/pull/9225)。
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index 0a5314e1d..d4c92da6f 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -19,6 +19,8 @@
 
 ## Bug fixes
 
+- Fix create trace sometime failed by end_at time has already passed. [#9302](https://github.com/emqx/emqx/pull/9302)
+
 - Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
 
 - Fix that JWT ACL rules are only applied if an `exp` claim is set [#9368](https://github.com/emqx/emqx/pull/9368).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 6bc968008..ae42aa0ae 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -17,6 +17,8 @@
 
 ## 修复
 
+- 修复创建追踪日志时偶尔会报`end_at time has already passed`错误，导致创建失败。[#9302](https://github.com/emqx/emqx/pull/9302)
+
 - 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。
 
 - 修复 JWT ACL 规则只在设置了超期时间时才生效的问题 [#9368](https://github.com/emqx/emqx/pull/9368)。

From 0dbeab88a2027c4b761b4045b546f10536c8d006 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Mon, 21 Nov 2022 14:38:59 +0800
Subject: [PATCH 30/69] chore: update emqx_statsd.app.src descriptions

Co-authored-by: Zaiming (Stone) Shi <zmstone@gmail.com>
---
 apps/emqx_statsd/src/emqx_statsd.app.src | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src
index d8ec71da0..5f32567d6 100644
--- a/apps/emqx_statsd/src/emqx_statsd.app.src
+++ b/apps/emqx_statsd/src/emqx_statsd.app.src
@@ -1,6 +1,6 @@
 %% -*- mode: erlang -*-
 {application, emqx_statsd, [
-    {description, "EMQ X Statsd"},
+    {description, "EMQX Statsd"},
     {vsn, "5.0.3"},
     {registered, []},
     {mod, {emqx_statsd_app, []}},

From 4e97344149aeae0ba46281dc68e7f1d2aa440ea4 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Mon, 21 Nov 2022 16:16:40 +0800
Subject: [PATCH 31/69] chore: apply suggestions from code review

Co-authored-by: JianBo He <heeejianbo@163.com>
---
 changes/v5.0.11-en.md | 2 +-
 changes/v5.0.11-zh.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index d4c92da6f..45547e1eb 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -19,7 +19,7 @@
 
 ## Bug fixes
 
-- Fix create trace sometime failed by end_at time has already passed. [#9302](https://github.com/emqx/emqx/pull/9302)
+- Fix create trace sometime failed by end_at time has already passed. [#9303](https://github.com/emqx/emqx/pull/9303)
 
 - Return 404 for status of unknown authenticator in `/authenticator/{id}/status` [#9328](https://github.com/emqx/emqx/pull/9328).
 
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index ae42aa0ae..d372c15cb 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -17,7 +17,7 @@
 
 ## 修复
 
-- 修复创建追踪日志时偶尔会报`end_at time has already passed`错误，导致创建失败。[#9302](https://github.com/emqx/emqx/pull/9302)
+- 修复创建追踪日志时偶尔会报`end_at time has already passed`错误，导致创建失败。[#9303](https://github.com/emqx/emqx/pull/9303)
 
 - 通过 `/authenticator/{id}/status` 请求未知认证器的状态时，将会返回 404。
 

From fcbf1bc8904ce219107734ff01538ccc1c001f0f Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 21 Nov 2022 21:41:00 +0800
Subject: [PATCH 32/69] fix(will-msg): fix mountpoint not working for will-msg.

---
 apps/emqx/src/emqx_channel.erl        |  8 ++++++--
 apps/emqx/test/emqx_channel_SUITE.erl | 16 ++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index ea35abfba..3b18c20cb 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -2134,10 +2134,14 @@ will_delay_interval(WillMsg) ->
         0
     ).
 
-publish_will_msg(ClientInfo, Msg = #message{topic = Topic}) ->
+publish_will_msg(
+    ClientInfo = #{mountpoint := MountPoint},
+    Msg = #message{topic = Topic}
+) ->
     case emqx_access_control:authorize(ClientInfo, publish, Topic) of
         allow ->
-            _ = emqx_broker:publish(Msg),
+            NMsg = emqx_mountpoint:mount(MountPoint, Msg),
+            _ = emqx_broker:publish(NMsg),
             ok;
         deny ->
             ?tp(
diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl
index a3fa3e5bc..153f0417d 100644
--- a/apps/emqx/test/emqx_channel_SUITE.erl
+++ b/apps/emqx/test/emqx_channel_SUITE.erl
@@ -728,6 +728,22 @@ t_quota_qos2(_) ->
     del_bucket(),
     esockd_limiter:stop().
 
+t_mount_will_msg(_) ->
+    Self = self(),
+    ClientInfo = clientinfo(#{mountpoint => <<"prefix/">>}),
+    Msg = emqx_message:make(test, <<"will_topic">>, <<"will_payload">>),
+    Channel = channel(#{clientinfo => ClientInfo, will_msg => Msg}),
+
+    ok = meck:expect(emqx_broker, publish, fun(M) -> Self ! {pub, M} end),
+
+    {shutdown, kicked, ok, ?DISCONNECT_PACKET(?RC_ADMINISTRATIVE_ACTION), _} = emqx_channel:handle_call(
+        kick, Channel
+    ),
+    receive
+        {pub, #message{topic = <<"prefix/will_topic">>}} -> ok
+    after 200 -> exit(will_message_not_published_or_not_correct)
+    end.
+
 %%--------------------------------------------------------------------
 %% Test cases for handle_deliver
 %%--------------------------------------------------------------------

From a6ca124772ab7833656aa41a39655e8f5a52c205 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 21 Nov 2022 21:45:02 +0800
Subject: [PATCH 33/69] chore: update changes

---
 changes/v5.0.11-en.md | 2 ++
 changes/v5.0.11-zh.md | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index 59ef42b4d..d83f9d435 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -29,3 +29,5 @@
 - Fix that JWT ACL rules are only applied if an `exp` claim is set [#9368](https://github.com/emqx/emqx/pull/9368).
 
 - Fix that `/configs/global_zone` API cannot get the default value of the configuration [#9392](https://github.com/emqx/emqx/pull/9392).
+
+- Fix mountpoint not working for will-msg [#9399](https://github.com/emqx/emqx/pull/9399).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index b48c09e7a..6bc12c5cd 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -26,3 +26,5 @@
 - 修复 JWT ACL 规则只在设置了超期时间时才生效的问题 [#9368](https://github.com/emqx/emqx/pull/9368)。
 
 - 修复 `/configs/global_zone` API 无法正确获取配置的默认值问题 [#9392](https://github.com/emqx/emqx/pull/9392)。
+
+- 修复 mountpoint 配置未对遗嘱消息生效的问题 [#9399](https://github.com/emqx/emqx/pull/9399)

From 118a97a0d88155abb21c57b0ce7e68ce6f47674c Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Tue, 22 Nov 2022 17:18:02 +0800
Subject: [PATCH 34/69] fix: change the defualt value of `max_topic_levels` to
 128

---
 apps/emqx/src/emqx_schema.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 4c26f86f9..1024cef48 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -399,7 +399,7 @@ fields("mqtt") ->
             sc(
                 range(1, 65535),
                 #{
-                    default => 65535,
+                    default => 128,
                     desc => ?DESC(mqtt_max_topic_levels)
                 }
             )},

From 461aa7fe9bb6e96b8e2a61eae5325414d1525042 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Tue, 22 Nov 2022 18:53:09 +0800
Subject: [PATCH 35/69] chore: update changes

---
 changes/v5.0.11-en.md | 1 +
 changes/v5.0.11-zh.md | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index abcf14bc4..def22e17c 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -19,6 +19,7 @@
 
 - Improve node name generation rules to avoid potential atom table overflow risk [#9387](https://github.com/emqx/emqx/pull/9387).
 
+- Set the default value for the maximum level of a topic to 128 [#9406](https://github.com/emqx/emqx/pull/9406).
 
 ## Bug fixes
 
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 19ee5c125..7a4e770f9 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -17,6 +17,8 @@
 
 - 改进了节点名称生成规则，以避免潜在的原子表溢出风险 [#9387](https://github.com/emqx/emqx/pull/9387)。
 
+- 将主题的最大层级限制的默认值设置为128 [#9406](https://github.com/emqx/emqx/pull/9406)。
+
 ## 修复
 
 - 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。

From 00c57de4c3ebef0f076952a4f62dfe1d92f64695 Mon Sep 17 00:00:00 2001
From: kraftwerk28 <kefirchik3@gmail.com>
Date: Wed, 26 Oct 2022 15:57:17 +0300
Subject: [PATCH 36/69] feat: do not drop MQTTv5 properties in rule/bridge

---
 apps/emqx/src/emqx_misc.erl                   | 17 +++-
 .../emqx_connector/src/emqx_connector.app.src |  2 +-
 .../src/mqtt/emqx_connector_mqtt_msg.erl      | 10 ++-
 .../src/emqx_rule_actions.erl                 | 27 +++++-
 .../src/emqx_rule_engine.app.src              |  2 +-
 .../emqx_rule_engine/src/emqx_rule_events.erl |  2 +-
 .../test/emqx_rule_engine_SUITE.erl           | 90 +++++++++++++++++--
 changes/v5.0.11-en.md                         |  2 +
 changes/v5.0.11-zh.md                         |  2 +
 9 files changed, 135 insertions(+), 19 deletions(-)

diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl
index 674465e6a..9683b1a8b 100644
--- a/apps/emqx/src/emqx_misc.erl
+++ b/apps/emqx/src/emqx_misc.erl
@@ -54,7 +54,8 @@
     pmap/3,
     readable_error_msg/1,
     safe_to_existing_atom/1,
-    safe_to_existing_atom/2
+    safe_to_existing_atom/2,
+    pub_props_to_packet/1
 ]).
 
 -export([
@@ -568,3 +569,17 @@ ipv6_probe_test() ->
     end.
 
 -endif.
+
+pub_props_to_packet(Properties) ->
+    F = fun
+        ('User-Property', M) ->
+            case is_map(M) andalso map_size(M) > 0 of
+                true -> {true, maps:to_list(M)};
+                false -> false
+            end;
+        ('User-Property-Pairs', _) ->
+            false;
+        (_, _) ->
+            true
+    end,
+    maps:filtermap(F, Properties).
diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src
index 64003209f..547a37b8e 100644
--- a/apps/emqx_connector/src/emqx_connector.app.src
+++ b/apps/emqx_connector/src/emqx_connector.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "An OTP application"},
-    {vsn, "0.1.7"},
+    {vsn, "0.1.8"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
index 43700506b..469dd952b 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl
@@ -77,17 +77,20 @@ to_remote_msg(MapMsg, #{
     Payload = process_payload(PayloadToken, MapMsg),
     QoS = replace_simple_var(QoSToken, MapMsg),
     Retain = replace_simple_var(RetainToken, MapMsg),
+    PubProps = maps:get(pub_props, MapMsg, #{}),
     #mqtt_msg{
         qos = QoS,
         retain = Retain,
         topic = topic(Mountpoint, Topic),
-        props = #{},
+        props = emqx_misc:pub_props_to_packet(PubProps),
         payload = Payload
     };
 to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) ->
     Msg#message{topic = topic(Mountpoint, Topic)}.
 
 %% published from remote node over a MQTT connection
+to_broker_msg(Msg, Vars, undefined) ->
+    to_broker_msg(Msg, Vars, #{});
 to_broker_msg(
     #{dup := Dup} = MapMsg,
     #{
@@ -103,8 +106,9 @@ to_broker_msg(
     Payload = process_payload(PayloadToken, MapMsg),
     QoS = replace_simple_var(QoSToken, MapMsg),
     Retain = replace_simple_var(RetainToken, MapMsg),
+    PubProps = maps:get(pub_props, MapMsg, #{}),
     set_headers(
-        Props,
+        Props#{properties => emqx_misc:pub_props_to_packet(PubProps)},
         emqx_message:set_flags(
             #{dup => Dup, retain => Retain},
             emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload)
@@ -151,8 +155,6 @@ estimate_size(#{topic := Topic, payload := Payload}) ->
 estimate_size(Term) ->
     erlang:external_size(Term).
 
-set_headers(undefined, Msg) ->
-    Msg;
 set_headers(Val, Msg) ->
     emqx_message:set_headers(Val, Msg).
 topic(undefined, Topic) -> Topic;
diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index 998fc1a5e..dd26f0a29 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -110,8 +110,9 @@ republish(
     Payload = format_msg(PayloadTks, Selected),
     QoS = replace_simple_var(QoSTks, Selected, 0),
     Retain = replace_simple_var(RetainTks, Selected, false),
+    PubProps = format_pub_props(maps:get(<<"pub_props">>, Selected, #{})),
     ?TRACE("RULE", "republish_message", #{topic => Topic, payload => Payload}),
-    safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload);
+    safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload, PubProps);
 %% in case this is a "$events/" event
 republish(
     Selected,
@@ -129,8 +130,9 @@ republish(
     Payload = format_msg(PayloadTks, Selected),
     QoS = replace_simple_var(QoSTks, Selected, 0),
     Retain = replace_simple_var(RetainTks, Selected, false),
+    PubProps = maps:get(pub_props, Selected, #{}),
     ?TRACE("RULE", "republish_message_with_flags", #{topic => Topic, payload => Payload}),
-    safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload).
+    safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload, PubProps).
 
 %%--------------------------------------------------------------------
 %% internal functions
@@ -168,13 +170,16 @@ pre_process_args(Mod, Func, Args) ->
         false -> Args
     end.
 
-safe_publish(RuleId, Topic, QoS, Flags, Payload) ->
+safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) ->
     Msg = #message{
         id = emqx_guid:gen(),
         qos = QoS,
         from = RuleId,
         flags = Flags,
-        headers = #{republish_by => RuleId},
+        headers = #{
+            republish_by => RuleId,
+            properties => emqx_misc:pub_props_to_packet(PubProps)
+        },
         topic = Topic,
         payload = Payload,
         timestamp = erlang:system_time(millisecond)
@@ -201,3 +206,17 @@ format_msg([], Selected) ->
     emqx_json:encode(Selected);
 format_msg(Tokens, Selected) ->
     emqx_plugin_libs_rule:proc_tmpl(Tokens, Selected).
+
+format_pub_props(Props) ->
+    maps:fold(fun format_pub_prop/3, #{}, Props).
+
+format_pub_prop(K, V, Acc) when is_atom(K) ->
+    Acc#{K => V};
+format_pub_prop(K, V, Acc) when is_binary(K) ->
+    try
+        K1 = erlang:binary_to_existing_atom(K),
+        format_pub_prop(K1, V, Acc)
+    catch
+        _:_ ->
+            Acc#{K => V}
+    end.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index 6bb9ad010..6419e4184 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.3"},
+    {vsn, "5.0.4"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [kernel, stdlib, rulesql, getopt]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl
index 2935aeeb9..b80b9777a 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_events.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl
@@ -1060,7 +1060,7 @@ printable_maps(Headers) ->
             (K, V, AccIn) ->
                 AccIn#{K => V}
         end,
-        #{},
+        #{'User-Property' => #{}},
         Headers
     ).
 
diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
index 17fe1a36c..087a73c34 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
@@ -59,11 +59,13 @@ groups() ->
             t_sqlselect_0,
             t_sqlselect_00,
             t_sqlselect_001,
+            t_sqlselect_inject_props,
             t_sqlselect_01,
             t_sqlselect_02,
             t_sqlselect_1,
             t_sqlselect_2,
             t_sqlselect_3,
+            t_sqlselect_message_publish_event,
             t_sqlparse_event_1,
             t_sqlparse_event_2,
             t_sqlparse_event_3,
@@ -936,9 +938,11 @@ t_sqlselect_001(_Config) ->
         )
     ).
 
-t_sqlselect_01(_Config) ->
+t_sqlselect_inject_props(_Config) ->
     SQL =
-        "SELECT json_decode(payload) as p, payload "
+        "SELECT json_decode(payload) as p, payload, "
+        "map_put('discard', 'discard', pub_props) as pub_props, "
+        "map_put('inject_key', 'inject_val', pub_props.'User-Property') as pub_props.'User-Property' "
         "FROM \"t3/#\", \"t1\" "
         "WHERE p.x = 1",
     Repub = republish_action(<<"t2">>),
@@ -949,10 +953,41 @@ t_sqlselect_01(_Config) ->
             actions => [Repub]
         }
     ),
-    {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
+    Props = user_properties(#{<<"inject_key">> => <<"inject_val">>}),
+    {ok, Client} = emqtt:start_link([{username, <<"emqx">>}, {proto_ver, v5}]),
     {ok, _} = emqtt:connect(Client),
     {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
-    emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0),
+    emqtt:publish(Client, <<"t1">>, #{}, <<"{\"x\":1}">>, [{qos, 0}]),
+    ct:sleep(100),
+    receive
+        {publish, #{topic := T, payload := Payload, properties := Props2}} ->
+            ?assertEqual(Props, Props2),
+            ?assertEqual(<<"t2">>, T),
+            ?assertEqual(<<"{\"x\":1}">>, Payload)
+    after 1000 ->
+        ct:fail(wait_for_t2)
+    end,
+    emqtt:stop(Client),
+    delete_rule(TopicRule1).
+
+t_sqlselect_01(_Config) ->
+    SQL =
+        "SELECT json_decode(payload) as p, payload, pub_props "
+        "FROM \"t3/#\", \"t1\" "
+        "WHERE p.x = 1",
+    Repub = republish_action(<<"t2">>),
+    {ok, TopicRule1} = emqx_rule_engine:create_rule(
+        #{
+            sql => SQL,
+            id => ?TMP_RULEID,
+            actions => [Repub]
+        }
+    ),
+    Props = user_properties(#{<<"mykey">> => <<"myval">>}),
+    {ok, Client} = emqtt:start_link([{username, <<"emqx">>}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(Client),
+    {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
+    emqtt:publish(Client, <<"t1">>, Props, <<"{\"x\":1}">>, [{qos, 0}]),
     ct:sleep(100),
     receive
         {publish, #{topic := T, payload := Payload}} ->
@@ -962,7 +997,7 @@ t_sqlselect_01(_Config) ->
         ct:fail(wait_for_t2)
     end,
 
-    emqtt:publish(Client, <<"t1">>, <<"{\"x\":2}">>, 0),
+    emqtt:publish(Client, <<"t1">>, Props, <<"{\"x\":2}">>, [{qos, 0}]),
     receive
         {publish, #{topic := <<"t2">>, payload := _}} ->
             ct:fail(unexpected_t2)
@@ -970,9 +1005,10 @@ t_sqlselect_01(_Config) ->
         ok
     end,
 
-    emqtt:publish(Client, <<"t3/a">>, <<"{\"x\":1}">>, 0),
+    emqtt:publish(Client, <<"t3/a">>, Props, <<"{\"x\":1}">>, [{qos, 0}]),
     receive
-        {publish, #{topic := T3, payload := Payload3}} ->
+        {publish, #{topic := T3, payload := Payload3, properties := Props2}} ->
+            ?assertEqual(Props, Props2),
             ?assertEqual(<<"t2">>, T3),
             ?assertEqual(<<"{\"x\":1}">>, Payload3)
     after 1000 ->
@@ -1135,6 +1171,43 @@ t_sqlselect_3(_Config) ->
     emqtt:stop(Client),
     delete_rule(TopicRule).
 
+t_sqlselect_message_publish_event(_Config) ->
+    %% republish the client.connected msg
+    Topic = <<"foo/bar/1">>,
+    SQL = <<
+        "SELECT clientid, pub_props "
+        "FROM \"$events/message_dropped\" "
+    >>,
+
+    %"WHERE topic = \"", Topic/binary, "\"">>,
+    Repub = republish_action(<<"t2">>, <<"clientid=${clientid}">>),
+    {ok, TopicRule} = emqx_rule_engine:create_rule(
+        #{
+            sql => SQL,
+            id => ?TMP_RULEID,
+            actions => [Repub]
+        }
+    ),
+    {ok, Client1} = emqtt:start_link([{clientid, <<"sub-01">>}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(Client1),
+    {ok, _, _} = emqtt:subscribe(Client1, <<"t2">>, 1),
+    ct:sleep(200),
+    {ok, Client2} = emqtt:start_link([{clientid, <<"pub-02">>}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(Client2),
+    Props = user_properties(#{<<"mykey">> => <<"222222222222">>}),
+    emqtt:publish(Client2, Topic, Props, <<"{\"x\":1}">>, [{qos, 1}]),
+    receive
+        {publish, #{topic := T, payload := Payload, properties := Props1}} ->
+            ?assertEqual(Props1, Props),
+            ?assertEqual(<<"t2">>, T),
+            ?assertEqual(<<"clientid=pub-02">>, Payload)
+    after 1000 ->
+        ct:fail(wait_for_t2)
+    end,
+    emqtt:stop(Client2),
+    emqtt:stop(Client1),
+    delete_rule(TopicRule).
+
 t_sqlparse_event_1(_Config) ->
     Sql =
         "select topic as tp "
@@ -2869,6 +2942,9 @@ verify_ipaddr(IPAddrS) ->
 init_events_counters() ->
     ets:new(events_record_tab, [named_table, bag, public]).
 
+user_properties(PairsMap) ->
+    #{'User-Property' => maps:to_list(PairsMap)}.
+
 %%------------------------------------------------------------------------------
 %% Start Apps
 %%------------------------------------------------------------------------------
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index def22e17c..e53c5785e 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -21,6 +21,8 @@
 
 - Set the default value for the maximum level of a topic to 128 [#9406](https://github.com/emqx/emqx/pull/9406).
 
+- Keep MQTT v5 User-Property pairs from bridge ingested MQTT messsages to bridge target [#9398](https://github.com/emqx/emqx/pull/9398).
+
 ## Bug fixes
 
 - Fix `ssl.existingName` option of  helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 7a4e770f9..3ea516dad 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -19,6 +19,8 @@
 
 - 将主题的最大层级限制的默认值设置为128 [#9406](https://github.com/emqx/emqx/pull/9406)。
 
+- 为桥接收到的 MQTT v5 消息再转发时保留 User-Property 列表 [#9398](https://github.com/emqx/emqx/pull/9398)。
+
 ## 修复
 
 - 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。

From f3df2c80d8b3a5b604ec8518091146d4e7cbb671 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Tue, 22 Nov 2022 19:39:21 +0100
Subject: [PATCH 37/69] feat: add user_properties arg for republish action

---
 .../i18n/emqx_rule_engine_schema.conf         | 26 +++++-
 .../src/emqx_rule_actions.erl                 | 88 ++++++++++---------
 .../src/emqx_rule_engine_schema.erl           |  9 ++
 .../test/emqx_rule_engine_SUITE.erl           | 87 +++++++++++++-----
 4 files changed, 148 insertions(+), 62 deletions(-)

diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf b/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf
index c0009a040..bc5735c67 100644
--- a/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf
+++ b/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf
@@ -218,7 +218,7 @@ Defaults to ${payload}. If variable ${payload} is not found from the selected re
 of the rule, then the string "undefined" is used.
 """
                          zh: """
-要重新发布的消息的有效负载。允许使用带有变量的模板，请参阅“republish_args”的描述。。
+要重新发布的消息的有效负载。允许使用带有变量的模板，请参阅“republish_args”的描述。
 默认为 ${payload}。 如果从所选结果中未找到变量 ${payload}，则使用字符串 "undefined"。
 """
                         }
@@ -227,6 +227,30 @@ of the rule, then the string "undefined" is used.
                            zh: "消息负载"
                           }
                   }
+    republish_args_user_properties {
+        desc {
+            en: """
+From which variable should the MQTT message's User-Property pairs be taken from.
+The value must be a map.
+You may configure it to <code>${pub_props.'User-Property'}</code> or
+use <code>SELECT *,pub_props.'User-Property' as user_properties</code>
+to forward the original user properties to the republished message.
+You may also call <code>map_put</code> function like
+<code>map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties</code>
+to inject user properties.
+NOTE: MQTT spec allows duplicated user property names, but EMQX Rule-Engine does not.
+"""
+            zh: """
+指定使用哪个变量来填充 MQTT 消息的 User-Property 列表。这个变量的值必须是一个 map 类型。
+可以设置成 <code>${pub_props.'User-Property'}</code> 或者
+使用 <code>SELECT *,pub_props.'User-Property' as user_properties</code> 来把源 MQTT 消息
+的 User-Property 列表用于填充。
+也可以使用 <code>map_put</code> 函数来添加新的 User-Property，
+<code>map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties</code>
+注意：MQTT 协议允许一个消息中出现多次同一个 property 名，但是 EMQX 的规则引擎不允许。
+"""
+        }
+    }
 
     rule_engine_ignore_sys_message {
                    desc {
diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index dd26f0a29..8971159e7 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -37,6 +37,8 @@
 
 -callback pre_process_action_args(FuncName :: atom(), action_fun_args()) -> action_fun_args().
 
+-define(ORIGINAL_USER_PROPERTIES, original).
+
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -57,7 +59,8 @@ pre_process_action_args(
         topic := Topic,
         qos := QoS,
         retain := Retain,
-        payload := Payload
+        payload := Payload,
+        user_properties := UserProperties
     } = Args
 ) ->
     Args#{
@@ -65,7 +68,8 @@ pre_process_action_args(
             topic => emqx_plugin_libs_rule:preproc_tmpl(Topic),
             qos => preproc_vars(QoS),
             retain => preproc_vars(Retain),
-            payload => emqx_plugin_libs_rule:preproc_tmpl(Payload)
+            payload => emqx_plugin_libs_rule:preproc_tmpl(Payload),
+            user_properties => preproc_user_properties(UserProperties)
         }
     };
 pre_process_action_args(_, Args) ->
@@ -93,16 +97,16 @@ republish(
     _Args
 ) ->
     ?SLOG(error, #{msg => "recursive_republish_detected", topic => Topic});
-%% republish a PUBLISH message
 republish(
     Selected,
-    #{flags := Flags, metadata := #{rule_id := RuleId}},
+    #{metadata := #{rule_id := RuleId}} = Env,
     #{
         preprocessed_tmpl := #{
             qos := QoSTks,
             retain := RetainTks,
             topic := TopicTks,
-            payload := PayloadTks
+            payload := PayloadTks,
+            user_properties := UserPropertiesTks
         }
     }
 ) ->
@@ -110,29 +114,22 @@ republish(
     Payload = format_msg(PayloadTks, Selected),
     QoS = replace_simple_var(QoSTks, Selected, 0),
     Retain = replace_simple_var(RetainTks, Selected, false),
-    PubProps = format_pub_props(maps:get(<<"pub_props">>, Selected, #{})),
-    ?TRACE("RULE", "republish_message", #{topic => Topic, payload => Payload}),
-    safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload, PubProps);
-%% in case this is a "$events/" event
-republish(
-    Selected,
-    #{metadata := #{rule_id := RuleId}},
-    #{
-        preprocessed_tmpl := #{
-            qos := QoSTks,
-            retain := RetainTks,
-            topic := TopicTks,
-            payload := PayloadTks
+    %% 'flags' is set for message re-publishes or message related
+    %% events such as message.acked and message.dropped
+    Flags0 = maps:get(flags, Env, #{}),
+    Flags = Flags0#{retain => Retain},
+    PubProps = format_pub_props(UserPropertiesTks, Selected, Env),
+    ?TRACE(
+        "RULE",
+        "republish_message",
+        #{
+            flags => Flags,
+            topic => Topic,
+            payload => Payload,
+            pub_props => PubProps
         }
-    }
-) ->
-    Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected),
-    Payload = format_msg(PayloadTks, Selected),
-    QoS = replace_simple_var(QoSTks, Selected, 0),
-    Retain = replace_simple_var(RetainTks, Selected, false),
-    PubProps = maps:get(pub_props, Selected, #{}),
-    ?TRACE("RULE", "republish_message_with_flags", #{topic => Topic, payload => Payload}),
-    safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload, PubProps).
+    ),
+    safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps).
 
 %%--------------------------------------------------------------------
 %% internal functions
@@ -192,6 +189,19 @@ preproc_vars(Data) when is_binary(Data) ->
 preproc_vars(Data) ->
     Data.
 
+preproc_user_properties(<<"${pub_props.'User-Property'}">>) ->
+    %% keep the original
+    %% avoid processing this special variable because
+    %% we do not want to force users to select the value
+    %% the value will be taken from Env.pub_props directly
+    ?ORIGINAL_USER_PROPERTIES;
+preproc_user_properties(<<"${", _/binary>> = V) ->
+    %% use a variable
+    emqx_plugin_libs_rule:preproc_tmpl(V);
+preproc_user_properties(_) ->
+    %% invalid, discard
+    undefined.
+
 replace_simple_var(Tokens, Data, Default) when is_list(Tokens) ->
     [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}),
     case Var of
@@ -207,16 +217,14 @@ format_msg([], Selected) ->
 format_msg(Tokens, Selected) ->
     emqx_plugin_libs_rule:proc_tmpl(Tokens, Selected).
 
-format_pub_props(Props) ->
-    maps:fold(fun format_pub_prop/3, #{}, Props).
-
-format_pub_prop(K, V, Acc) when is_atom(K) ->
-    Acc#{K => V};
-format_pub_prop(K, V, Acc) when is_binary(K) ->
-    try
-        K1 = erlang:binary_to_existing_atom(K),
-        format_pub_prop(K1, V, Acc)
-    catch
-        _:_ ->
-            Acc#{K => V}
-    end.
+format_pub_props(UserPropertiesTks, Selected, Env) ->
+    UserProperties =
+        case UserPropertiesTks of
+            ?ORIGINAL_USER_PROPERTIES ->
+                maps:get('User-Property', maps:get(pub_props, Env, #{}), #{});
+            undefined ->
+                #{};
+            _ ->
+                replace_simple_var(UserPropertiesTks, Selected, #{})
+        end,
+    #{'User-Property' => UserProperties}.
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
index eec41bde8..d299a6bb4 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
@@ -173,6 +173,15 @@ fields("republish_args") ->
                     default => <<"${payload}">>,
                     example => <<"${payload}">>
                 }
+            )},
+        {user_properties,
+            ?HOCON(
+                binary(),
+                #{
+                    desc => ?DESC("republish_args_user_properties"),
+                    default => <<"${user_properties}">>,
+                    example => <<"${pub_props.'User-Property'}">>
+                }
             )}
     ].
 
diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
index 087a73c34..50bb55fe1 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
@@ -65,7 +65,8 @@ groups() ->
             t_sqlselect_1,
             t_sqlselect_2,
             t_sqlselect_3,
-            t_sqlselect_message_publish_event,
+            t_sqlselect_message_publish_event_keep_original_props_1,
+            t_sqlselect_message_publish_event_keep_original_props_2,
             t_sqlparse_event_1,
             t_sqlparse_event_2,
             t_sqlparse_event_3,
@@ -941,8 +942,7 @@ t_sqlselect_001(_Config) ->
 t_sqlselect_inject_props(_Config) ->
     SQL =
         "SELECT json_decode(payload) as p, payload, "
-        "map_put('discard', 'discard', pub_props) as pub_props, "
-        "map_put('inject_key', 'inject_val', pub_props.'User-Property') as pub_props.'User-Property' "
+        "map_put('inject_key', 'inject_val', user_properties) as user_properties "
         "FROM \"t3/#\", \"t1\" "
         "WHERE p.x = 1",
     Repub = republish_action(<<"t2">>),
@@ -958,13 +958,12 @@ t_sqlselect_inject_props(_Config) ->
     {ok, _} = emqtt:connect(Client),
     {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
     emqtt:publish(Client, <<"t1">>, #{}, <<"{\"x\":1}">>, [{qos, 0}]),
-    ct:sleep(100),
     receive
         {publish, #{topic := T, payload := Payload, properties := Props2}} ->
             ?assertEqual(Props, Props2),
             ?assertEqual(<<"t2">>, T),
             ?assertEqual(<<"{\"x\":1}">>, Payload)
-    after 1000 ->
+    after 2000 ->
         ct:fail(wait_for_t2)
     end,
     emqtt:stop(Client),
@@ -972,10 +971,10 @@ t_sqlselect_inject_props(_Config) ->
 
 t_sqlselect_01(_Config) ->
     SQL =
-        "SELECT json_decode(payload) as p, payload, pub_props "
+        "SELECT json_decode(payload) as p, payload "
         "FROM \"t3/#\", \"t1\" "
         "WHERE p.x = 1",
-    Repub = republish_action(<<"t2">>),
+    Repub = republish_action(<<"t2">>, <<"${payload}">>, <<"${pub_props.'User-Property'}">>),
     {ok, TopicRule1} = emqx_rule_engine:create_rule(
         #{
             sql => SQL,
@@ -988,12 +987,11 @@ t_sqlselect_01(_Config) ->
     {ok, _} = emqtt:connect(Client),
     {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
     emqtt:publish(Client, <<"t1">>, Props, <<"{\"x\":1}">>, [{qos, 0}]),
-    ct:sleep(100),
     receive
         {publish, #{topic := T, payload := Payload}} ->
             ?assertEqual(<<"t2">>, T),
             ?assertEqual(<<"{\"x\":1}">>, Payload)
-    after 1000 ->
+    after 2000 ->
         ct:fail(wait_for_t2)
     end,
 
@@ -1001,7 +999,7 @@ t_sqlselect_01(_Config) ->
     receive
         {publish, #{topic := <<"t2">>, payload := _}} ->
             ct:fail(unexpected_t2)
-    after 1000 ->
+    after 2000 ->
         ok
     end,
 
@@ -1011,8 +1009,8 @@ t_sqlselect_01(_Config) ->
             ?assertEqual(Props, Props2),
             ?assertEqual(<<"t2">>, T3),
             ?assertEqual(<<"{\"x\":1}">>, Payload3)
-    after 1000 ->
-        ct:fail(wait_for_t2)
+    after 2000 ->
+        ct:fail(wait_for_t3)
     end,
 
     emqtt:stop(Client),
@@ -1080,13 +1078,12 @@ t_sqlselect_1(_Config) ->
     {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]),
     {ok, _} = emqtt:connect(Client),
     {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
-    ct:sleep(200),
     emqtt:publish(Client, <<"t1">>, <<"{\"x\":1,\"y\":2}">>, 0),
     receive
         {publish, #{topic := T, payload := Payload}} ->
             ?assertEqual(<<"t2">>, T),
             ?assertEqual(<<"{\"x\":1,\"y\":2}">>, Payload)
-    after 1000 ->
+    after 2000 ->
         ct:fail(wait_for_t2)
     end,
 
@@ -1149,14 +1146,13 @@ t_sqlselect_3(_Config) ->
     {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]),
     {ok, _} = emqtt:connect(Client),
     {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
-    ct:sleep(200),
     {ok, Client1} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]),
     {ok, _} = emqtt:connect(Client1),
     receive
         {publish, #{topic := T, payload := Payload}} ->
             ?assertEqual(<<"t2">>, T),
             ?assertEqual(<<"clientid=c_emqx1">>, Payload)
-    after 1000 ->
+    after 2000 ->
         ct:fail(wait_for_t2)
     end,
 
@@ -1171,11 +1167,51 @@ t_sqlselect_3(_Config) ->
     emqtt:stop(Client),
     delete_rule(TopicRule).
 
-t_sqlselect_message_publish_event(_Config) ->
+t_sqlselect_message_publish_event_keep_original_props_1(_Config) ->
     %% republish the client.connected msg
     Topic = <<"foo/bar/1">>,
     SQL = <<
-        "SELECT clientid, pub_props "
+        "SELECT clientid "
+        "FROM \"$events/message_dropped\" "
+    >>,
+
+    %"WHERE topic = \"", Topic/binary, "\"">>,
+    Repub = republish_action(
+        <<"t2">>,
+        <<"clientid=${clientid}">>,
+        <<"${pub_props.'User-Property'}">>
+    ),
+    {ok, TopicRule} = emqx_rule_engine:create_rule(
+        #{
+            sql => SQL,
+            id => ?TMP_RULEID,
+            actions => [Repub]
+        }
+    ),
+    {ok, Client1} = emqtt:start_link([{clientid, <<"sub-01">>}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(Client1),
+    {ok, _, _} = emqtt:subscribe(Client1, <<"t2">>, 1),
+    {ok, Client2} = emqtt:start_link([{clientid, <<"pub-02">>}, {proto_ver, v5}]),
+    {ok, _} = emqtt:connect(Client2),
+    Props = user_properties(#{<<"mykey">> => <<"111111">>}),
+    emqtt:publish(Client2, Topic, Props, <<"{\"x\":1}">>, [{qos, 1}]),
+    receive
+        {publish, #{topic := T, payload := Payload, properties := Props1}} ->
+            ?assertEqual(Props1, Props),
+            ?assertEqual(<<"t2">>, T),
+            ?assertEqual(<<"clientid=pub-02">>, Payload)
+    after 2000 ->
+        ct:fail(wait_for_t2)
+    end,
+    emqtt:stop(Client2),
+    emqtt:stop(Client1),
+    delete_rule(TopicRule).
+
+t_sqlselect_message_publish_event_keep_original_props_2(_Config) ->
+    %% republish the client.connected msg
+    Topic = <<"foo/bar/1">>,
+    SQL = <<
+        "SELECT clientid, pub_props.'User-Property' as user_properties "
         "FROM \"$events/message_dropped\" "
     >>,
 
@@ -1191,7 +1227,6 @@ t_sqlselect_message_publish_event(_Config) ->
     {ok, Client1} = emqtt:start_link([{clientid, <<"sub-01">>}, {proto_ver, v5}]),
     {ok, _} = emqtt:connect(Client1),
     {ok, _, _} = emqtt:subscribe(Client1, <<"t2">>, 1),
-    ct:sleep(200),
     {ok, Client2} = emqtt:start_link([{clientid, <<"pub-02">>}, {proto_ver, v5}]),
     {ok, _} = emqtt:connect(Client2),
     Props = user_properties(#{<<"mykey">> => <<"222222222222">>}),
@@ -1201,7 +1236,7 @@ t_sqlselect_message_publish_event(_Config) ->
             ?assertEqual(Props1, Props),
             ?assertEqual(<<"t2">>, T),
             ?assertEqual(<<"clientid=pub-02">>, Payload)
-    after 1000 ->
+    after 2000 ->
         ct:fail(wait_for_t2)
     end,
     emqtt:stop(Client2),
@@ -2553,10 +2588,20 @@ t_get_basic_usage_info_1(_Config) ->
 
 republish_action(Topic) ->
     republish_action(Topic, <<"${payload}">>).
+
 republish_action(Topic, Payload) ->
+    republish_action(Topic, Payload, <<"${user_properties}">>).
+
+republish_action(Topic, Payload, UserProperties) ->
     #{
         function => republish,
-        args => #{payload => Payload, topic => Topic, qos => 0, retain => false}
+        args => #{
+            payload => Payload,
+            topic => Topic,
+            qos => 0,
+            retain => false,
+            user_properties => UserProperties
+        }
     }.
 
 make_simple_rule_with_ts(RuleId, Ts) when is_binary(RuleId) ->

From 19405114e20dd0c64d012a41d52f7ce59897284a Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Wed, 23 Nov 2022 11:07:20 +0800
Subject: [PATCH 38/69] fix: fix newly found unsafe `binary_to_atom`

---
 apps/emqx_bridge/src/emqx_bridge_resource.erl | 11 ++++++++--
 .../src/emqx_dashboard_monitor_api.erl        | 22 +++++++++++++------
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl
index cbfeda4ba..c64d16d19 100644
--- a/apps/emqx_bridge/src/emqx_bridge_resource.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl
@@ -62,8 +62,15 @@ bridge_id(BridgeType, BridgeName) ->
 -spec parse_bridge_id(list() | binary() | atom()) -> {atom(), binary()}.
 parse_bridge_id(BridgeId) ->
     case string:split(bin(BridgeId), ":", all) of
-        [Type, Name] -> {binary_to_atom(Type, utf8), Name};
-        _ -> error({invalid_bridge_id, BridgeId})
+        [Type, Name] ->
+            case emqx_misc:safe_to_existing_atom(Type, utf8) of
+                {ok, Type1} ->
+                    {Type1, Name};
+                _ ->
+                    error({invalid_bridge_id, BridgeId})
+            end;
+        _ ->
+            error({invalid_bridge_id, BridgeId})
     end.
 
 reset_metrics(ResourceId) ->
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
index 50349bc40..01f53b2b2 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
@@ -121,13 +121,21 @@ fields(sampler_current) ->
 
 monitor(get, #{query_string := QS, bindings := Bindings}) ->
     Latest = maps:get(<<"latest">>, QS, infinity),
-    Node = binary_to_atom(maps:get(node, Bindings, <<"all">>)),
-    case emqx_dashboard_monitor:samplers(Node, Latest) of
-        {badrpc, {Node, Reason}} ->
-            Message = list_to_binary(io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason])),
-            {400, 'BAD_RPC', Message};
-        Samplers ->
-            {200, Samplers}
+    RawNode = maps:get(node, Bindings, all),
+    case emqx_misc:safe_to_existing_atom(RawNode, utf8) of
+        {ok, Node} ->
+            case emqx_dashboard_monitor:samplers(Node, Latest) of
+                {badrpc, {Node, Reason}} ->
+                    Message = list_to_binary(
+                        io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason])
+                    ),
+                    {400, 'BAD_RPC', Message};
+                Samplers ->
+                    {200, Samplers}
+            end;
+        _ ->
+            Message = list_to_binary(io_lib:format("Bad node ~p", [RawNode])),
+            {400, 'BAD_RPC', Message}
     end.
 
 monitor_current(get, #{bindings := Bindings}) ->

From 1719414c048b5952adf8f37227d266c512dae297 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Wed, 23 Nov 2022 11:42:49 +0800
Subject: [PATCH 39/69] chore: update app versions

---
 apps/emqx_bridge/src/emqx_bridge.app.src       | 2 +-
 apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src
index 7890853e4..6d01e004e 100644
--- a/apps/emqx_bridge/src/emqx_bridge.app.src
+++ b/apps/emqx_bridge/src/emqx_bridge.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "An OTP application"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {mod, {emqx_bridge_app, []}},
     {applications, [
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src
index fb290c551..9e639bcf8 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src
@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.7"},
+    {vsn, "5.0.8"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx]},

From acef85e11a14cf37c289ca8221a4460746942269 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Thu, 17 Nov 2022 16:28:26 +0100
Subject: [PATCH 40/69] test: initialize clients in test setup, teardown after
 test

---
 .../test/emqx_mgmt_api_publish_SUITE.erl      | 54 ++++++++++---------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl
index 0ebaf7195..1e82f5587 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl
@@ -20,9 +20,7 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
-
--define(CLIENTID, <<"api_clientid">>).
--define(USERNAME, <<"api_username">>).
+-include_lib("common_test/include/ct.hrl").
 
 -define(TOPIC1, <<"api_topic1">>).
 -define(TOPIC2, <<"api_topic2">>).
@@ -44,16 +42,21 @@ end_per_testcase(Case, Config) ->
     ?MODULE:Case({'end', Config}).
 
 t_publish_api({init, Config}) ->
-    Config;
-t_publish_api({'end', _Config}) ->
-    ok;
-t_publish_api(_) ->
-    {ok, Client} = emqtt:start_link(#{
-        username => <<"api_username">>, clientid => <<"api_clientid">>
-    }),
+    {ok, Client} = emqtt:start_link(
+        #{
+            username => <<"api_username">>,
+            clientid => <<"api_clientid">>
+        }
+    ),
     {ok, _} = emqtt:connect(Client),
     {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1),
     {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2),
+    [{client, Client} | Config];
+t_publish_api({'end', Config}) ->
+    Client = ?config(client, Config),
+    emqtt:stop(Client),
+    ok;
+t_publish_api(_) ->
     Payload = <<"hello">>,
     Path = emqx_mgmt_api_test_util:api_path(["publish"]),
     Auth = emqx_mgmt_api_test_util:auth_header_(),
@@ -61,8 +64,7 @@ t_publish_api(_) ->
     {ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body),
     ResponseMap = decode_json(Response),
     ?assertEqual([<<"id">>], lists:sort(maps:keys(ResponseMap))),
-    ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)),
-    emqtt:stop(Client).
+    ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)).
 
 t_publish_no_subscriber({init, Config}) ->
     Config;
@@ -163,16 +165,18 @@ t_publish_bad_topic_bulk(_Config) ->
     ).
 
 t_publish_bulk_api({init, Config}) ->
-    Config;
-t_publish_bulk_api({'end', _Config}) ->
-    ok;
-t_publish_bulk_api(_) ->
     {ok, Client} = emqtt:start_link(#{
         username => <<"api_username">>, clientid => <<"api_clientid">>
     }),
     {ok, _} = emqtt:connect(Client),
     {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1),
     {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2),
+    [{client, Client} | Config];
+t_publish_bulk_api({'end', Config}) ->
+    Client = ?config(client, Config),
+    emqtt:stop(Client),
+    ok;
+t_publish_bulk_api(_) ->
     Payload = <<"hello">>,
     Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]),
     Auth = emqx_mgmt_api_test_util:auth_header_(),
@@ -200,8 +204,7 @@ t_publish_bulk_api(_) ->
         ResponseList
     ),
     ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)),
-    ?assertEqual(ok, receive_assert(?TOPIC2, 0, Payload)),
-    emqtt:stop(Client).
+    ?assertEqual(ok, receive_assert(?TOPIC2, 0, Payload)).
 
 t_publish_no_subscriber_bulk({init, Config}) ->
     Config;
@@ -267,17 +270,19 @@ t_publish_bulk_dispatch_one_message_invalid_topic(Config) when is_list(Config) -
 t_publish_bulk_dispatch_failure({init, Config}) ->
     meck:new(emqx, [no_link, passthrough, no_history]),
     meck:expect(emqx, is_running, fun() -> false end),
-    Config;
-t_publish_bulk_dispatch_failure({'end', _Config}) ->
-    meck:unload(emqx),
-    ok;
-t_publish_bulk_dispatch_failure(Config) when is_list(Config) ->
     {ok, Client} = emqtt:start_link(#{
         username => <<"api_username">>, clientid => <<"api_clientid">>
     }),
     {ok, _} = emqtt:connect(Client),
     {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC1),
     {ok, _, [0]} = emqtt:subscribe(Client, ?TOPIC2),
+    [{client, Client} | Config];
+t_publish_bulk_dispatch_failure({'end', Config}) ->
+    meck:unload(emqx),
+    Client = ?config(client, Config),
+    emqtt:stop(Client),
+    ok;
+t_publish_bulk_dispatch_failure(Config) when is_list(Config) ->
     Payload = <<"hello">>,
     Path = emqx_mgmt_api_test_util:api_path(["publish", "bulk"]),
     Auth = emqx_mgmt_api_test_util:auth_header_(),
@@ -303,8 +308,7 @@ t_publish_bulk_dispatch_failure(Config) when is_list(Config) ->
             #{<<"reason_code">> := ?RC_NO_MATCHING_SUBSCRIBERS}
         ],
         decode_json(ResponseBody)
-    ),
-    emqtt:stop(Client).
+    ).
 
 receive_assert(Topic, Qos, Payload) ->
     receive

From afcb33aa200b12f471de4e2a6b40113b91471a83 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Thu, 17 Nov 2022 15:57:12 +0100
Subject: [PATCH 41/69] feat: support publish with properties

---
 .../i18n/emqx_mgmt_api_publish_i18n.conf      | 45 ++++++++++++
 .../src/emqx_mgmt_api_publish.erl             | 68 ++++++++++++++++++-
 .../test/emqx_mgmt_api_publish_SUITE.erl      | 51 ++++++++++----
 changes/v5.0.11-en.md                         |  2 +
 changes/v5.0.11-zh.md                         |  2 +
 5 files changed, 155 insertions(+), 13 deletions(-)

diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf
index 2a7c9def8..d845bff4b 100644
--- a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf
+++ b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf
@@ -124,4 +124,49 @@ MQTT 消息发布的错误码，这些错误码也是 MQTT 规范中 PUBACK 消
             zh: "失败的详细原因。"
         }
     }
+    message_properties {
+        desc {
+             en: "The Properties of the PUBLISH message."
+             zh: "PUBLISH 消息里的 Property 字段。"
+        }
+    }
+    msg_payload_format_indicator {
+        desc {
+             en: """0 (0x00) Byte Indicates that the Payload is unspecified bytes, which is equivalent to not sending a Payload Format Indicator.
+
+1 (0x01) Byte Indicates that the Payload is UTF-8 Encoded Character Data. The UTF-8 data in the Payload MUST be well-formed UTF-8 as defined by the Unicode specification and restated in RFC 3629.
+"""
+             zh: "载荷格式指示标识符，0 表示载荷是未指定格式的数据，相当于没有发送载荷格式指示；1 表示载荷是 UTF-8 编码的字符数据，载荷中的 UTF-8 数据必须是按照 Unicode 的规范和 RFC 3629 的标准要求进行编码的。"
+        }
+    }
+    msg_message_expiry_interval {
+        desc {
+             en: "Identifier of the Message Expiry Interval. If the Message Expiry Interval has passed and the Server has not managed to start onward delivery to a matching subscriber, then it MUST delete the copy of the message for that subscriber."
+             zh: "消息过期间隔标识符，以秒为单位。当消失已经过期时，如果服务端还没有开始向匹配的订阅者投递该消息，则服务端会删除该订阅者的消息副本。如果不设置，则消息永远不会过期"
+        }
+    }
+    msg_response_topic {
+        desc {
+             en: "Identifier of the Response Topic.The Response Topic MUST be a UTF-8 Encoded, It MUST NOT contain wildcard characters."
+             zh: "响应主题标识符， UTF-8 编码的字符串，用作响应消息的主题名。响应主题不能包含通配符，也不能包含多个主题，否则将造成协议错误。当存在响应主题时，消息将被视作请求报文。服务端在收到应用消息时必须将响应主题原封不动的发送给所有的订阅者。"
+        }
+    }
+    msg_correlation_data {
+        desc {
+             en: "Identifier of the Correlation Data. The Server MUST send the Correlation Data unaltered to all subscribers receiving the Application Message."
+             zh: "对比数据标识符，服务端在收到应用消息时必须原封不动的把对比数据发送给所有的订阅者。对比数据只对请求消息(Request Message)的发送端和响应消息(Response Message)的接收端有意义。"
+        }
+    }
+    msg_user_properties {
+        desc {
+             en: "The User-Property key-value pairs. Note: in case there are duplicated keys, only the last one will be used."
+             zh: "指定 MQTT 消息的 User Property 键值对。注意，如果出现重复的键，只有最后一个会保留。"
+        }
+    }
+    msg_content_type {
+        desc {
+             en: "The Content Type MUST be a UTF-8 Encoded String."
+             zh: "内容类型标识符，以 UTF-8 格式编码的字符串，用来描述应用消息的内容，服务端必须把收到的应用消息中的内容类型原封不动的发送给所有的订阅者。"
+        }
+    }
 }
diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl
index 1678c56e0..b1b1f1b5e 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl
@@ -114,6 +114,11 @@ fields(message) ->
                 required => true,
                 example => <<"hello emqx api">>
             })},
+        {properties,
+            hoconsc:mk(hoconsc:ref(?MODULE, message_properties), #{
+                desc => ?DESC(message_properties),
+                required => false
+            })},
         {retain,
             hoconsc:mk(boolean(), #{
                 desc => ?DESC(retain),
@@ -130,6 +135,43 @@ fields(publish_message) ->
                 default => plain
             })}
     ] ++ fields(message);
+fields(message_properties) ->
+    [
+        {'payload_format_indicator',
+            hoconsc:mk(typerefl:range(0, 1), #{
+                desc => ?DESC(msg_payload_format_indicator),
+                required => false,
+                example => 0
+            })},
+        {'message_expiry_interval',
+            hoconsc:mk(integer(), #{
+                desc => ?DESC(msg_message_expiry_interval),
+                required => false
+            })},
+        {'response_topic',
+            hoconsc:mk(binary(), #{
+                desc => ?DESC(msg_response_topic),
+                required => false,
+                example => <<"some_other_topic">>
+            })},
+        {'correlation_data',
+            hoconsc:mk(binary(), #{
+                desc => ?DESC(msg_correlation_data),
+                required => false
+            })},
+        {'user_properties',
+            hoconsc:mk(map(), #{
+                desc => ?DESC(msg_user_properties),
+                required => false,
+                example => #{<<"foo">> => <<"bar">>}
+            })},
+        {'content_type',
+            hoconsc:mk(binary(), #{
+                desc => ?DESC(msg_content_type),
+                required => false,
+                example => <<"text/plain">>
+            })}
+    ];
 fields(publish_ok) ->
     [
         {id,
@@ -288,13 +330,23 @@ make_message(Map) ->
             QoS = maps:get(<<"qos">>, Map, 0),
             Topic = maps:get(<<"topic">>, Map),
             Retain = maps:get(<<"retain">>, Map, false),
+            Headers =
+                case maps:get(<<"properties">>, Map, #{}) of
+                    Properties when
+                        is_map(Properties) andalso
+                            map_size(Properties) > 0
+                    ->
+                        #{properties => to_msg_properties(Properties)};
+                    _ ->
+                        #{}
+                end,
             try
                 _ = emqx_topic:validate(name, Topic)
             catch
                 error:_Reason ->
                     throw(invalid_topic_name)
             end,
-            Message = emqx_message:make(From, QoS, Topic, Payload, #{retain => Retain}, #{}),
+            Message = emqx_message:make(From, QoS, Topic, Payload, #{retain => Retain}, Headers),
             Size = emqx_message:estimate_size(Message),
             (Size > size_limit()) andalso throw(packet_too_large),
             {ok, Message};
@@ -302,6 +354,20 @@ make_message(Map) ->
             {error, R}
     end.
 
+to_msg_properties(Properties) ->
+    maps:fold(
+        fun to_property/3,
+        #{},
+        Properties
+    ).
+
+to_property(<<"payload_format_indicator">>, V, M) -> M#{'Payload-Format-Indicator' => V};
+to_property(<<"message_expiry_interval">>, V, M) -> M#{'Message-Expiry-Interval' => V};
+to_property(<<"response_topic">>, V, M) -> M#{'Response-Topic' => V};
+to_property(<<"correlation_data">>, V, M) -> M#{'Correlation-Data' => V};
+to_property(<<"user_properties">>, V, M) -> M#{'User-Property' => maps:to_list(V)};
+to_property(<<"content_type">>, V, M) -> M#{'Content-Type' => V}.
+
 %% get the global packet size limit since HTTP API does not belong to any zone.
 size_limit() ->
     try
diff --git a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl
index 1e82f5587..7622b0d17 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl
@@ -45,7 +45,8 @@ t_publish_api({init, Config}) ->
     {ok, Client} = emqtt:start_link(
         #{
             username => <<"api_username">>,
-            clientid => <<"api_clientid">>
+            clientid => <<"api_clientid">>,
+            proto_ver => v5
         }
     ),
     {ok, _} = emqtt:connect(Client),
@@ -60,11 +61,37 @@ t_publish_api(_) ->
     Payload = <<"hello">>,
     Path = emqx_mgmt_api_test_util:api_path(["publish"]),
     Auth = emqx_mgmt_api_test_util:auth_header_(),
-    Body = #{topic => ?TOPIC1, payload => Payload},
+    UserProperties = #{<<"foo">> => <<"bar">>},
+    Properties =
+        #{
+            <<"payload_format_indicator">> => 0,
+            <<"message_expiry_interval">> => 1000,
+            <<"response_topic">> => ?TOPIC2,
+            <<"correlation_data">> => <<"some_correlation_id">>,
+            <<"user_properties">> => UserProperties,
+            <<"content_type">> => <<"application/json">>
+        },
+    Body = #{topic => ?TOPIC1, payload => Payload, properties => Properties},
     {ok, Response} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body),
     ResponseMap = decode_json(Response),
     ?assertEqual([<<"id">>], lists:sort(maps:keys(ResponseMap))),
-    ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)).
+    {ok, Message} = receive_assert(?TOPIC1, 0, Payload),
+    RecvProperties = maps:get(properties, Message),
+    UserPropertiesList = maps:to_list(UserProperties),
+    #{
+        'Payload-Format-Indicator' := 0,
+        'Message-Expiry-Interval' := RecvMessageExpiry,
+        'Correlation-Data' := <<"some_correlation_id">>,
+        'User-Property' := UserPropertiesList,
+        'Content-Type' := <<"application/json">>
+    } = RecvProperties,
+    ?assert(RecvMessageExpiry =< 1000),
+    %% note: without props this time
+    Body2 = #{topic => ?TOPIC2, payload => Payload},
+    {ok, Response2} = emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, Body2),
+    ResponseMap2 = decode_json(Response2),
+    ?assertEqual([<<"id">>], lists:sort(maps:keys(ResponseMap2))),
+    ?assertEqual(ok, element(1, receive_assert(?TOPIC2, 0, Payload))).
 
 t_publish_no_subscriber({init, Config}) ->
     Config;
@@ -203,8 +230,8 @@ t_publish_bulk_api(_) ->
         end,
         ResponseList
     ),
-    ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)),
-    ?assertEqual(ok, receive_assert(?TOPIC2, 0, Payload)).
+    ?assertEqual(ok, element(1, receive_assert(?TOPIC1, 0, Payload))),
+    ?assertEqual(ok, element(1, receive_assert(?TOPIC2, 0, Payload))).
 
 t_publish_no_subscriber_bulk({init, Config}) ->
     Config;
@@ -235,8 +262,8 @@ t_publish_no_subscriber_bulk(_) ->
         ],
         ResponseList
     ),
-    ?assertEqual(ok, receive_assert(?TOPIC1, 0, Payload)),
-    ?assertEqual(ok, receive_assert(?TOPIC2, 0, Payload)),
+    ?assertEqual(ok, element(1, receive_assert(?TOPIC1, 0, Payload))),
+    ?assertEqual(ok, element(1, receive_assert(?TOPIC2, 0, Payload))),
     emqtt:stop(Client).
 
 t_publish_bulk_dispatch_one_message_invalid_topic({init, Config}) ->
@@ -316,12 +343,12 @@ receive_assert(Topic, Qos, Payload) ->
             ReceiveTopic = maps:get(topic, Message),
             ReceiveQos = maps:get(qos, Message),
             ReceivePayload = maps:get(payload, Message),
-            ?assertEqual(ReceiveTopic, Topic),
-            ?assertEqual(ReceiveQos, Qos),
-            ?assertEqual(ReceivePayload, Payload),
-            ok
+            ?assertEqual(Topic, ReceiveTopic),
+            ?assertEqual(Qos, ReceiveQos),
+            ?assertEqual(Payload, ReceivePayload),
+            {ok, Message}
     after 5000 ->
-        timeout
+        {error, timeout}
     end.
 
 decode_json(In) ->
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index e53c5785e..798ad8ce2 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -23,6 +23,8 @@
 
 - Keep MQTT v5 User-Property pairs from bridge ingested MQTT messsages to bridge target [#9398](https://github.com/emqx/emqx/pull/9398).
 
+- Support message properties in `/publish` API [#9401](https://github.com/emqx/emqx/pull/9401).
+
 ## Bug fixes
 
 - Fix `ssl.existingName` option of  helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 3ea516dad..34612d601 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -21,6 +21,8 @@
 
 - 为桥接收到的 MQTT v5 消息再转发时保留 User-Property 列表 [#9398](https://github.com/emqx/emqx/pull/9398)。
 
+- 支持在 /publish API 中添加消息属性 [#9401](https://github.com/emqx/emqx/pull/9401)。
+
 ## 修复
 
 - 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。

From 6dc5078a9678b008fd1f3d1af994503b5400182f Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Wed, 23 Nov 2022 17:40:10 +0800
Subject: [PATCH 42/69] fix: fix obsolete SSL files aren't deleted after the
 bridge configuration update

---
 apps/emqx_bridge/src/emqx_bridge_app.erl      |  3 +-
 apps/emqx_bridge/test/data/certs/cafile       | 29 ++++++
 apps/emqx_bridge/test/data/certs/certfile     | 24 +++++
 apps/emqx_bridge/test/data/certs/keyfile      | 27 ++++++
 apps/emqx_bridge/test/emqx_bridge_SUITE.erl   | 95 +++++++++++++++++++
 .../emqx_connector/src/emqx_connector_ssl.erl | 56 ++++++++---
 6 files changed, 222 insertions(+), 12 deletions(-)
 create mode 100644 apps/emqx_bridge/test/data/certs/cafile
 create mode 100644 apps/emqx_bridge/test/data/certs/certfile
 create mode 100644 apps/emqx_bridge/test/data/certs/keyfile

diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl
index 3fc4d57ba..958bbf288 100644
--- a/apps/emqx_bridge/src/emqx_bridge_app.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_app.erl
@@ -58,7 +58,8 @@ pre_config_update(Path, Conf, _OldConfig) when is_map(Conf) ->
 
 post_config_update(Path, '$remove', _, OldConf, _AppEnvs) ->
     _ = emqx_connector_ssl:clear_certs(filename:join(Path), OldConf);
-post_config_update(_Path, _Req, _, _OldConf, _AppEnvs) ->
+post_config_update(Path, _Req, NewConf, OldConf, _AppEnvs) ->
+    _ = emqx_connector_ssl:try_clear_certs(filename:join(Path), NewConf, OldConf),
     ok.
 
 %% internal functions
diff --git a/apps/emqx_bridge/test/data/certs/cafile b/apps/emqx_bridge/test/data/certs/cafile
new file mode 100644
index 000000000..8a9dafccd
--- /dev/null
+++ b/apps/emqx_bridge/test/data/certs/cafile
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE5DCCAswCCQCF3o0gIdaNDjANBgkqhkiG9w0BAQsFADA0MRIwEAYDVQQKDAlF
+TVFYIFRlc3QxHjAcBgNVBAMMFUNlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMTEy
+MzAwODQxMTFaFw00OTA1MTcwODQxMTFaMDQxEjAQBgNVBAoMCUVNUVggVGVzdDEe
+MBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF
+AAOCAg8AMIICCgKCAgEAqmqSrxyH16j63QhqGLT1UO8I+m6BM3HfnJQM8laQdtJ0
+WgHqCh0/OphH3S7v4SfF4fNJDEJWMWuuzJzU9cTqHPLzhvo3+ZHcMIENgtY2p2Cf
+7AQjEqFViEDyv2ZWNEe76BJeShntdY5NZr4gIPar99YGG/Ln8YekspleV+DU38rE
+EX9WzhgBr02NN9z4NzIxeB+jdvPnxcXs3WpUxzfnUjOQf/T1tManvSdRbFmKMbxl
+A8NLYK3oAYm8EbljWUINUNN6loqYhbigKv8bvo5S4xvRqmX86XB7sc0SApngtNcg
+O0EKn8z/KVPDskE+8lMfGMiU2e2Tzw6Rph57mQPOPtIp5hPiKRik7ST9n0p6piXW
+zRLplJEzSjf40I1u+VHmpXlWI/Fs8b1UkDSMiMVJf0LyWb4ziBSZOY2LtZzWHbWj
+LbNgxQcwSS29tKgUwfEFmFcm+iOM59cPfkl2IgqVLh5h4zmKJJbfQKSaYb5fcKRf
+50b1qsN40VbR3Pk/0lJ0/WqgF6kZCExmT1qzD5HJES/5grjjKA4zIxmHOVU86xOF
+ouWvtilVR4PGkzmkFvwK5yRhBUoGH/A9BurhqOc0QCGay1kqHQFA6se4JJS+9KOS
+x8Rn1Nm6Pi7sd6Le3cKmHTlyl5a/ofKqTCX2Qh+v/7y62V1V1wnoh3ipRjdPTnMC
+AwEAATANBgkqhkiG9w0BAQsFAAOCAgEARCqaocvlMFUQjtFtepO2vyG1krn11xJ0
+e7md26i+g8SxCCYqQ9IqGmQBg0Im8fyNDKRN/LZoj5+A4U4XkG1yya91ZIrPpWyF
+KUiRAItchNj3g1kHmI2ckl1N//6Kpx3DPaS7qXZaN3LTExf6Ph+StE1FnS0wVF+s
+tsNIf6EaQ+ZewW3pjdlLeAws3jvWKUkROc408Ngvx74zbbKo/zAC4tz8oH9ZcpsT
+WD8enVVEeUQKI6ItcpZ9HgTI9TFWgfZ1vYwvkoRwNIeabYI62JKmLEo2vGfGwWKr
+c+GjnJ/tlVI2DpPljfWOnQ037/7yyJI/zo65+HPRmGRD6MuW/BdPDYOvOZUTcQKh
+kANi5THSbJJgZcG3jb1NLebaUQ1H0zgVjn0g3KhUV+NJQYk8RQ7rHtB+MySqTKlM
+kRkRjfTfR0Ykxpks7Mjvsb6NcZENf08ZFPd45+e/ptsxpiKu4e4W4bV7NZDvNKf9
+0/aD3oGYNMiP7s+KJ1lRSAjnBuG21Yk8FpzG+yr8wvJhV8aFgNQ5wIH86SuUTmN0
+5bVzFEIcUejIwvGoQEctNHBlOwHrb7zmB6OwyZeMapdXBQ+9UDhYg8ehDqdDOdfn
+wsBcnjD2MwNhlE1hjL+tZWLNwSHiD6xx3LvNoXZu2HK8Cp3SOrkE69cFghYMIZZb
+T+fp6tNL6LE=
+-----END CERTIFICATE-----
diff --git a/apps/emqx_bridge/test/data/certs/certfile b/apps/emqx_bridge/test/data/certs/certfile
new file mode 100644
index 000000000..a198faf61
--- /dev/null
+++ b/apps/emqx_bridge/test/data/certs/certfile
@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID/jCCAeagAwIBAgIJAKTICmq1Lg6dMA0GCSqGSIb3DQEBCwUAMDQxEjAQBgNV
+BAoMCUVNUVggVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MB4X
+DTIxMTIzMDA4NDExMloXDTQ5MDUxNzA4NDExMlowJTESMBAGA1UECgwJRU1RWCBU
+ZXN0MQ8wDQYDVQQDDAZjbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDzrujfx6XZTH0MWqLO6kNAeHndUZ+OGaURXvxKMPMF5dA40lxNG6cEzzlq
+0Rm61adlv8tF4kRJrs6EnRjEVoMImrdh07vGFdOTYqP01LjiBhErAzyRtSn2X8FT
+Te8ExoCRs3x61SPebGY2hOvFxuO6YDPVOSDvbbxvRgqIlM1ZXC8dOvPSSGZ+P8hV
+56EPayRthfu1FVptnkW9CyZCRI0gg95Hv8RC7bGG+tuWpkN9ZrRvohhgGR1+bDUi
+BNBpncEsSh+UgWaj8KRN8D16H6m/Im6ty467j0at49FvPx5nACL48/ghtYvzgKLc
+uKHtokKUuuzebDK/hQxN3mUSAJStAgMBAAGjIjAgMAsGA1UdDwQEAwIFoDARBglg
+hkgBhvhCAQEEBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAIlVyPhOpkz3MNzQmjX7
+xgJ3vGPK5uK11n/wfjRwe2qXwZbrI2sYLVtTpUgvLDuP0gB73Vwfu7xAMdue6TRm
+CKr9z0lkQsVBtgoqzZCjd4PYLfHm4EhsOMi98OGKU5uOGD4g3yLwQWXHhbYtiZMO
+Jsj0hebYveYJt/BYTd1syGQcIcYCyVExWvSWjidfpAqjT6EF7whdubaFtuF2kaGF
+IO9yn9rWtXB5yK99uCguEmKhx3fAQxomzqweTu3WRvy9axsUH3WAUW9a4DIBSz2+
+ZSJNheFn5GktgggygJUGYqpSZHooUJW0UBs/8vX6AP+8MtINmqOGZUawmNwLWLOq
+wHyVt2YGD5TXjzzsWNSQ4mqXxM6AXniZVZK0yYNjA4ATikX1AtwunyWBR4IjyE/D
+FxYPORdZCOtywRFE1R5KLTUq/C8BNGCkYnoO78DJBO+pT0oagkQGQb0CnmC6C1db
+4lWzA9K0i4B0PyooZA+gp+5FFgaLuX1DkyeaY1J204QhHR1z/Vcyl5dpqR9hqnYP
+t8raLk9ogMDKqKA9iG0wc3CBNckD4sjVWAEeovXhElG55fD21wwhF+AnDCvX8iVK
+cBfKV6z6uxfKjGIxc2I643I5DiIn+V3DnPxYyY74Ln1lWFYmt5JREhAxPu42zq74
+e6+eIMYFszB+5gKgt6pa6ZNI
+-----END CERTIFICATE-----
diff --git a/apps/emqx_bridge/test/data/certs/keyfile b/apps/emqx_bridge/test/data/certs/keyfile
new file mode 100644
index 000000000..2f0af5d41
--- /dev/null
+++ b/apps/emqx_bridge/test/data/certs/keyfile
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA867o38el2Ux9DFqizupDQHh53VGfjhmlEV78SjDzBeXQONJc
+TRunBM85atEZutWnZb/LReJESa7OhJ0YxFaDCJq3YdO7xhXTk2Kj9NS44gYRKwM8
+kbUp9l/BU03vBMaAkbN8etUj3mxmNoTrxcbjumAz1Tkg7228b0YKiJTNWVwvHTrz
+0khmfj/IVeehD2skbYX7tRVabZ5FvQsmQkSNIIPeR7/EQu2xhvrblqZDfWa0b6IY
+YBkdfmw1IgTQaZ3BLEoflIFmo/CkTfA9eh+pvyJurcuOu49GrePRbz8eZwAi+PP4
+IbWL84Ci3Lih7aJClLrs3mwyv4UMTd5lEgCUrQIDAQABAoIBAQDwEbBgznrIwn8r
+jZt5x/brbAV7Ea/kOcWSgIaCvQifFdJ2OGAwov5/UXwajNgRZe2d4z7qoUhvYuUY
+ZwCAZU6ASpRBr2v9cYFYYURvrqZaHmoJew3P6q/lhl6aqFvC06DUagRHqvXEafyk
+13zEAvZVpfNKrBaTawPKiDFWb2qDDc9D6hC07EuJ/DNeehiHvzHrSZSDVV5Ut7Bw
+YDm33XygheUPAlHfeCnaixzcs3osiVyFEmVjxcIaM0ZS1NgcSaohSpJHMzvEaohX
+e+v9vccraSVlw01AlvFwI2vHYUV8jT6HwglTPKKGOCzK/ace3wPdYSU9qLcqfuHn
+EFhNc3tNAoGBAPugLMgbReJg2gpbIPUkYyoMMAAU7llFU1WvPWwXzo1a9EBjBACw
+WfCZISNtANXR38zIYXzoH547uXi4YPks1Nne3sYuCDpvuX+iz7fIo4zHf1nFmxH7
+eE6GtQr2ubmuuipTc28S0wBMGT1/KybH0e2NKL6GaOkNDmAI0IbEMBrvAoGBAPfr
+Y1QYLhPhan6m5g/5s+bQpKtHfNH9TNkk13HuYu72zNuY3qL2GC7oSadR8vTbRXZg
+KQqfaO0IGRcdkSFTq/AEhSSqr2Ld5nPadMbKvSGrSCc1s8rFH97jRVQY56yhM7ti
+IW4+6cE8ylCMbdYB6wuduK/GIgNpqoF4xs1i2XojAoGACacBUMPLEH4Kny8TupOk
+wi4pgTdMVVxVcAoC3yyincWJbRbfRm99Y79cCBHcYFdmsGJXawU0gUtlN/5KqgRQ
+PfNQtGV7p1I12XGTakdmDrZwai8sXao52TlNpJgGU9siBRGicfZU5cQFi9he/WPY
+57XshDJ/v8DidkigRysrdT0CgYEA5iuO22tblC+KvK1dGOXeZWO+DhrfwuGlcFBp
+CaimB2/w/8vsn2VVTG9yujo2E6hj1CQw1mDrfG0xRim4LTXOgpbfugwRqvuTUmo2
+Ur21XEX2RhjwpEfhcACWxB4fMUG0krrniMA2K6axupi1/KNpQi6bYe3UdFCs8Wld
+QSAOAvsCgYBk/X5PmD44DvndE5FShM2w70YOoMr3Cgl5sdwAFUFE9yDuC14UhVxk
+oxnYxwtVI9uVVirET+LczP9JEvcvxnN/Xg3tH/qm0WlIxmTxyYrFFIK9j0rqeu9z
+blPu56OzNI2VMrR1GbOBLxQINLTIpaacjNJAlr8XOlegdUJsW/Jwqw==
+-----END RSA PRIVATE KEY-----
diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl
index d8266f83a..dca14b829 100644
--- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl
@@ -156,3 +156,98 @@ setup_fake_telemetry_data() ->
     {ok, _} = snabbkaffe_collector:receive_events(Sub),
     ok = snabbkaffe:stop(),
     ok.
+
+t_update_ssl_conf(_) ->
+    Path = [bridges, <<"mqtt">>, <<"ssl_update_test">>],
+    EnableSSLConf = #{
+        <<"connector">> =>
+            #{
+                <<"bridge_mode">> => false,
+                <<"clean_start">> => true,
+                <<"keepalive">> => <<"60s">>,
+                <<"mode">> => <<"cluster_shareload">>,
+                <<"proto_ver">> => <<"v4">>,
+                <<"server">> => <<"127.0.0.1:1883">>,
+                <<"ssl">> =>
+                    #{
+                        <<"cacertfile">> => cert_file("cafile"),
+                        <<"certfile">> => cert_file("certfile"),
+                        <<"enable">> => true,
+                        <<"keyfile">> => cert_file("keyfile"),
+                        <<"verify">> => <<"verify_peer">>
+                    }
+            },
+        <<"direction">> => <<"ingress">>,
+        <<"local_qos">> => 1,
+        <<"payload">> => <<"${payload}">>,
+        <<"remote_qos">> => 1,
+        <<"remote_topic">> => <<"t/#">>,
+        <<"retain">> => false
+    },
+
+    emqx:update_config(Path, EnableSSLConf),
+    ?assertMatch({ok, [_, _, _]}, list_pem_dir(Path)),
+    NoSSLConf = #{
+        <<"connector">> =>
+            #{
+                <<"bridge_mode">> => false,
+                <<"clean_start">> => true,
+                <<"keepalive">> => <<"60s">>,
+                <<"max_inflight">> => 32,
+                <<"mode">> => <<"cluster_shareload">>,
+                <<"password">> => <<>>,
+                <<"proto_ver">> => <<"v4">>,
+                <<"reconnect_interval">> => <<"15s">>,
+                <<"replayq">> =>
+                    #{<<"offload">> => false, <<"seg_bytes">> => <<"100MB">>},
+                <<"retry_interval">> => <<"15s">>,
+                <<"server">> => <<"127.0.0.1:1883">>,
+                <<"ssl">> =>
+                    #{
+                        <<"ciphers">> => <<>>,
+                        <<"depth">> => 10,
+                        <<"enable">> => false,
+                        <<"reuse_sessions">> => true,
+                        <<"secure_renegotiate">> => true,
+                        <<"user_lookup_fun">> => <<"emqx_tls_psk:lookup">>,
+                        <<"verify">> => <<"verify_peer">>,
+                        <<"versions">> =>
+                            [
+                                <<"tlsv1.3">>,
+                                <<"tlsv1.2">>,
+                                <<"tlsv1.1">>,
+                                <<"tlsv1">>
+                            ]
+                    },
+                <<"username">> => <<>>
+            },
+        <<"direction">> => <<"ingress">>,
+        <<"enable">> => true,
+        <<"local_qos">> => 1,
+        <<"payload">> => <<"${payload}">>,
+        <<"remote_qos">> => 1,
+        <<"remote_topic">> => <<"t/#">>,
+        <<"retain">> => false
+    },
+
+    emqx:update_config(Path, NoSSLConf),
+    ?assertMatch({error, not_dir}, list_pem_dir(Path)),
+    emqx:remove_config(Path),
+    ok.
+
+list_pem_dir(Path) ->
+    Dir = filename:join([emqx:mutable_certs_dir() | Path]),
+    case filelib:is_dir(Dir) of
+        true ->
+            file:list_dir(Dir);
+        _ ->
+            {error, not_dir}
+    end.
+
+data_file(Name) ->
+    Dir = code:lib_dir(emqx_bridge, test),
+    {ok, Bin} = file:read_file(filename:join([Dir, "data", Name])),
+    Bin.
+
+cert_file(Name) ->
+    data_file(filename:join(["certs", Name])).
diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl
index 7f2fc537b..4c8197efa 100644
--- a/apps/emqx_connector/src/emqx_connector_ssl.erl
+++ b/apps/emqx_connector/src/emqx_connector_ssl.erl
@@ -16,9 +16,12 @@
 
 -module(emqx_connector_ssl).
 
+-include_lib("emqx/include/logger.hrl").
+
 -export([
     convert_certs/2,
-    clear_certs/2
+    clear_certs/2,
+    try_clear_certs/3
 ]).
 
 %% TODO: rm `connector` case after `dev/ee5.0` merged into `master`.
@@ -43,21 +46,37 @@ convert_certs(RltvDir, #{ssl := SSL} = Config) ->
 convert_certs(_RltvDir, Config) ->
     {ok, Config}.
 
-clear_certs(RltvDir, #{<<"connector">> := Connector} = _Config) when
+clear_certs(RltvDir, Config) ->
+    clear_certs2(RltvDir, normalize_key_to_bin(Config)).
+
+clear_certs2(RltvDir, #{<<"connector">> := Connector} = _Config) when
     is_map(Connector)
 ->
     OldSSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
     ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
-clear_certs(RltvDir, #{connector := Connector} = _Config) when
-    is_map(Connector)
+clear_certs2(RltvDir, #{<<"ssl">> := OldSSL} = _Config) ->
+    ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
+clear_certs2(_RltvDir, _) ->
+    ok.
+
+try_clear_certs(RltvDir, NewConf, OldConf) ->
+    try_clear_certs2(
+        RltvDir,
+        normalize_key_to_bin(NewConf),
+        normalize_key_to_bin(OldConf)
+    ).
+
+try_clear_certs2(RltvDir, #{<<"connector">> := NewConnector}, #{<<"connector">> := OldConnector}) when
+    is_map(NewConnector),
+    is_map(OldConnector)
 ->
-    OldSSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
-    ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
-clear_certs(RltvDir, #{<<"ssl">> := OldSSL} = _Config) ->
-    ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
-clear_certs(RltvDir, #{ssl := OldSSL} = _Config) ->
-    ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
-clear_certs(_RltvDir, _) ->
+    NewSSL = map_get_oneof([<<"ssl">>, ssl], NewConnector, undefined),
+    OldSSL = map_get_oneof([<<"ssl">>, ssl], OldConnector, undefined),
+    ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL);
+try_clear_certs2(RltvDir, #{<<"ssl">> := NewSSL}, #{<<"ssl">> := OldSSL}) ->
+    ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL);
+try_clear_certs2(RltvDir, NewConf, OldConf) ->
+    ?SLOG(debug, #{msg => "unexpected_conf", path => RltvDir, new => NewConf, OldConf => OldConf}),
     ok.
 
 new_ssl_config(RltvDir, Config, SSL) ->
@@ -88,3 +107,18 @@ map_get_oneof([Key | Keys], Map, Default) ->
         {ok, Value} ->
             Value
     end.
+
+normalize_key_to_bin(Map) when is_map(Map) ->
+    maps:fold(
+        fun
+            (K, V, Acc) when is_atom(K) ->
+                Bin = erlang:atom_to_binary(K, utf8),
+                Acc#{Bin => V};
+            (K, V, Acc) ->
+                Acc#{K => V}
+        end,
+        #{},
+        Map
+    );
+normalize_key_to_bin(Any) ->
+    Any.

From dd82899118a3a6382be588ee29cb445d2c1dd96a Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Wed, 23 Nov 2022 18:36:30 +0800
Subject: [PATCH 43/69] chore: update app version && changes

---
 changes/v5.0.11-en.md | 2 ++
 changes/v5.0.11-zh.md | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index e53c5785e..87e67c794 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -36,3 +36,5 @@
 - Fix that `/configs/global_zone` API cannot get the default value of the configuration [#9392](https://github.com/emqx/emqx/pull/9392).
 
 - Fix mountpoint not working for will-msg [#9399](https://github.com/emqx/emqx/pull/9399).
+
+- Fix that the obsolete SSL files aren't deleted after the bridge configuration update [#9411](https://github.com/emqx/emqx/pull/9411).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 3ea516dad..812dc3cb0 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -34,3 +34,5 @@
 - 修复 `/configs/global_zone` API 无法正确获取配置的默认值问题 [#9392](https://github.com/emqx/emqx/pull/9392)。
 
 - 修复 mountpoint 配置未对遗嘱消息生效的问题 [#9399](https://github.com/emqx/emqx/pull/9399)
+
+- 修复桥接配置更新 SSL 相关配置后，过时的 SSL 文件没有被删除的问题 [#9411](https://github.com/emqx/emqx/pull/9411)。

From 6f67e3b3331b49dde4177f9d70508a98f96d483d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 23 Nov 2022 15:54:13 +0100
Subject: [PATCH 44/69] feat(emqx_map_lib): add a binary_key_map help function

---
 apps/emqx/src/emqx_map_lib.erl | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx/src/emqx_map_lib.erl
index c714d7dbc..b01391c7b 100644
--- a/apps/emqx/src/emqx_map_lib.erl
+++ b/apps/emqx/src/emqx_map_lib.erl
@@ -23,6 +23,7 @@
     deep_force_put/3,
     deep_remove/2,
     deep_merge/2,
+    binary_key_map/1,
     safe_atom_key_map/1,
     unsafe_atom_key_map/1,
     jsonable_map/1,
@@ -153,6 +154,17 @@ deep_convert(Val, _, _Args) ->
 unsafe_atom_key_map(Map) ->
     covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end).
 
+-spec binary_key_map(map()) -> map().
+binary_key_map(Map) ->
+    deep_convert(
+        Map,
+        fun
+            (K, V) when is_atom(K) -> {atom_to_binary(K, utf8), V};
+            (K, V) when is_binary(K) -> {K, V}
+        end,
+        []
+    ).
+
 -spec safe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}.
 safe_atom_key_map(Map) ->
     covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end).

From 67cef65fdbfd26c5f4944bef28ac75e10e87c92e Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 23 Nov 2022 23:48:05 +0100
Subject: [PATCH 45/69] refactor(emqx_connector_ssl): simplify implementation

---
 .../emqx_connector/src/emqx_connector_ssl.erl | 48 +++++--------------
 1 file changed, 13 insertions(+), 35 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl
index 4c8197efa..1395ddf0d 100644
--- a/apps/emqx_connector/src/emqx_connector_ssl.erl
+++ b/apps/emqx_connector/src/emqx_connector_ssl.erl
@@ -30,12 +30,12 @@
 convert_certs(RltvDir, #{<<"connector">> := Connector} = Config) when
     is_map(Connector)
 ->
-    SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
+    SSL = maps:get(<<"ssl">>, Connector, undefined),
     new_ssl_config(RltvDir, Config, SSL);
 convert_certs(RltvDir, #{connector := Connector} = Config) when
     is_map(Connector)
 ->
-    SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
+    SSL = maps:get(ssl, Connector, undefined),
     new_ssl_config(RltvDir, Config, SSL);
 %% for bridges without `connector` field. i.e. webhook
 convert_certs(RltvDir, #{<<"ssl">> := SSL} = Config) ->
@@ -52,7 +52,10 @@ clear_certs(RltvDir, Config) ->
 clear_certs2(RltvDir, #{<<"connector">> := Connector} = _Config) when
     is_map(Connector)
 ->
-    OldSSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
+    %% TODO remove the 'connector' clause after dev/ee5.0 is merged back to master
+    %% The `connector` config layer will be removed.
+    %% for bridges with `connector` field. i.e. `mqtt_source` and `mqtt_sink`
+    OldSSL = maps:get(<<"ssl">>, Connector, undefined),
     ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
 clear_certs2(RltvDir, #{<<"ssl">> := OldSSL} = _Config) ->
     ok = emqx_tls_lib:delete_ssl_files(RltvDir, undefined, OldSSL);
@@ -66,14 +69,11 @@ try_clear_certs(RltvDir, NewConf, OldConf) ->
         normalize_key_to_bin(OldConf)
     ).
 
-try_clear_certs2(RltvDir, #{<<"connector">> := NewConnector}, #{<<"connector">> := OldConnector}) when
-    is_map(NewConnector),
-    is_map(OldConnector)
-->
-    NewSSL = map_get_oneof([<<"ssl">>, ssl], NewConnector, undefined),
-    OldSSL = map_get_oneof([<<"ssl">>, ssl], OldConnector, undefined),
-    ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL);
-try_clear_certs2(RltvDir, #{<<"ssl">> := NewSSL}, #{<<"ssl">> := OldSSL}) ->
+try_clear_certs2(RltvDir, #{<<"connector">> := NewConnector}, #{<<"connector">> := OldConnector}) ->
+    try_clear_certs2(RltvDir, NewConnector, OldConnector);
+try_clear_certs2(RltvDir, NewSSL, OldSSL) when is_map(NewSSL) andalso is_map(OldSSL) ->
+    NewSSL = maps:get(<<"ssl">>, NewSSL, undefined),
+    OldSSL = maps:get(<<"ssl">>, OldSSL, undefined),
     ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL);
 try_clear_certs2(RltvDir, NewConf, OldConf) ->
     ?SLOG(debug, #{msg => "unexpected_conf", path => RltvDir, new => NewConf, OldConf => OldConf}),
@@ -98,27 +98,5 @@ new_ssl_config(#{<<"ssl">> := _} = Config, NewSSL) ->
 new_ssl_config(Config, _NewSSL) ->
     Config.
 
-map_get_oneof([], _Map, Default) ->
-    Default;
-map_get_oneof([Key | Keys], Map, Default) ->
-    case maps:find(Key, Map) of
-        error ->
-            map_get_oneof(Keys, Map, Default);
-        {ok, Value} ->
-            Value
-    end.
-
-normalize_key_to_bin(Map) when is_map(Map) ->
-    maps:fold(
-        fun
-            (K, V, Acc) when is_atom(K) ->
-                Bin = erlang:atom_to_binary(K, utf8),
-                Acc#{Bin => V};
-            (K, V, Acc) ->
-                Acc#{K => V}
-        end,
-        #{},
-        Map
-    );
-normalize_key_to_bin(Any) ->
-    Any.
+normalize_key_to_bin(Map) ->
+    emqx_map_lib:binary_key_map(Map).

From fc2793c3006f203c5fa52aa6a1ac89e1135d5f0e Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 24 Nov 2022 10:37:52 +0800
Subject: [PATCH 46/69] fix: fix upside down function clause

---
 apps/emqx_connector/src/emqx_connector_ssl.erl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl
index 1395ddf0d..7dc6179e1 100644
--- a/apps/emqx_connector/src/emqx_connector_ssl.erl
+++ b/apps/emqx_connector/src/emqx_connector_ssl.erl
@@ -70,10 +70,10 @@ try_clear_certs(RltvDir, NewConf, OldConf) ->
     ).
 
 try_clear_certs2(RltvDir, #{<<"connector">> := NewConnector}, #{<<"connector">> := OldConnector}) ->
-    try_clear_certs2(RltvDir, NewConnector, OldConnector);
+    NewSSL = maps:get(<<"ssl">>, NewConnector, undefined),
+    OldSSL = maps:get(<<"ssl">>, OldConnector, undefined),
+    try_clear_certs2(RltvDir, NewSSL, OldSSL);
 try_clear_certs2(RltvDir, NewSSL, OldSSL) when is_map(NewSSL) andalso is_map(OldSSL) ->
-    NewSSL = maps:get(<<"ssl">>, NewSSL, undefined),
-    OldSSL = maps:get(<<"ssl">>, OldSSL, undefined),
     ok = emqx_tls_lib:delete_ssl_files(RltvDir, NewSSL, OldSSL);
 try_clear_certs2(RltvDir, NewConf, OldConf) ->
     ?SLOG(debug, #{msg => "unexpected_conf", path => RltvDir, new => NewConf, OldConf => OldConf}),

From 2bc8b004199a7a5fd2942fa70c331468728ef98f Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 24 Nov 2022 11:16:25 +0800
Subject: [PATCH 47/69] feat(authn): support quick deny anonymous

---
 apps/emqx/i18n/emqx_schema_i18n.conf          | 14 +++++--
 apps/emqx/src/emqx_access_control.erl         | 17 ++++++--
 apps/emqx/src/emqx_authentication.erl         | 25 ++++++++++--
 apps/emqx/src/emqx_schema.erl                 |  2 +-
 apps/emqx/test/emqx_access_control_SUITE.erl  | 39 ++++++++++++++++++-
 .../src/mqttsn/emqx_sn_channel.erl            |  3 +-
 changes/v5.0.11-en.md                         |  2 +
 changes/v5.0.11-zh.md                         |  2 +
 8 files changed, 89 insertions(+), 15 deletions(-)

diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf
index 714a08704..e7c4890f7 100644
--- a/apps/emqx/i18n/emqx_schema_i18n.conf
+++ b/apps/emqx/i18n/emqx_schema_i18n.conf
@@ -2045,12 +2045,18 @@ Type of the rate limit.
 base_listener_enable_authn {
     desc {
         en: """
-Set <code>true</code> (default) to enable client authentication on this listener.
-When set to <code>false</code> clients will be allowed to connect without authentication.
+Set <code>true</code> (default) to enable client authentication on this listener, the authentication
+process goes through the configured authentication chain.
+When set to <code>false</code> to allow any clients with or without authentication information such as username or password to log in.
+When set to <code>quick_deny_anonymous<code>, it behaves like when set to <code>true</code> but clients will be
+denied immediately without going through any authenticators if <code>username</code> is not provided. This is useful to fence off
+anonymous clients early.
 """
         zh: """
-配置 <code>true</code> （默认值）启用客户端进行身份认证。
-配置 <code>false</code> 时，将不对客户端做任何认证。
+配置 <code>true</code> （默认值）启用客户端进行身份认证，通过检查认配置的认认证器链来决定是否允许接入。
+配置 <code>false</code> 时，将不对客户端做任何认证，任何客户端，不论是不是携带用户名等认证信息，都可以接入。
+配置 <code>quick_deny_anonymous</code> 时，行为跟 <code>true</code> 类似，但是会对匿名
+客户直接拒绝，不做使用任何认证器对客户端进行身份检查。
 """
     }
     label: {
diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl
index d99699a9a..30d56f257 100644
--- a/apps/emqx/src/emqx_access_control.erl
+++ b/apps/emqx/src/emqx_access_control.erl
@@ -38,11 +38,22 @@
     | {ok, map(), binary()}
     | {continue, map()}
     | {continue, binary(), map()}
-    | {error, term()}.
+    | {error, not_authorized}.
 authenticate(Credential) ->
-    case run_hooks('client.authenticate', [Credential], {ok, #{is_superuser => false}}) of
+    %% pre-hook quick authentication or
+    %% if auth backend returning nothing but just 'ok'
+    %% it means it's not a superuser, or there is no way to tell.
+    NotSuperUser = #{is_superuser => false},
+    case emqx_authentication:pre_hook_authenticate(Credential) of
         ok ->
-            {ok, #{is_superuser => false}};
+            {ok, NotSuperUser};
+        continue ->
+            case run_hooks('client.authenticate', [Credential], {ok, #{is_superuser => false}}) of
+                ok ->
+                    {ok, NotSuperUser};
+                Other ->
+                    Other
+            end;
         Other ->
             Other
     end.
diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl
index 964a97dfb..749f5bfd7 100644
--- a/apps/emqx/src/emqx_authentication.erl
+++ b/apps/emqx/src/emqx_authentication.erl
@@ -29,9 +29,13 @@
 -include_lib("stdlib/include/ms_transform.hrl").
 
 -define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
+-define(IS_UNDEFINED(X), (X =:= undefined orelse X =:= <<>>)).
 
 %% The authentication entrypoint.
--export([authenticate/2]).
+-export([
+    pre_hook_authenticate/1,
+    authenticate/2
+]).
 
 %% Authenticator manager process start/stop
 -export([
@@ -221,10 +225,23 @@ when
 %%------------------------------------------------------------------------------
 %% Authenticate
 %%------------------------------------------------------------------------------
-
-authenticate(#{enable_authn := false}, _AuthResult) ->
+-spec pre_hook_authenticate(emqx_types:clientinfo()) ->
+    ok | continue | {error, not_authorized}.
+pre_hook_authenticate(#{enable_authn := false}) ->
     inc_authenticate_metric('authentication.success.anonymous'),
-    ?TRACE_RESULT("authentication_result", ignore, enable_authn_false);
+    ?TRACE_RESULT("authentication_result", ok, enable_authn_false);
+pre_hook_authenticate(#{enable_authn := quick_deny_anonymous} = Credential) ->
+    case maps:get(username, Credential, undefined) of
+        U when ?IS_UNDEFINED(U) ->
+            ?TRACE_RESULT(
+                "authentication_result", {error, not_authorized}, enable_authn_false
+            );
+        _ ->
+            continue
+    end;
+pre_hook_authenticate(_) ->
+    continue.
+
 authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthResult) ->
     case get_authenticators(Listener, global_chain(Protocol)) of
         {ok, ChainName, Authenticators} ->
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 1024cef48..eb5238cfe 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -1668,7 +1668,7 @@ base_listener(Bind) ->
             )},
         {"enable_authn",
             sc(
-                boolean(),
+                hoconsc:enum([true, false, quick_deny_anonymous]),
                 #{
                     desc => ?DESC(base_listener_enable_authn),
                     default => true
diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl
index 7b6b4f463..c079ac125 100644
--- a/apps/emqx/test/emqx_access_control_SUITE.erl
+++ b/apps/emqx/test/emqx_access_control_SUITE.erl
@@ -37,7 +37,8 @@ init_per_testcase(_, Config) ->
     Config.
 
 end_per_testcase(_, _Config) ->
-    ok = emqx_hooks:del('client.authorize', {?MODULE, authz_stub}).
+    ok = emqx_hooks:del('client.authorize', {?MODULE, authz_stub}),
+    ok = emqx_hooks:del('client.authenticate', {?MODULE, quick_deny_anonymous_authn}).
 
 t_authenticate(_) ->
     ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())).
@@ -60,6 +61,37 @@ t_delayed_authorize(_) ->
     ?assertEqual(deny, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)),
     ok.
 
+t_quick_deny_anonymous(_) ->
+    ok = emqx_hooks:put(
+        'client.authenticate',
+        {?MODULE, quick_deny_anonymous_authn, []},
+        ?HP_AUTHN
+    ),
+
+    RawClient0 = clientinfo(),
+    RawClient = RawClient0#{username => undefined},
+
+    %% No name, No authn
+    Client1 = RawClient#{enable_authn => false},
+    ?assertMatch({ok, _}, emqx_access_control:authenticate(Client1)),
+
+    %% No name, With quick_deny_anonymous
+    Client2 = RawClient#{enable_authn => quick_deny_anonymous},
+    ?assertMatch({error, _}, emqx_access_control:authenticate(Client2)),
+
+    %% Bad name, With quick_deny_anonymous
+    Client3 = RawClient#{enable_authn => quick_deny_anonymous, username => <<"badname">>},
+    ?assertMatch({error, _}, emqx_access_control:authenticate(Client3)),
+
+    %% Good name, With quick_deny_anonymous
+    Client4 = RawClient#{enable_authn => quick_deny_anonymous, username => <<"goodname">>},
+    ?assertMatch({ok, _}, emqx_access_control:authenticate(Client4)),
+
+    %% Name, With authn
+    Client5 = RawClient#{enable_authn => true, username => <<"badname">>},
+    ?assertMatch({error, _}, emqx_access_control:authenticate(Client5)),
+    ok.
+
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------
@@ -67,6 +99,11 @@ t_delayed_authorize(_) ->
 authz_stub(_Client, _PubSub, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}};
 authz_stub(_Client, _PubSub, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}.
 
+quick_deny_anonymous_authn(#{username := <<"badname">>}, _AuthResult) ->
+    {stop, {error, not_authorized}};
+quick_deny_anonymous_authn(_ClientInfo, _AuthResult) ->
+    {stop, {ok, #{is_superuser => false}}}.
+
 clientinfo() -> clientinfo(#{}).
 clientinfo(InitProps) ->
     maps:merge(
diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
index b5e051193..df3b4018e 100644
--- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
+++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl
@@ -2319,5 +2319,4 @@ returncode_name(?SN_RC2_EXCEED_LIMITATION) -> rejected_exceed_limitation;
 returncode_name(?SN_RC2_REACHED_MAX_RETRY) -> reached_max_retry_times;
 returncode_name(_) -> accepted.
 
-name_to_returncode(not_authorized) -> ?SN_RC2_NOT_AUTHORIZE;
-name_to_returncode(_) -> ?SN_RC2_NOT_AUTHORIZE.
+name_to_returncode(not_authorized) -> ?SN_RC2_NOT_AUTHORIZE.
diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index e53c5785e..6ad40c7fe 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -23,6 +23,8 @@
 
 - Keep MQTT v5 User-Property pairs from bridge ingested MQTT messsages to bridge target [#9398](https://github.com/emqx/emqx/pull/9398).
 
+- Add a new config `quick_deny_anonymous` to allow quick deny of anonymous clients (without username) so the auth backend checks can be skipped [#8516](https://github.com/emqx/emqx/pull/8516).
+
 ## Bug fixes
 
 - Fix `ssl.existingName` option of  helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index 3ea516dad..b5fe7c4bd 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -21,6 +21,8 @@
 
 - 为桥接收到的 MQTT v5 消息再转发时保留 User-Property 列表 [#9398](https://github.com/emqx/emqx/pull/9398)。
 
+- 添加了一个名为 `quick_deny_anonymous` 的新配置，用来在不调用认证链的情况下，快速的拒绝掉匿名用户，从而提高认证效率 [#8516](https://github.com/emqx/emqx/pull/8516)。
+
 ## 修复
 
 - 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。

From 17ec202b57ac2016b93a3d17b5bea8321c02f952 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 24 Nov 2022 11:01:17 +0100
Subject: [PATCH 48/69] chore: upgrade to dashboard v1.1.2

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 246a2d522..02bcbf458 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.1.1
+export EMQX_DASHBOARD_VERSION ?= v1.1.2
 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1

From 08121e7df6cb98b1fa60c40f35f86a56d6b3b929 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 10 Nov 2022 13:45:47 +0800
Subject: [PATCH 49/69] fix(mgmt): optimize the speed of query tail pages

In the previous, when you query the tail pages, all the front of rows
will be queried out and formatted. It greatly hurts the speed of query.

Currently, we only format the final result rows. i.e, the query for the
last page of data will be 10x faster.
---
 apps/emqx/src/emqx_alarm.erl                  |  14 +-
 .../emqx_enhanced_authn_scram_mnesia.erl      |  15 +-
 .../src/simple_authn/emqx_authn_mnesia.erl    |  15 +-
 apps/emqx_authz/src/emqx_authz_api_mnesia.erl |  18 +-
 .../src/emqx_gateway_api_clients.erl          |  22 +-
 apps/emqx_management/src/emqx_mgmt_api.erl    | 239 ++++++++++--------
 .../src/emqx_mgmt_api_alarms.erl              |  20 +-
 .../src/emqx_mgmt_api_clients.erl             |  26 +-
 .../src/emqx_mgmt_api_subscriptions.erl       |  22 +-
 .../src/emqx_mgmt_api_topics.erl              |   7 +-
 apps/emqx_modules/src/emqx_delayed.erl        |   7 +-
 .../src/emqx_rule_engine_api.erl              |   5 +-
 12 files changed, 227 insertions(+), 183 deletions(-)

diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl
index a89063870..7368b442d 100644
--- a/apps/emqx/src/emqx_alarm.erl
+++ b/apps/emqx/src/emqx_alarm.erl
@@ -41,7 +41,8 @@
     delete_all_deactivated_alarms/0,
     get_alarms/0,
     get_alarms/1,
-    format/1
+    format/1,
+    format/2
 ]).
 
 %% gen_server callbacks
@@ -169,12 +170,15 @@ get_alarms(activated) ->
 get_alarms(deactivated) ->
     gen_server:call(?MODULE, {get_alarms, deactivated}).
 
-format(#activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) ->
+format(Alarm) ->
+    format(node(), Alarm).
+
+format(Node, #activated_alarm{name = Name, message = Message, activate_at = At, details = Details}) ->
     Now = erlang:system_time(microsecond),
     %% mnesia db stored microsecond for high frequency alarm
     %% format for dashboard using millisecond
     #{
-        node => node(),
+        node => Node,
         name => Name,
         message => Message,
         %% to millisecond
@@ -182,7 +186,7 @@ format(#activated_alarm{name = Name, message = Message, activate_at = At, detail
         activate_at => to_rfc3339(At),
         details => Details
     };
-format(#deactivated_alarm{
+format(Node, #deactivated_alarm{
     name = Name,
     message = Message,
     activate_at = At,
@@ -190,7 +194,7 @@ format(#deactivated_alarm{
     deactivate_at = DAt
 }) ->
     #{
-        node => node(),
+        node => Node,
         name => Name,
         message => Message,
         %% to millisecond
diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
index cb3fad7dc..1fcc00dc9 100644
--- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
+++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
@@ -262,7 +262,14 @@ lookup_user(UserID, #{user_group := UserGroup}) ->
 
 list_users(QueryString, #{user_group := UserGroup}) ->
     NQueryString = QueryString#{<<"user_group">> => UserGroup},
-    emqx_mgmt_api:node_query(node(), NQueryString, ?TAB, ?AUTHN_QSCHEMA, ?QUERY_FUN).
+    emqx_mgmt_api:node_query(
+        node(),
+        NQueryString,
+        ?TAB,
+        ?AUTHN_QSCHEMA,
+        ?QUERY_FUN,
+        fun ?MODULE:format_user_info/1
+    ).
 
 %%--------------------------------------------------------------------
 %% Query Functions
@@ -273,8 +280,7 @@ query(Tab, {QString, []}, Continuation, Limit) ->
         Tab,
         Ms,
         Continuation,
-        Limit,
-        fun format_user_info/1
+        Limit
     );
 query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
     Ms = ms_from_qstring(QString),
@@ -283,8 +289,7 @@ query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
         Tab,
         {Ms, FuzzyFilterFun},
         Continuation,
-        Limit,
-        fun format_user_info/1
+        Limit
     ).
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
index 7276ad428..f84bfc195 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
@@ -288,7 +288,14 @@ lookup_user(UserID, #{user_group := UserGroup}) ->
 
 list_users(QueryString, #{user_group := UserGroup}) ->
     NQueryString = QueryString#{<<"user_group">> => UserGroup},
-    emqx_mgmt_api:node_query(node(), NQueryString, ?TAB, ?AUTHN_QSCHEMA, ?QUERY_FUN).
+    emqx_mgmt_api:node_query(
+        node(),
+        NQueryString,
+        ?TAB,
+        ?AUTHN_QSCHEMA,
+        ?QUERY_FUN,
+        fun ?MODULE:format_user_info/1
+    ).
 
 %%--------------------------------------------------------------------
 %% Query Functions
@@ -299,8 +306,7 @@ query(Tab, {QString, []}, Continuation, Limit) ->
         Tab,
         Ms,
         Continuation,
-        Limit,
-        fun format_user_info/1
+        Limit
     );
 query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
     Ms = ms_from_qstring(QString),
@@ -309,8 +315,7 @@ query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
         Tab,
         {Ms, FuzzyFilterFun},
         Continuation,
-        Limit,
-        fun format_user_info/1
+        Limit
     ).
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
index 609974e98..8edd62e21 100644
--- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
@@ -408,7 +408,8 @@ users(get, #{query_string := QueryString}) ->
             QueryString,
             ?ACL_TABLE,
             ?ACL_USERNAME_QSCHEMA,
-            ?QUERY_USERNAME_FUN
+            ?QUERY_USERNAME_FUN,
+            fun ?MODULE:format_result/1
         )
     of
         {error, page_limit_invalid} ->
@@ -443,7 +444,8 @@ clients(get, #{query_string := QueryString}) ->
             QueryString,
             ?ACL_TABLE,
             ?ACL_CLIENTID_QSCHEMA,
-            ?QUERY_CLIENTID_FUN
+            ?QUERY_CLIENTID_FUN,
+            fun ?MODULE:format_result/1
         )
     of
         {error, page_limit_invalid} ->
@@ -582,8 +584,7 @@ query_username(Tab, {_QString, []}, Continuation, Limit) ->
         Tab,
         Ms,
         Continuation,
-        Limit,
-        fun format_result/1
+        Limit
     );
 query_username(Tab, {_QString, FuzzyQString}, Continuation, Limit) ->
     Ms = emqx_authz_mnesia:list_username_rules(),
@@ -592,8 +593,7 @@ query_username(Tab, {_QString, FuzzyQString}, Continuation, Limit) ->
         Tab,
         {Ms, FuzzyFilterFun},
         Continuation,
-        Limit,
-        fun format_result/1
+        Limit
     ).
 
 query_clientid(Tab, {_QString, []}, Continuation, Limit) ->
@@ -602,8 +602,7 @@ query_clientid(Tab, {_QString, []}, Continuation, Limit) ->
         Tab,
         Ms,
         Continuation,
-        Limit,
-        fun format_result/1
+        Limit
     );
 query_clientid(Tab, {_QString, FuzzyQString}, Continuation, Limit) ->
     Ms = emqx_authz_mnesia:list_clientid_rules(),
@@ -612,8 +611,7 @@ query_clientid(Tab, {_QString, FuzzyQString}, Continuation, Limit) ->
         Tab,
         {Ms, FuzzyFilterFun},
         Continuation,
-        Limit,
-        fun format_result/1
+        Limit
     ).
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
index 5f6cc25b6..f0fbcf968 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
@@ -56,7 +56,8 @@
 %% internal exports (for client query)
 -export([
     query/4,
-    format_channel_info/1
+    format_channel_info/1,
+    format_channel_info/2
 ]).
 
 -define(TAGS, [<<"Gateway Clients">>]).
@@ -112,7 +113,8 @@ clients(get, #{
                         QString,
                         TabName,
                         ?CLIENT_QSCHEMA,
-                        ?QUERY_FUN
+                        ?QUERY_FUN,
+                        fun ?MODULE:format_channel_info/2
                     );
                 Node0 ->
                     case emqx_misc:safe_to_existing_atom(Node0) of
@@ -123,7 +125,8 @@ clients(get, #{
                                 QStringWithoutNode,
                                 TabName,
                                 ?CLIENT_QSCHEMA,
-                                ?QUERY_FUN
+                                ?QUERY_FUN,
+                                fun ?MODULE:format_channel_info/2
                             );
                         {error, _} ->
                             {error, Node0, {badrpc, <<"invalid node">>}}
@@ -272,8 +275,7 @@ query(Tab, {Qs, []}, Continuation, Limit) ->
         Tab,
         Ms,
         Continuation,
-        Limit,
-        fun format_channel_info/1
+        Limit
     );
 query(Tab, {Qs, Fuzzy}, Continuation, Limit) ->
     Ms = qs2ms(Qs),
@@ -282,8 +284,7 @@ query(Tab, {Qs, Fuzzy}, Continuation, Limit) ->
         Tab,
         {Ms, FuzzyFilterFun},
         Continuation,
-        Limit,
-        fun format_channel_info/1
+        Limit
     ).
 
 qs2ms(Qs) ->
@@ -363,8 +364,11 @@ run_fuzzy_filter(
 %%--------------------------------------------------------------------
 %% format funcs
 
-format_channel_info({_, Infos, Stats} = R) ->
-    Node = maps:get(node, Infos, node()),
+format_channel_info(ChannInfo) ->
+    format_channel_info(node(), ChannInfo).
+
+format_channel_info(WhichNode, {_, Infos, Stats} = R) ->
+    Node = maps:get(node, Infos, WhichNode),
     ClientInfo = maps:get(clientinfo, Infos, #{}),
     ConnInfo = maps:get(conninfo, Infos, #{}),
     SessInfo = maps:get(session, Infos, #{}),
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 342228b19..cc0a81864 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -29,9 +29,9 @@
 
 %% first_next query APIs
 -export([
-    node_query/5,
-    cluster_query/4,
-    select_table_with_count/5,
+    node_query/6,
+    cluster_query/5,
+    select_table_with_count/4,
     b2i/1
 ]).
 
@@ -117,30 +117,24 @@ limit(Params) when is_map(Params) ->
 limit(Params) ->
     proplists:get_value(<<"limit">>, Params, emqx_mgmt:max_row_limit()).
 
-init_meta(Params) ->
-    Limit = b2i(limit(Params)),
-    Page = b2i(page(Params)),
-    #{
-        page => Page,
-        limit => Limit,
-        count => 0
-    }.
-
 %%--------------------------------------------------------------------
 %% Node Query
 %%--------------------------------------------------------------------
 
-node_query(Node, QString, Tab, QSchema, QueryFun) ->
-    {_CodCnt, NQString} = parse_qstring(QString, QSchema),
-    page_limit_check_query(
-        init_meta(QString),
-        {fun do_node_query/5, [Node, Tab, NQString, QueryFun, init_meta(QString)]}
-    ).
+node_query(Node, QString, Tab, QSchema, QueryFun, FmtFun) ->
+    case parse_pager_params(QString) of
+        false ->
+            {error, page_limit_invalid};
+        Meta ->
+            {_CodCnt, NQString} = parse_qstring(QString, QSchema),
+            ResultAcc = #{cursor => 0, count => 0, rows => []},
+            NResultAcc = do_node_query(
+                Node, Tab, NQString, QueryFun, ?FRESH_SELECT, Meta, ResultAcc
+            ),
+            format_query_result(FmtFun, Meta, NResultAcc)
+    end.
 
 %% @private
-do_node_query(Node, Tab, QString, QueryFun, Meta) ->
-    do_node_query(Node, Tab, QString, QueryFun, _Continuation = ?FRESH_SELECT, Meta, _Results = []).
-
 do_node_query(
     Node,
     Tab,
@@ -148,45 +142,44 @@ do_node_query(
     QueryFun,
     Continuation,
     Meta = #{limit := Limit},
-    Results
+    ResultAcc
 ) ->
     case do_query(Node, Tab, QString, QueryFun, Continuation, Limit) of
         {error, {badrpc, R}} ->
             {error, Node, {badrpc, R}};
-        {Len, Rows, ?FRESH_SELECT} ->
-            {NMeta, NResults} = sub_query_result(Len, Rows, Limit, Results, Meta),
-            #{meta => NMeta, data => NResults};
-        {Len, Rows, NContinuation} ->
-            {NMeta, NResults} = sub_query_result(Len, Rows, Limit, Results, Meta),
-            do_node_query(Node, Tab, QString, QueryFun, NContinuation, NMeta, NResults)
+        {Rows, ?FRESH_SELECT} ->
+            {_, NResultAcc} = accumulate_query_rows(Node, Rows, ResultAcc, Meta),
+            NResultAcc;
+        {Rows, NContinuation} ->
+            case accumulate_query_rows(Node, Rows, ResultAcc, Meta) of
+                {enough, NResultAcc} ->
+                    NResultAcc;
+                {more, NResultAcc} ->
+                    do_node_query(Node, Tab, QString, QueryFun, NContinuation, Meta, NResultAcc)
+            end
     end.
 
 %%--------------------------------------------------------------------
 %% Cluster Query
 %%--------------------------------------------------------------------
 
-cluster_query(QString, Tab, QSchema, QueryFun) ->
-    {_CodCnt, NQString} = parse_qstring(QString, QSchema),
-    Nodes = mria_mnesia:running_nodes(),
-    page_limit_check_query(
-        init_meta(QString),
-        {fun do_cluster_query/5, [Nodes, Tab, NQString, QueryFun, init_meta(QString)]}
-    ).
+cluster_query(QString, Tab, QSchema, QueryFun, FmtFun) ->
+    case parse_pager_params(QString) of
+        false ->
+            {error, page_limit_invalid};
+        Meta ->
+            {_CodCnt, NQString} = parse_qstring(QString, QSchema),
+            Nodes = mria_mnesia:running_nodes(),
+            ResultAcc = #{cursor => 0, count => 0, rows => []},
+            NResultAcc = do_cluster_query(
+                Nodes, Tab, NQString, QueryFun, ?FRESH_SELECT, Meta, ResultAcc
+            ),
+            format_query_result(FmtFun, Meta, NResultAcc)
+    end.
 
 %% @private
-do_cluster_query(Nodes, Tab, QString, QueryFun, Meta) ->
-    do_cluster_query(
-        Nodes,
-        Tab,
-        QString,
-        QueryFun,
-        _Continuation = ?FRESH_SELECT,
-        Meta,
-        _Results = []
-    ).
-
-do_cluster_query([], _Tab, _QString, _QueryFun, _Continuation, Meta, Results) ->
-    #{meta => Meta, data => Results};
+do_cluster_query([], _Tab, _QString, _QueryFun, _Continuation, _Meta, ResultAcc) ->
+    ResultAcc;
 do_cluster_query(
     [Node | Tail] = Nodes,
     Tab,
@@ -194,17 +187,27 @@ do_cluster_query(
     QueryFun,
     Continuation,
     Meta = #{limit := Limit},
-    Results
+    ResultAcc
 ) ->
     case do_query(Node, Tab, QString, QueryFun, Continuation, Limit) of
         {error, {badrpc, R}} ->
-            {error, Node, {bar_rpc, R}};
-        {Len, Rows, ?FRESH_SELECT} ->
-            {NMeta, NResults} = sub_query_result(Len, Rows, Limit, Results, Meta),
-            do_cluster_query(Tail, Tab, QString, QueryFun, ?FRESH_SELECT, NMeta, NResults);
-        {Len, Rows, NContinuation} ->
-            {NMeta, NResults} = sub_query_result(Len, Rows, Limit, Results, Meta),
-            do_cluster_query(Nodes, Tab, QString, QueryFun, NContinuation, NMeta, NResults)
+            {error, Node, {badrpc, R}};
+        {Rows, NContinuation} ->
+            case accumulate_query_rows(Node, Rows, ResultAcc, Meta) of
+                {enough, NResultAcc} ->
+                    NResultAcc;
+                {more, NResultAcc} ->
+                    case NContinuation of
+                        ?FRESH_SELECT ->
+                            do_cluster_query(
+                                Tail, Tab, QString, QueryFun, ?FRESH_SELECT, Meta, NResultAcc
+                            );
+                        _ ->
+                            do_cluster_query(
+                                Nodes, Tab, QString, QueryFun, NContinuation, Meta, NResultAcc
+                            )
+                    end
+            end
     end.
 
 %%--------------------------------------------------------------------
@@ -228,60 +231,76 @@ do_query(Node, Tab, QString, QueryFun, Continuation, Limit) ->
         Ret -> Ret
     end.
 
-sub_query_result(Len, Rows, Limit, Results, Meta) ->
-    {Flag, NMeta} = judge_page_with_counting(Len, Meta),
-    NResults =
-        case Flag of
-            more ->
-                [];
-            cutrows ->
-                {SubStart, NeedNowNum} = rows_sub_params(Len, NMeta),
-                ThisRows = lists:sublist(Rows, SubStart, NeedNowNum),
-                lists:sublist(lists:append(Results, ThisRows), SubStart, Limit);
-            enough ->
-                lists:sublist(lists:append(Results, Rows), 1, Limit)
-        end,
-    {NMeta, NResults}.
+%% ResultAcc :: #{count := integer(),
+%%                cursor := integer(),
+%%                rows  := [{node(), Rows :: list()}]
+%%               }
+accumulate_query_rows(
+    Node,
+    Rows,
+    ResultAcc = #{cursor := Cursor, count := Count, rows := RowsAcc},
+    _Meta = #{page := Page, limit := Limit}
+) ->
+    PageStart = (Page - 1) * Limit + 1,
+    PageEnd = Page * Limit,
+    Len = length(Rows),
+    case Cursor + Len of
+        NCursor when NCursor < PageStart ->
+            {more, ResultAcc#{cursor => NCursor}};
+        NCursor when NCursor < PageEnd ->
+            {more, ResultAcc#{
+                cursor => NCursor,
+                count => Count + length(Rows),
+                rows => [{Node, Rows} | RowsAcc]
+            }};
+        NCursor when NCursor >= PageEnd ->
+            SubRows = lists:sublist(Rows, Limit - Count),
+            {enough, ResultAcc#{
+                cursor => NCursor,
+                count => Count + length(SubRows),
+                rows => [{Node, SubRows} | RowsAcc]
+            }}
+    end.
 
 %%--------------------------------------------------------------------
 %% Table Select
 %%--------------------------------------------------------------------
 
-select_table_with_count(Tab, {Ms, FuzzyFilterFun}, ?FRESH_SELECT, Limit, FmtFun) when
+select_table_with_count(Tab, {Ms, FuzzyFilterFun}, ?FRESH_SELECT, Limit) when
     is_function(FuzzyFilterFun) andalso Limit > 0
 ->
     case ets:select(Tab, Ms, Limit) of
         '$end_of_table' ->
-            {0, [], ?FRESH_SELECT};
+            {[], ?FRESH_SELECT};
         {RawResult, NContinuation} ->
             Rows = FuzzyFilterFun(RawResult),
-            {length(Rows), lists:map(FmtFun, Rows), NContinuation}
+            {Rows, NContinuation}
     end;
-select_table_with_count(_Tab, {Ms, FuzzyFilterFun}, Continuation, _Limit, FmtFun) when
+select_table_with_count(_Tab, {Ms, FuzzyFilterFun}, Continuation, _Limit) when
     is_function(FuzzyFilterFun)
 ->
     case ets:select(ets:repair_continuation(Continuation, Ms)) of
         '$end_of_table' ->
-            {0, [], ?FRESH_SELECT};
+            {[], ?FRESH_SELECT};
         {RawResult, NContinuation} ->
             Rows = FuzzyFilterFun(RawResult),
-            {length(Rows), lists:map(FmtFun, Rows), NContinuation}
+            {Rows, NContinuation}
     end;
-select_table_with_count(Tab, Ms, ?FRESH_SELECT, Limit, FmtFun) when
+select_table_with_count(Tab, Ms, ?FRESH_SELECT, Limit) when
     Limit > 0
 ->
     case ets:select(Tab, Ms, Limit) of
         '$end_of_table' ->
-            {0, [], ?FRESH_SELECT};
+            {[], ?FRESH_SELECT};
         {RawResult, NContinuation} ->
-            {length(RawResult), lists:map(FmtFun, RawResult), NContinuation}
+            {RawResult, NContinuation}
     end;
-select_table_with_count(_Tab, Ms, Continuation, _Limit, FmtFun) ->
+select_table_with_count(_Tab, Ms, Continuation, _Limit) ->
     case ets:select(ets:repair_continuation(Continuation, Ms)) of
         '$end_of_table' ->
-            {0, [], ?FRESH_SELECT};
+            {[], ?FRESH_SELECT};
         {RawResult, NContinuation} ->
-            {length(RawResult), lists:map(FmtFun, RawResult), NContinuation}
+            {RawResult, NContinuation}
     end.
 
 %%--------------------------------------------------------------------
@@ -379,40 +398,38 @@ is_fuzzy_key(<<"match_", _/binary>>) ->
 is_fuzzy_key(_) ->
     false.
 
-page_start(1, _) -> 1;
-page_start(Page, Limit) -> (Page - 1) * Limit + 1.
+format_query_result(_FmtFun, _Meta, Error = {error, _Node, _Reason}) ->
+    Error;
+format_query_result(
+    FmtFun, Meta, _ResultAcc = #{count := _Count, cursor := Cursor, rows := RowsAcc}
+) ->
+    #{
+        meta => Meta#{count => Cursor},
+        data => lists:flatten(
+            lists:foldr(
+                fun({Node, Rows}, Acc) ->
+                    [lists:map(fun(Row) -> exec_format_fun(FmtFun, Node, Row) end, Rows) | Acc]
+                end,
+                [],
+                RowsAcc
+            )
+        )
+    }.
 
-judge_page_with_counting(Len, Meta = #{page := Page, limit := Limit, count := Count}) ->
-    PageStart = page_start(Page, Limit),
-    PageEnd = Page * Limit,
-    case Count + Len of
-        NCount when NCount < PageStart ->
-            {more, Meta#{count => NCount}};
-        NCount when NCount < PageEnd ->
-            {cutrows, Meta#{count => NCount}};
-        NCount when NCount >= PageEnd ->
-            {enough, Meta#{count => NCount}}
+exec_format_fun(FmtFun, Node, Row) ->
+    case erlang:fun_info(FmtFun, arity) of
+        {arity, 1} -> FmtFun(Row);
+        {arity, 2} -> FmtFun(Node, Row)
     end.
 
-rows_sub_params(Len, _Meta = #{page := Page, limit := Limit, count := Count}) ->
-    PageStart = page_start(Page, Limit),
-    case (Count - Len) < PageStart of
+parse_pager_params(Params) ->
+    Page = b2i(page(Params)),
+    Limit = b2i(limit(Params)),
+    case Page > 0 andalso Limit > 0 of
         true ->
-            NeedNowNum = Count - PageStart + 1,
-            SubStart = Len - NeedNowNum + 1,
-            {SubStart, NeedNowNum};
+            #{page => Page, limit => Limit, count => 0};
         false ->
-            {_SubStart = 1, _NeedNowNum = Len}
-    end.
-
-page_limit_check_query(Meta, {F, A}) ->
-    case Meta of
-        #{page := Page, limit := Limit} when
-            Page < 1; Limit < 1
-        ->
-            {error, page_limit_invalid};
-        _ ->
-            erlang:apply(F, A)
+            false
     end.
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl
index 36845e4e7..d574ad4ee 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl
@@ -24,7 +24,7 @@
 
 -export([api_spec/0, paths/0, schema/1, fields/1]).
 
--export([alarms/2]).
+-export([alarms/2, format_alarm/2]).
 
 -define(TAGS, [<<"Alarms">>]).
 
@@ -112,7 +112,15 @@ alarms(get, #{query_string := QString}) ->
             true -> ?ACTIVATED_ALARM;
             false -> ?DEACTIVATED_ALARM
         end,
-    case emqx_mgmt_api:cluster_query(QString, Table, [], {?MODULE, query}) of
+    case
+        emqx_mgmt_api:cluster_query(
+            QString,
+            Table,
+            [],
+            {?MODULE, query},
+            fun ?MODULE:format_alarm/2
+        )
+    of
         {error, page_limit_invalid} ->
             {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}};
         {error, Node, {badrpc, R}} ->
@@ -130,9 +138,7 @@ alarms(delete, _Params) ->
 
 query(Table, _QsSpec, Continuation, Limit) ->
     Ms = [{'$1', [], ['$1']}],
-    emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit, fun format_alarm/1).
+    emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit).
 
-format_alarm(Alarms) when is_list(Alarms) ->
-    [emqx_alarm:format(Alarm) || Alarm <- Alarms];
-format_alarm(Alarm) ->
-    emqx_alarm:format(Alarm).
+format_alarm(WhichNode, Alarm) ->
+    emqx_alarm:format(WhichNode, Alarm).
diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index f4fe0387f..56730fe3e 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -47,7 +47,8 @@
 
 -export([
     query/4,
-    format_channel_info/1
+    format_channel_info/1,
+    format_channel_info/2
 ]).
 
 %% for batch operation
@@ -645,7 +646,8 @@ list_clients(QString) ->
                     QString,
                     ?CLIENT_QTAB,
                     ?CLIENT_QSCHEMA,
-                    ?QUERY_FUN
+                    ?QUERY_FUN,
+                    fun ?MODULE:format_channel_info/2
                 );
             Node0 ->
                 case emqx_misc:safe_to_existing_atom(Node0) of
@@ -656,7 +658,8 @@ list_clients(QString) ->
                             QStringWithoutNode,
                             ?CLIENT_QTAB,
                             ?CLIENT_QSCHEMA,
-                            ?QUERY_FUN
+                            ?QUERY_FUN,
+                            fun ?MODULE:format_channel_info/2
                         );
                     {error, _} ->
                         {error, Node0, {badrpc, <<"invalid node">>}}
@@ -789,8 +792,7 @@ query(Tab, {QString, []}, Continuation, Limit) ->
         Tab,
         Ms,
         Continuation,
-        Limit,
-        fun format_channel_info/1
+        Limit
     );
 query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
     Ms = qs2ms(QString),
@@ -799,8 +801,7 @@ query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
         Tab,
         {Ms, FuzzyFilterFun},
         Continuation,
-        Limit,
-        fun format_channel_info/1
+        Limit
     ).
 
 %%--------------------------------------------------------------------
@@ -876,12 +877,11 @@ run_fuzzy_filter(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr} |
 %%--------------------------------------------------------------------
 %% format funcs
 
-format_channel_info({_, ClientInfo0, ClientStats}) ->
-    Node =
-        case ClientInfo0 of
-            #{node := N} -> N;
-            _ -> node()
-        end,
+format_channel_info(ChannInfo = {_, _ClientInfo, _ClientStats}) ->
+    format_channel_info(node(), ChannInfo).
+
+format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}) ->
+    Node = maps:get(node, ClientInfo0, WhichNode),
     ClientInfo1 = emqx_map_lib:deep_remove([conninfo, clientid], ClientInfo0),
     ClientInfo2 = emqx_map_lib:deep_remove([conninfo, username], ClientInfo1),
     StatsMap = maps:without(
diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
index 03b833e84..eb0f83cc0 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
@@ -33,7 +33,7 @@
 
 -export([
     query/4,
-    format/1
+    format/2
 ]).
 
 -define(SUBS_QTABLE, emqx_suboption).
@@ -142,7 +142,8 @@ subscriptions(get, #{query_string := QString}) ->
                     QString,
                     ?SUBS_QTABLE,
                     ?SUBS_QSCHEMA,
-                    ?QUERY_FUN
+                    ?QUERY_FUN,
+                    fun ?MODULE:format/2
                 );
             Node0 ->
                 case emqx_misc:safe_to_existing_atom(Node0) of
@@ -152,7 +153,8 @@ subscriptions(get, #{query_string := QString}) ->
                             QString,
                             ?SUBS_QTABLE,
                             ?SUBS_QSCHEMA,
-                            ?QUERY_FUN
+                            ?QUERY_FUN,
+                            fun ?MODULE:format/2
                         );
                     {error, _} ->
                         {error, Node0, {badrpc, <<"invalid node">>}}
@@ -168,16 +170,12 @@ subscriptions(get, #{query_string := QString}) ->
             {200, Result}
     end.
 
-format(Items) when is_list(Items) ->
-    [format(Item) || Item <- Items];
-format({{Subscriber, Topic}, Options}) ->
-    format({Subscriber, Topic, Options});
-format({_Subscriber, Topic, Options}) ->
+format(WhichNode, {{_Subscriber, Topic}, Options}) ->
     maps:merge(
         #{
             topic => get_topic(Topic, Options),
             clientid => maps:get(subid, Options),
-            node => node()
+            node => WhichNode
         },
         maps:with([qos, nl, rap, rh], Options)
     ).
@@ -199,8 +197,7 @@ query(Tab, {Qs, []}, Continuation, Limit) ->
         Tab,
         Ms,
         Continuation,
-        Limit,
-        fun format/1
+        Limit
     );
 query(Tab, {Qs, Fuzzy}, Continuation, Limit) ->
     Ms = qs2ms(Qs),
@@ -209,8 +206,7 @@ query(Tab, {Qs, Fuzzy}, Continuation, Limit) ->
         Tab,
         {Ms, FuzzyFilterFun},
         Continuation,
-        Limit,
-        fun format/1
+        Limit
     ).
 
 fuzzy_filter_fun(Fuzzy) ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl
index c357855b2..3cc2168e7 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl
@@ -109,7 +109,12 @@ topic(get, #{bindings := Bindings}) ->
 do_list(Params) ->
     case
         emqx_mgmt_api:node_query(
-            node(), Params, emqx_route, ?TOPICS_QUERY_SCHEMA, {?MODULE, query}
+            node(),
+            Params,
+            emqx_route,
+            ?TOPICS_QUERY_SCHEMA,
+            {?MODULE, query},
+            fun ?MODULE:format/1
         )
     of
         {error, page_limit_invalid} ->
diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl
index f511d74d9..44e7a85e3 100644
--- a/apps/emqx_modules/src/emqx_delayed.erl
+++ b/apps/emqx_modules/src/emqx_delayed.erl
@@ -166,11 +166,14 @@ list(Params) ->
     emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN).
 
 cluster_list(Params) ->
-    emqx_mgmt_api:cluster_query(Params, ?TAB, [], {?MODULE, cluster_query}).
+    %% FIXME: why cluster_query???
+    emqx_mgmt_api:cluster_query(
+        Params, ?TAB, [], {?MODULE, cluster_query}, fun ?MODULE:format_delayed/1
+    ).
 
 cluster_query(Table, _QsSpec, Continuation, Limit) ->
     Ms = [{'$1', [], ['$1']}],
-    emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit, fun format_delayed/1).
+    emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit).
 
 format_delayed(Delayed) ->
     format_delayed(Delayed, false).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index 597ee838f..2157b108c 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -277,7 +277,8 @@ param_path_id() ->
             QueryString,
             ?RULE_TAB,
             ?RULE_QS_SCHEMA,
-            {?MODULE, query}
+            {?MODULE, query},
+            fun ?MODULE:format_rule_resp/1
         )
     of
         {error, page_limit_invalid} ->
@@ -556,7 +557,7 @@ query(Tab, {Qs, Fuzzy}, Start, Limit) ->
     Ms = qs2ms(),
     FuzzyFun = fuzzy_match_fun(Qs, Ms, Fuzzy),
     emqx_mgmt_api:select_table_with_count(
-        Tab, {Ms, FuzzyFun}, Start, Limit, fun format_rule_resp/1
+        Tab, {Ms, FuzzyFun}, Start, Limit
     ).
 
 %% rule is not a record, so everything is fuzzy filter.

From 1fe9c105aa6a801346a0ec0e03b710a7c5a57d56 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 16 Nov 2022 15:49:06 +0800
Subject: [PATCH 50/69] refactor(mgmt): smplify the node_query/cluster_query
 implementation

---
 .../emqx_enhanced_authn_scram_mnesia.erl      |  34 +---
 .../src/simple_authn/emqx_authn_mnesia.erl    |  34 +---
 apps/emqx_authz/src/emqx_authz_api_mnesia.erl |  62 ++----
 .../src/emqx_gateway_api_clients.erl          |  36 ++--
 apps/emqx_management/src/emqx_mgmt_api.erl    | 177 ++++++++++--------
 .../src/emqx_mgmt_api_alarms.erl              |  12 +-
 .../src/emqx_mgmt_api_clients.erl             |  38 ++--
 .../src/emqx_mgmt_api_subscriptions.erl       |  73 +++-----
 .../src/emqx_mgmt_api_topics.erl              |  21 +--
 apps/emqx_modules/src/emqx_delayed.erl        |  18 +-
 .../src/emqx_rule_engine_api.erl              |  13 +-
 11 files changed, 217 insertions(+), 301 deletions(-)

diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
index 1fcc00dc9..dfa2f32ef 100644
--- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
+++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
@@ -47,7 +47,7 @@
 ]).
 
 -export([
-    query/4,
+    qs2ms/2,
     format_user_info/1,
     group_match_spec/1
 ]).
@@ -66,7 +66,6 @@
     {<<"user_group">>, binary},
     {<<"is_superuser">>, atom}
 ]).
--define(QUERY_FUN, {?MODULE, query}).
 
 -type user_group() :: binary().
 
@@ -264,38 +263,23 @@ list_users(QueryString, #{user_group := UserGroup}) ->
     NQueryString = QueryString#{<<"user_group">> => UserGroup},
     emqx_mgmt_api:node_query(
         node(),
-        NQueryString,
         ?TAB,
+        NQueryString,
         ?AUTHN_QSCHEMA,
-        ?QUERY_FUN,
+        fun ?MODULE:qs2ms/2,
         fun ?MODULE:format_user_info/1
     ).
 
 %%--------------------------------------------------------------------
-%% Query Functions
+%% QueryString to MatchSpec
 
-query(Tab, {QString, []}, Continuation, Limit) ->
-    Ms = ms_from_qstring(QString),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        Ms,
-        Continuation,
-        Limit
-    );
-query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
-    Ms = ms_from_qstring(QString),
-    FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        {Ms, FuzzyFilterFun},
-        Continuation,
-        Limit
-    ).
-
-%%--------------------------------------------------------------------
-%% Match funcs
+-spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+qs2ms(_Tab, {QString, Fuzzy}) ->
+    {ms_from_qstring(QString), fuzzy_filter_fun(Fuzzy)}.
 
 %% Fuzzy username funcs
+fuzzy_filter_fun([]) ->
+    undefined;
 fuzzy_filter_fun(Fuzzy) ->
     fun(MsRaws) when is_list(MsRaws) ->
         lists:filter(
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
index f84bfc195..50edb6612 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
@@ -49,7 +49,7 @@
 ]).
 
 -export([
-    query/4,
+    qs2ms/2,
     format_user_info/1,
     group_match_spec/1
 ]).
@@ -84,7 +84,6 @@
     {<<"user_group">>, binary},
     {<<"is_superuser">>, atom}
 ]).
--define(QUERY_FUN, {?MODULE, query}).
 
 %%------------------------------------------------------------------------------
 %% Mnesia bootstrap
@@ -290,38 +289,23 @@ list_users(QueryString, #{user_group := UserGroup}) ->
     NQueryString = QueryString#{<<"user_group">> => UserGroup},
     emqx_mgmt_api:node_query(
         node(),
-        NQueryString,
         ?TAB,
+        NQueryString,
         ?AUTHN_QSCHEMA,
-        ?QUERY_FUN,
+        fun ?MODULE:qs2ms/2,
         fun ?MODULE:format_user_info/1
     ).
 
 %%--------------------------------------------------------------------
-%% Query Functions
+%% QueryString to MatchSpec
 
-query(Tab, {QString, []}, Continuation, Limit) ->
-    Ms = ms_from_qstring(QString),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        Ms,
-        Continuation,
-        Limit
-    );
-query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
-    Ms = ms_from_qstring(QString),
-    FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        {Ms, FuzzyFilterFun},
-        Continuation,
-        Limit
-    ).
-
-%%--------------------------------------------------------------------
-%% Match funcs
+-spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+qs2ms(_Tab, {QString, FuzzyQString}) ->
+    {ms_from_qstring(QString), fuzzy_filter_fun(FuzzyQString)}.
 
 %% Fuzzy username funcs
+fuzzy_filter_fun([]) ->
+    undefined;
 fuzzy_filter_fun(Fuzzy) ->
     fun(MsRaws) when is_list(MsRaws) ->
         lists:filter(
diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
index 8edd62e21..1ab7feea8 100644
--- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
@@ -24,8 +24,8 @@
 
 -import(hoconsc, [mk/1, mk/2, ref/1, ref/2, array/1, enum/1]).
 
--define(QUERY_USERNAME_FUN, {?MODULE, query_username}).
--define(QUERY_CLIENTID_FUN, {?MODULE, query_clientid}).
+-define(QUERY_USERNAME_FUN, fun ?MODULE:query_username/2).
+-define(QUERY_CLIENTID_FUN, fun ?MODULE:query_clientid/2).
 
 -define(ACL_USERNAME_QSCHEMA, [{<<"like_username">>, binary}]).
 -define(ACL_CLIENTID_QSCHEMA, [{<<"like_clientid">>, binary}]).
@@ -49,12 +49,11 @@
 
 %% query funs
 -export([
-    query_username/4,
-    query_clientid/4
+    query_username/2,
+    query_clientid/2,
+    format_result/1
 ]).
 
--export([format_result/1]).
-
 -define(BAD_REQUEST, 'BAD_REQUEST').
 -define(NOT_FOUND, 'NOT_FOUND').
 -define(ALREADY_EXISTS, 'ALREADY_EXISTS').
@@ -405,8 +404,8 @@ users(get, #{query_string := QueryString}) ->
     case
         emqx_mgmt_api:node_query(
             node(),
-            QueryString,
             ?ACL_TABLE,
+            QueryString,
             ?ACL_USERNAME_QSCHEMA,
             ?QUERY_USERNAME_FUN,
             fun ?MODULE:format_result/1
@@ -441,8 +440,8 @@ clients(get, #{query_string := QueryString}) ->
     case
         emqx_mgmt_api:node_query(
             node(),
-            QueryString,
             ?ACL_TABLE,
+            QueryString,
             ?ACL_CLIENTID_QSCHEMA,
             ?QUERY_CLIENTID_FUN,
             fun ?MODULE:format_result/1
@@ -576,48 +575,19 @@ purge(delete, _) ->
     end.
 
 %%--------------------------------------------------------------------
-%% Query Functions
+%% QueryString to MatchSpec
 
-query_username(Tab, {_QString, []}, Continuation, Limit) ->
-    Ms = emqx_authz_mnesia:list_username_rules(),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        Ms,
-        Continuation,
-        Limit
-    );
-query_username(Tab, {_QString, FuzzyQString}, Continuation, Limit) ->
-    Ms = emqx_authz_mnesia:list_username_rules(),
-    FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        {Ms, FuzzyFilterFun},
-        Continuation,
-        Limit
-    ).
+-spec query_username(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+query_username(_Tab, {_QString, FuzzyQString}) ->
+    {emqx_authz_mnesia:list_username_rules(), fuzzy_filter_fun(FuzzyQString)}.
 
-query_clientid(Tab, {_QString, []}, Continuation, Limit) ->
-    Ms = emqx_authz_mnesia:list_clientid_rules(),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        Ms,
-        Continuation,
-        Limit
-    );
-query_clientid(Tab, {_QString, FuzzyQString}, Continuation, Limit) ->
-    Ms = emqx_authz_mnesia:list_clientid_rules(),
-    FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        {Ms, FuzzyFilterFun},
-        Continuation,
-        Limit
-    ).
-
-%%--------------------------------------------------------------------
-%% Match funcs
+-spec query_clientid(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+query_clientid(_Tab, {_QString, FuzzyQString}) ->
+    {emqx_authz_mnesia:list_clientid_rules(), fuzzy_filter_fun(FuzzyQString)}.
 
 %% Fuzzy username funcs
+fuzzy_filter_fun([]) ->
+    undefined;
 fuzzy_filter_fun(Fuzzy) ->
     fun(MsRaws) when is_list(MsRaws) ->
         lists:filter(
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
index f0fbcf968..ba0f34206 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
@@ -55,7 +55,7 @@
 
 %% internal exports (for client query)
 -export([
-    query/4,
+    qs2ms/2,
     format_channel_info/1,
     format_channel_info/2
 ]).
@@ -98,8 +98,6 @@ paths() ->
     {<<"lte_lifetime">>, integer}
 ]).
 
--define(QUERY_FUN, {?MODULE, query}).
-
 clients(get, #{
     bindings := #{name := Name0},
     query_string := QString
@@ -110,10 +108,10 @@ clients(get, #{
             case maps:get(<<"node">>, QString, undefined) of
                 undefined ->
                     emqx_mgmt_api:cluster_query(
-                        QString,
                         TabName,
+                        QString,
                         ?CLIENT_QSCHEMA,
-                        ?QUERY_FUN,
+                        fun ?MODULE:qs2ms/2,
                         fun ?MODULE:format_channel_info/2
                     );
                 Node0 ->
@@ -122,10 +120,10 @@ clients(get, #{
                             QStringWithoutNode = maps:without([<<"node">>], QString),
                             emqx_mgmt_api:node_query(
                                 Node1,
-                                QStringWithoutNode,
                                 TabName,
+                                QStringWithoutNode,
                                 ?CLIENT_QSCHEMA,
-                                ?QUERY_FUN,
+                                fun ?MODULE:qs2ms/2,
                                 fun ?MODULE:format_channel_info/2
                             );
                         {error, _} ->
@@ -267,25 +265,11 @@ extra_sub_props(Props) ->
     ).
 
 %%--------------------------------------------------------------------
-%% query funcs
+%% QueryString to MatchSpec
 
-query(Tab, {Qs, []}, Continuation, Limit) ->
-    Ms = qs2ms(Qs),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        Ms,
-        Continuation,
-        Limit
-    );
-query(Tab, {Qs, Fuzzy}, Continuation, Limit) ->
-    Ms = qs2ms(Qs),
-    FuzzyFilterFun = fuzzy_filter_fun(Fuzzy),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        {Ms, FuzzyFilterFun},
-        Continuation,
-        Limit
-    ).
+-spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+qs2ms(_Tab, {Qs, Fuzzy}) ->
+    {qs2ms(Qs), fuzzy_filter_fun(Fuzzy)}.
 
 qs2ms(Qs) ->
     {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}),
@@ -340,6 +324,8 @@ ms(lifetime, X) ->
 %%--------------------------------------------------------------------
 %% Fuzzy filter funcs
 
+fuzzy_filter_fun([]) ->
+    undefined;
 fuzzy_filter_fun(Fuzzy) ->
     fun(MsRaws) when is_list(MsRaws) ->
         lists:filter(
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index cc0a81864..5fd16e2ff 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -31,11 +31,10 @@
 -export([
     node_query/6,
     cluster_query/5,
-    select_table_with_count/4,
     b2i/1
 ]).
 
--export([do_query/6]).
+-export([do_query/5]).
 
 paginate(Tables, Params, {Module, FormatFun}) ->
     Qh = query_handle(Tables),
@@ -121,15 +120,37 @@ limit(Params) ->
 %% Node Query
 %%--------------------------------------------------------------------
 
-node_query(Node, QString, Tab, QSchema, QueryFun, FmtFun) ->
+-type query_params() :: list() | map().
+
+-type query_schema() :: [{Key :: binary(), Type :: atom | integer | timestamp | ip | ip_port}].
+
+-type query_to_match_spec_fun() ::
+    fun((list(), list()) -> {ets:match_spec(), fun()}).
+
+-type format_result_fun() ::
+    fun((node(), term()) -> term())
+    | fun((term()) -> term()).
+
+-type query_return() :: #{meta := map(), data := [term()]}.
+
+-spec node_query(
+    node(),
+    atom(),
+    query_params(),
+    query_schema(),
+    query_to_match_spec_fun(),
+    format_result_fun()
+) -> {error, page_limit_invalid} | {error, atom(), term()} | query_return().
+node_query(Node, Tab, QString, QSchema, MsFun, FmtFun) ->
     case parse_pager_params(QString) of
         false ->
             {error, page_limit_invalid};
         Meta ->
             {_CodCnt, NQString} = parse_qstring(QString, QSchema),
             ResultAcc = #{cursor => 0, count => 0, rows => []},
+            QueryState = init_query_state(Meta),
             NResultAcc = do_node_query(
-                Node, Tab, NQString, QueryFun, ?FRESH_SELECT, Meta, ResultAcc
+                Node, Tab, NQString, MsFun, QueryState, ResultAcc
             ),
             format_query_result(FmtFun, Meta, NResultAcc)
     end.
@@ -139,31 +160,36 @@ do_node_query(
     Node,
     Tab,
     QString,
-    QueryFun,
-    Continuation,
-    Meta = #{limit := Limit},
+    MsFun,
+    QueryState,
     ResultAcc
 ) ->
-    case do_query(Node, Tab, QString, QueryFun, Continuation, Limit) of
+    case do_query(Node, Tab, QString, MsFun, QueryState) of
         {error, {badrpc, R}} ->
             {error, Node, {badrpc, R}};
-        {Rows, ?FRESH_SELECT} ->
-            {_, NResultAcc} = accumulate_query_rows(Node, Rows, ResultAcc, Meta),
+        {Rows, NQueryState = #{continuation := ?FRESH_SELECT}} ->
+            {_, NResultAcc} = accumulate_query_rows(Node, Rows, NQueryState, ResultAcc),
             NResultAcc;
-        {Rows, NContinuation} ->
-            case accumulate_query_rows(Node, Rows, ResultAcc, Meta) of
+        {Rows, NQueryState} ->
+            case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
                 {enough, NResultAcc} ->
                     NResultAcc;
                 {more, NResultAcc} ->
-                    do_node_query(Node, Tab, QString, QueryFun, NContinuation, Meta, NResultAcc)
+                    do_node_query(Node, Tab, QString, MsFun, NQueryState, NResultAcc)
             end
     end.
 
 %%--------------------------------------------------------------------
 %% Cluster Query
 %%--------------------------------------------------------------------
-
-cluster_query(QString, Tab, QSchema, QueryFun, FmtFun) ->
+-spec cluster_query(
+    atom(),
+    query_params(),
+    query_schema(),
+    query_to_match_spec_fun(),
+    format_result_fun()
+) -> {error, page_limit_invalid} | {error, atom(), term()} | query_return().
+cluster_query(Tab, QString, QSchema, MsFun, FmtFun) ->
     case parse_pager_params(QString) of
         false ->
             {error, page_limit_invalid};
@@ -171,42 +197,38 @@ cluster_query(QString, Tab, QSchema, QueryFun, FmtFun) ->
             {_CodCnt, NQString} = parse_qstring(QString, QSchema),
             Nodes = mria_mnesia:running_nodes(),
             ResultAcc = #{cursor => 0, count => 0, rows => []},
+            QueryState = init_query_state(Meta),
             NResultAcc = do_cluster_query(
-                Nodes, Tab, NQString, QueryFun, ?FRESH_SELECT, Meta, ResultAcc
+                Nodes, Tab, NQString, MsFun, QueryState, ResultAcc
             ),
             format_query_result(FmtFun, Meta, NResultAcc)
     end.
 
 %% @private
-do_cluster_query([], _Tab, _QString, _QueryFun, _Continuation, _Meta, ResultAcc) ->
+do_cluster_query([], _Tab, _QString, _QueryFun, _QueryState, ResultAcc) ->
     ResultAcc;
 do_cluster_query(
     [Node | Tail] = Nodes,
     Tab,
     QString,
-    QueryFun,
-    Continuation,
-    Meta = #{limit := Limit},
+    MsFun,
+    QueryState,
     ResultAcc
 ) ->
-    case do_query(Node, Tab, QString, QueryFun, Continuation, Limit) of
+    case do_query(Node, Tab, QString, MsFun, QueryState) of
         {error, {badrpc, R}} ->
             {error, Node, {badrpc, R}};
-        {Rows, NContinuation} ->
-            case accumulate_query_rows(Node, Rows, ResultAcc, Meta) of
+        {Rows, NQueryState} ->
+            case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
                 {enough, NResultAcc} ->
                     NResultAcc;
                 {more, NResultAcc} ->
-                    case NContinuation of
-                        ?FRESH_SELECT ->
-                            do_cluster_query(
-                                Tail, Tab, QString, QueryFun, ?FRESH_SELECT, Meta, NResultAcc
-                            );
-                        _ ->
-                            do_cluster_query(
-                                Nodes, Tab, QString, QueryFun, NContinuation, Meta, NResultAcc
-                            )
-                    end
+                    NextNodes =
+                        case NQueryState of
+                            #{continuation := ?FRESH_SELECT} -> Tail;
+                            _ -> Nodes
+                        end,
+                    do_cluster_query(NextNodes, Tab, QString, MsFun, NQueryState, NResultAcc)
             end
     end.
 
@@ -214,16 +236,31 @@ do_cluster_query(
 %% Do Query (or rpc query)
 %%--------------------------------------------------------------------
 
+%% QueryState ::
+%%  #{continuation := ets:continuation(),
+%%    page := pos_integer(),
+%%    limit := pos_integer(),
+%%    total := #{node() := non_neg_integer()}
+%%    }
+init_query_state(_Meta = #{page := Page, limit := Limit}) ->
+    #{
+        continuation => ?FRESH_SELECT,
+        page => Page,
+        limit => Limit,
+        total => []
+    }.
+
 %% @private This function is exempt from BPAPI
-do_query(Node, Tab, QString, {M, F}, Continuation, Limit) when Node =:= node() ->
-    erlang:apply(M, F, [Tab, QString, Continuation, Limit]);
-do_query(Node, Tab, QString, QueryFun, Continuation, Limit) ->
+do_query(Node, Tab, QString, MsFun, QueryState) when Node =:= node(), is_function(MsFun) ->
+    {Ms, FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
+    do_select(Tab, Ms, FuzzyFun, QueryState);
+do_query(Node, Tab, QString, MsFun, QueryState) when is_function(MsFun) ->
     case
         rpc:call(
             Node,
             ?MODULE,
             do_query,
-            [Node, Tab, QString, QueryFun, Continuation, Limit],
+            [Node, Tab, QString, MsFun, QueryState],
             50000
         )
     of
@@ -231,6 +268,31 @@ do_query(Node, Tab, QString, QueryFun, Continuation, Limit) ->
         Ret -> Ret
     end.
 
+do_select(
+    Tab,
+    Ms,
+    FuzzyFun,
+    QueryState = #{continuation := Continuation, limit := Limit}
+) ->
+    Result =
+        case Continuation of
+            ?FRESH_SELECT ->
+                ets:select(Tab, Ms, Limit);
+            _ ->
+                ets:select(ets:repair_continuation(Continuation, Ms))
+        end,
+    case Result of
+        '$end_of_table' ->
+            {[], QueryState#{continuation => ?FRESH_SELECT}};
+        {Rows, NContinuation} ->
+            NRows =
+                case is_function(FuzzyFun) of
+                    true -> FuzzyFun(Rows);
+                    false -> Rows
+                end,
+            {NRows, QueryState#{continuation => NContinuation}}
+    end.
+
 %% ResultAcc :: #{count := integer(),
 %%                cursor := integer(),
 %%                rows  := [{node(), Rows :: list()}]
@@ -238,8 +300,8 @@ do_query(Node, Tab, QString, QueryFun, Continuation, Limit) ->
 accumulate_query_rows(
     Node,
     Rows,
-    ResultAcc = #{cursor := Cursor, count := Count, rows := RowsAcc},
-    _Meta = #{page := Page, limit := Limit}
+    _QueryState = #{page := Page, limit := Limit},
+    ResultAcc = #{cursor := Cursor, count := Count, rows := RowsAcc}
 ) ->
     PageStart = (Page - 1) * Limit + 1,
     PageEnd = Page * Limit,
@@ -266,43 +328,6 @@ accumulate_query_rows(
 %% Table Select
 %%--------------------------------------------------------------------
 
-select_table_with_count(Tab, {Ms, FuzzyFilterFun}, ?FRESH_SELECT, Limit) when
-    is_function(FuzzyFilterFun) andalso Limit > 0
-->
-    case ets:select(Tab, Ms, Limit) of
-        '$end_of_table' ->
-            {[], ?FRESH_SELECT};
-        {RawResult, NContinuation} ->
-            Rows = FuzzyFilterFun(RawResult),
-            {Rows, NContinuation}
-    end;
-select_table_with_count(_Tab, {Ms, FuzzyFilterFun}, Continuation, _Limit) when
-    is_function(FuzzyFilterFun)
-->
-    case ets:select(ets:repair_continuation(Continuation, Ms)) of
-        '$end_of_table' ->
-            {[], ?FRESH_SELECT};
-        {RawResult, NContinuation} ->
-            Rows = FuzzyFilterFun(RawResult),
-            {Rows, NContinuation}
-    end;
-select_table_with_count(Tab, Ms, ?FRESH_SELECT, Limit) when
-    Limit > 0
-->
-    case ets:select(Tab, Ms, Limit) of
-        '$end_of_table' ->
-            {[], ?FRESH_SELECT};
-        {RawResult, NContinuation} ->
-            {RawResult, NContinuation}
-    end;
-select_table_with_count(_Tab, Ms, Continuation, _Limit) ->
-    case ets:select(ets:repair_continuation(Continuation, Ms)) of
-        '$end_of_table' ->
-            {[], ?FRESH_SELECT};
-        {RawResult, NContinuation} ->
-            {RawResult, NContinuation}
-    end.
-
 %%--------------------------------------------------------------------
 %% Internal Functions
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl
index d574ad4ee..895a5bcf8 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl
@@ -29,7 +29,7 @@
 -define(TAGS, [<<"Alarms">>]).
 
 %% internal export (for query)
--export([query/4]).
+-export([qs2ms/2]).
 
 api_spec() ->
     emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
@@ -114,10 +114,10 @@ alarms(get, #{query_string := QString}) ->
         end,
     case
         emqx_mgmt_api:cluster_query(
-            QString,
             Table,
+            QString,
             [],
-            {?MODULE, query},
+            fun ?MODULE:qs2ms/2,
             fun ?MODULE:format_alarm/2
         )
     of
@@ -136,9 +136,9 @@ alarms(delete, _Params) ->
 %%%==============================================================================================
 %% internal
 
-query(Table, _QsSpec, Continuation, Limit) ->
-    Ms = [{'$1', [], ['$1']}],
-    emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit).
+-spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+qs2ms(_Tab, {_Qs, _Fuzzy}) ->
+    {[{'$1', [], ['$1']}], undefined}.
 
 format_alarm(WhichNode, Alarm) ->
     emqx_alarm:format(WhichNode, Alarm).
diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index 56730fe3e..cec9b9889 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -46,7 +46,7 @@
 ]).
 
 -export([
-    query/4,
+    qs2ms/2,
     format_channel_info/1,
     format_channel_info/2
 ]).
@@ -74,7 +74,6 @@
     {<<"lte_connected_at">>, timestamp}
 ]).
 
--define(QUERY_FUN, {?MODULE, query}).
 -define(FORMAT_FUN, {?MODULE, format_channel_info}).
 
 -define(CLIENT_ID_NOT_FOUND,
@@ -643,10 +642,10 @@ list_clients(QString) ->
         case maps:get(<<"node">>, QString, undefined) of
             undefined ->
                 emqx_mgmt_api:cluster_query(
-                    QString,
                     ?CLIENT_QTAB,
+                    QString,
                     ?CLIENT_QSCHEMA,
-                    ?QUERY_FUN,
+                    fun ?MODULE:qs2ms/2,
                     fun ?MODULE:format_channel_info/2
                 );
             Node0 ->
@@ -655,10 +654,10 @@ list_clients(QString) ->
                         QStringWithoutNode = maps:without([<<"node">>], QString),
                         emqx_mgmt_api:node_query(
                             Node1,
-                            QStringWithoutNode,
                             ?CLIENT_QTAB,
+                            QStringWithoutNode,
                             ?CLIENT_QSCHEMA,
-                            ?QUERY_FUN,
+                            fun ?MODULE:qs2ms/2,
                             fun ?MODULE:format_channel_info/2
                         );
                     {error, _} ->
@@ -783,30 +782,13 @@ do_unsubscribe(ClientID, Topic) ->
             Res
     end.
 
-%%--------------------------------------------------------------------
-%% Query Functions
-
-query(Tab, {QString, []}, Continuation, Limit) ->
-    Ms = qs2ms(QString),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        Ms,
-        Continuation,
-        Limit
-    );
-query(Tab, {QString, FuzzyQString}, Continuation, Limit) ->
-    Ms = qs2ms(QString),
-    FuzzyFilterFun = fuzzy_filter_fun(FuzzyQString),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        {Ms, FuzzyFilterFun},
-        Continuation,
-        Limit
-    ).
-
 %%--------------------------------------------------------------------
 %% QueryString to Match Spec
 
+-spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+qs2ms(_Tab, {QString, FuzzyQString}) ->
+    {qs2ms(QString), fuzzy_filter_fun(FuzzyQString)}.
+
 -spec qs2ms(list()) -> ets:match_spec().
 qs2ms(Qs) ->
     {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}),
@@ -856,6 +838,8 @@ ms(created_at, X) ->
 %%--------------------------------------------------------------------
 %% Match funcs
 
+fuzzy_filter_fun([]) ->
+    undefined;
 fuzzy_filter_fun(Fuzzy) ->
     fun(MsRaws) when is_list(MsRaws) ->
         lists:filter(
diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
index eb0f83cc0..e4678d641 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
@@ -32,7 +32,7 @@
 -export([subscriptions/2]).
 
 -export([
-    query/4,
+    qs2ms/2,
     format/2
 ]).
 
@@ -47,8 +47,6 @@
     {<<"match_topic">>, binary}
 ]).
 
--define(QUERY_FUN, {?MODULE, query}).
-
 api_spec() ->
     emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
 
@@ -139,10 +137,10 @@ subscriptions(get, #{query_string := QString}) ->
         case maps:get(<<"node">>, QString, undefined) of
             undefined ->
                 emqx_mgmt_api:cluster_query(
-                    QString,
                     ?SUBS_QTABLE,
+                    QString,
                     ?SUBS_QSCHEMA,
-                    ?QUERY_FUN,
+                    fun ?MODULE:qs2ms/2,
                     fun ?MODULE:format/2
                 );
             Node0 ->
@@ -150,10 +148,10 @@ subscriptions(get, #{query_string := QString}) ->
                     {ok, Node1} ->
                         emqx_mgmt_api:node_query(
                             Node1,
-                            QString,
                             ?SUBS_QTABLE,
+                            QString,
                             ?SUBS_QSCHEMA,
-                            ?QUERY_FUN,
+                            fun ?MODULE:qs2ms/2,
                             fun ?MODULE:format/2
                         );
                     {error, _} ->
@@ -188,26 +186,30 @@ get_topic(Topic, _) ->
     Topic.
 
 %%--------------------------------------------------------------------
-%% Query Function
+%% QueryString to MatchSpec
 %%--------------------------------------------------------------------
 
-query(Tab, {Qs, []}, Continuation, Limit) ->
-    Ms = qs2ms(Qs),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        Ms,
-        Continuation,
-        Limit
-    );
-query(Tab, {Qs, Fuzzy}, Continuation, Limit) ->
-    Ms = qs2ms(Qs),
-    FuzzyFilterFun = fuzzy_filter_fun(Fuzzy),
-    emqx_mgmt_api:select_table_with_count(
-        Tab,
-        {Ms, FuzzyFilterFun},
-        Continuation,
-        Limit
-    ).
+-spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+qs2ms(_Tab, {Qs, Fuzzy}) ->
+    {gen_match_spec(Qs), fuzzy_filter_fun(Fuzzy)}.
+
+gen_match_spec(Qs) ->
+    MtchHead = gen_match_spec(Qs, {{'_', '_'}, #{}}),
+    [{MtchHead, [], ['$_']}].
+
+gen_match_spec([], MtchHead) ->
+    MtchHead;
+gen_match_spec([{Key, '=:=', Value} | More], MtchHead) ->
+    gen_match_spec(More, update_ms(Key, Value, MtchHead)).
+
+update_ms(clientid, X, {{Pid, Topic}, Opts}) ->
+    {{Pid, Topic}, Opts#{subid => X}};
+update_ms(topic, X, {{Pid, _Topic}, Opts}) ->
+    {{Pid, X}, Opts};
+update_ms(share_group, X, {{Pid, Topic}, Opts}) ->
+    {{Pid, Topic}, Opts#{share => X}};
+update_ms(qos, X, {{Pid, Topic}, Opts}) ->
+    {{Pid, Topic}, Opts#{qos => X}}.
 
 fuzzy_filter_fun(Fuzzy) ->
     fun(MsRaws) when is_list(MsRaws) ->
@@ -221,24 +223,3 @@ run_fuzzy_filter(_, []) ->
     true;
 run_fuzzy_filter(E = {{_, Topic}, _}, [{topic, match, TopicFilter} | Fuzzy]) ->
     emqx_topic:match(Topic, TopicFilter) andalso run_fuzzy_filter(E, Fuzzy).
-
-%%--------------------------------------------------------------------
-%% Query String to Match Spec
-
-qs2ms(Qs) ->
-    MtchHead = qs2ms(Qs, {{'_', '_'}, #{}}),
-    [{MtchHead, [], ['$_']}].
-
-qs2ms([], MtchHead) ->
-    MtchHead;
-qs2ms([{Key, '=:=', Value} | More], MtchHead) ->
-    qs2ms(More, update_ms(Key, Value, MtchHead)).
-
-update_ms(clientid, X, {{Pid, Topic}, Opts}) ->
-    {{Pid, Topic}, Opts#{subid => X}};
-update_ms(topic, X, {{Pid, _Topic}, Opts}) ->
-    {{Pid, X}, Opts};
-update_ms(share_group, X, {{Pid, Topic}, Opts}) ->
-    {{Pid, Topic}, Opts#{share => X}};
-update_ms(qos, X, {{Pid, Topic}, Opts}) ->
-    {{Pid, Topic}, Opts#{qos => X}}.
diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl
index 3cc2168e7..ca707ba20 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl
@@ -34,7 +34,7 @@
     topic/2
 ]).
 
--export([query/4]).
+-export([qs2ms/2, format/1]).
 
 -define(TOPIC_NOT_FOUND, 'TOPIC_NOT_FOUND').
 
@@ -110,10 +110,10 @@ do_list(Params) ->
     case
         emqx_mgmt_api:node_query(
             node(),
-            Params,
             emqx_route,
+            Params,
             ?TOPICS_QUERY_SCHEMA,
-            {?MODULE, query},
+            fun ?MODULE:qs2ms/2,
             fun ?MODULE:format/1
         )
     of
@@ -143,16 +143,15 @@ generate_topic(Params = #{topic := Topic}) ->
 generate_topic(Params) ->
     Params.
 
-query(Tab, {Qs, _}, Continuation, Limit) ->
-    Ms = qs2ms(Qs, [{{route, '_', '_'}, [], ['$_']}]),
-    emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, fun format/1).
+qs2ms(_Tab, {Qs, _}) ->
+    {gen_match_spec(Qs, [{{route, '_', '_'}, [], ['$_']}]), undefined}.
 
-qs2ms([], Res) ->
+gen_match_spec([], Res) ->
     Res;
-qs2ms([{topic, '=:=', T} | Qs], [{{route, _, N}, [], ['$_']}]) ->
-    qs2ms(Qs, [{{route, T, N}, [], ['$_']}]);
-qs2ms([{node, '=:=', N} | Qs], [{{route, T, _}, [], ['$_']}]) ->
-    qs2ms(Qs, [{{route, T, N}, [], ['$_']}]).
+gen_match_spec([{topic, '=:=', T} | Qs], [{{route, _, N}, [], ['$_']}]) ->
+    gen_match_spec(Qs, [{{route, T, N}, [], ['$_']}]);
+gen_match_spec([{node, '=:=', N} | Qs], [{{route, T, _}, [], ['$_']}]) ->
+    gen_match_spec(Qs, [{{route, T, N}, [], ['$_']}]).
 
 format(#route{topic = Topic, dest = {_, Node}}) ->
     #{topic => Topic, node => Node};
diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl
index 44e7a85e3..0d83e65f1 100644
--- a/apps/emqx_modules/src/emqx_delayed.erl
+++ b/apps/emqx_modules/src/emqx_delayed.erl
@@ -56,10 +56,12 @@
     get_delayed_message/2,
     delete_delayed_message/1,
     delete_delayed_message/2,
-    cluster_list/1,
-    cluster_query/4
+    cluster_list/1
 ]).
 
+%% internal exports
+-export([qs2ms/2]).
+
 -export([
     post_config_update/5
 ]).
@@ -168,12 +170,16 @@ list(Params) ->
 cluster_list(Params) ->
     %% FIXME: why cluster_query???
     emqx_mgmt_api:cluster_query(
-        Params, ?TAB, [], {?MODULE, cluster_query}, fun ?MODULE:format_delayed/1
+        ?TAB,
+        Params,
+        [],
+        fun ?MODULE:qs2ms/2,
+        fun ?MODULE:format_delayed/1
     ).
 
-cluster_query(Table, _QsSpec, Continuation, Limit) ->
-    Ms = [{'$1', [], ['$1']}],
-    emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit).
+-spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+qs2ms(_Table, {_Qs, _Fuzzy}) ->
+    {[{'$1', [], ['$1']}], undefined}.
 
 format_delayed(Delayed) ->
     format_delayed(Delayed, false).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index 2157b108c..f2d5914d4 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -34,7 +34,7 @@
 -export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]).
 
 %% query callback
--export([query/4]).
+-export([qs2ms/2, format_rule_resp/1]).
 
 -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))).
 -define(ERR_BADARGS(REASON), begin
@@ -274,10 +274,10 @@ param_path_id() ->
     case
         emqx_mgmt_api:node_query(
             node(),
-            QueryString,
             ?RULE_TAB,
+            QueryString,
             ?RULE_QS_SCHEMA,
-            {?MODULE, query},
+            fun ?MODULE:qs2ms/2,
             fun ?MODULE:format_rule_resp/1
         )
     of
@@ -553,12 +553,9 @@ filter_out_request_body(Conf) ->
     ],
     maps:without(ExtraConfs, Conf).
 
-query(Tab, {Qs, Fuzzy}, Start, Limit) ->
+qs2ms(_Tab, {Qs, Fuzzy}) ->
     Ms = qs2ms(),
-    FuzzyFun = fuzzy_match_fun(Qs, Ms, Fuzzy),
-    emqx_mgmt_api:select_table_with_count(
-        Tab, {Ms, FuzzyFun}, Start, Limit
-    ).
+    {Ms, fuzzy_match_fun(Qs, Ms, Fuzzy)}.
 
 %% rule is not a record, so everything is fuzzy filter.
 qs2ms() ->

From 28d391f26c893f97742fc1140cab6291b70abd3d Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 16 Nov 2022 17:16:17 +0800
Subject: [PATCH 51/69] fix(mgmt): collect total number in
 node_query/cluster_query

---
 apps/emqx_management/src/emqx_mgmt_api.erl | 98 ++++++++++++++++++----
 1 file changed, 81 insertions(+), 17 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 5fd16e2ff..da5a75c62 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -147,7 +147,7 @@ node_query(Node, Tab, QString, QSchema, MsFun, FmtFun) ->
             {error, page_limit_invalid};
         Meta ->
             {_CodCnt, NQString} = parse_qstring(QString, QSchema),
-            ResultAcc = #{cursor => 0, count => 0, rows => []},
+            ResultAcc = init_query_result(),
             QueryState = init_query_state(Meta),
             NResultAcc = do_node_query(
                 Node, Tab, NQString, MsFun, QueryState, ResultAcc
@@ -196,7 +196,7 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) ->
         Meta ->
             {_CodCnt, NQString} = parse_qstring(QString, QSchema),
             Nodes = mria_mnesia:running_nodes(),
-            ResultAcc = #{cursor => 0, count => 0, rows => []},
+            ResultAcc = init_query_result(),
             QueryState = init_query_state(Meta),
             NResultAcc = do_cluster_query(
                 Nodes, Tab, NQString, MsFun, QueryState, ResultAcc
@@ -205,7 +205,7 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) ->
     end.
 
 %% @private
-do_cluster_query([], _Tab, _QString, _QueryFun, _QueryState, ResultAcc) ->
+do_cluster_query([], _Tab, _QString, _MsFun, _QueryState, ResultAcc) ->
     ResultAcc;
 do_cluster_query(
     [Node | Tail] = Nodes,
@@ -221,7 +221,7 @@ do_cluster_query(
         {Rows, NQueryState} ->
             case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
                 {enough, NResultAcc} ->
-                    NResultAcc;
+                    maybe_collect_total_from_tail_nodes(Tail, Tab, QString, MsFun, NResultAcc);
                 {more, NResultAcc} ->
                     NextNodes =
                         case NQueryState of
@@ -232,6 +232,29 @@ do_cluster_query(
             end
     end.
 
+maybe_collect_total_from_tail_nodes([], _Tab, _QString, _MsFun, ResultAcc) ->
+    ResultAcc;
+maybe_collect_total_from_tail_nodes(Nodes, Tab, QString, MsFun, ResultAcc = #{total := TotalAcc}) ->
+    {Ms, FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
+    case is_countable_total(Ms, FuzzyFun) of
+        true ->
+            %% XXX: badfun risk? if the FuzzyFun is an anonumous func in local node
+            case rpc:multicall(Nodes, ?MODULE, apply_total_query, [Tab, Ms, FuzzyFun]) of
+                {_, [Node | _]} ->
+                    {error, Node, {badrpc, badnode}};
+                {ResL0, []} ->
+                    ResL = lists:zip(Nodes, ResL0),
+                    case lists:filter(fun({_, I}) -> not is_integer(I) end, ResL) of
+                        [{Node, {badrpc, Reason}} | _] ->
+                            {error, Node, {badrpc, Reason}};
+                        [] ->
+                            ResultAcc#{total => ResL ++ TotalAcc}
+                    end
+            end;
+        false ->
+            ResultAcc
+    end.
+
 %%--------------------------------------------------------------------
 %% Do Query (or rpc query)
 %%--------------------------------------------------------------------
@@ -240,7 +263,7 @@ do_cluster_query(
 %%  #{continuation := ets:continuation(),
 %%    page := pos_integer(),
 %%    limit := pos_integer(),
-%%    total := #{node() := non_neg_integer()}
+%%    total := [{node(), non_neg_integer()}]
 %%    }
 init_query_state(_Meta = #{page := Page, limit := Limit}) ->
     #{
@@ -253,7 +276,7 @@ init_query_state(_Meta = #{page := Page, limit := Limit}) ->
 %% @private This function is exempt from BPAPI
 do_query(Node, Tab, QString, MsFun, QueryState) when Node =:= node(), is_function(MsFun) ->
     {Ms, FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
-    do_select(Tab, Ms, FuzzyFun, QueryState);
+    do_select(Node, Tab, Ms, FuzzyFun, QueryState);
 do_query(Node, Tab, QString, MsFun, QueryState) when is_function(MsFun) ->
     case
         rpc:call(
@@ -269,11 +292,13 @@ do_query(Node, Tab, QString, MsFun, QueryState) when is_function(MsFun) ->
     end.
 
 do_select(
+    Node,
     Tab,
     Ms,
     FuzzyFun,
-    QueryState = #{continuation := Continuation, limit := Limit}
+    QueryState0 = #{continuation := Continuation, limit := Limit}
 ) ->
+    QueryState = maybe_apply_total_query(Node, Tab, Ms, FuzzyFun, QueryState0),
     Result =
         case Continuation of
             ?FRESH_SELECT ->
@@ -293,14 +318,48 @@ do_select(
             {NRows, QueryState#{continuation => NContinuation}}
     end.
 
+maybe_apply_total_query(Node, Tab, Ms, FuzzyFun, QueryState = #{total := TotalAcc}) ->
+    case proplists:get_value(Node, TotalAcc, undefined) of
+        undefined ->
+            Total = apply_total_query(Tab, Ms, FuzzyFun),
+            QueryState#{total := [{Node, Total} | TotalAcc]};
+        _ ->
+            QueryState
+    end.
+
+%% XXX: Calculating the total number of data that match a certain condition under a large table
+%% is very expensive because the entire ETS table needs to be scanned.
+apply_total_query(Tab, Ms, FuzzyFun) ->
+    case is_countable_total(Ms, FuzzyFun) of
+        true ->
+            ets:info(Tab, size);
+        false ->
+            %% return a fake total number if the query have any conditions
+            0
+    end.
+
+is_countable_total(Ms, FuzzyFun) ->
+    FuzzyFun =:= undefined andalso is_non_conditions_match_spec(Ms).
+
+is_non_conditions_match_spec([{_MatchHead, _Conds = [], _Return} | More]) ->
+    is_non_conditions_match_spec(More);
+is_non_conditions_match_spec([{_MatchHead, Conds, _Return} | _More]) when length(Conds) =/= 0 ->
+    false;
+is_non_conditions_match_spec([]) ->
+    true.
+
 %% ResultAcc :: #{count := integer(),
 %%                cursor := integer(),
-%%                rows  := [{node(), Rows :: list()}]
+%%                rows  := [{node(), Rows :: list()}],
+%%                total := [{node() => integer()}]
 %%               }
+init_query_result() ->
+    #{cursor => 0, count => 0, rows => [], total => []}.
+
 accumulate_query_rows(
     Node,
     Rows,
-    _QueryState = #{page := Page, limit := Limit},
+    _QueryState = #{page := Page, limit := Limit, total := TotalAcc},
     ResultAcc = #{cursor := Cursor, count := Count, rows := RowsAcc}
 ) ->
     PageStart = (Page - 1) * Limit + 1,
@@ -308,11 +367,12 @@ accumulate_query_rows(
     Len = length(Rows),
     case Cursor + Len of
         NCursor when NCursor < PageStart ->
-            {more, ResultAcc#{cursor => NCursor}};
+            {more, ResultAcc#{cursor => NCursor, total => TotalAcc}};
         NCursor when NCursor < PageEnd ->
             {more, ResultAcc#{
                 cursor => NCursor,
                 count => Count + length(Rows),
+                total => TotalAcc,
                 rows => [{Node, Rows} | RowsAcc]
             }};
         NCursor when NCursor >= PageEnd ->
@@ -320,6 +380,7 @@ accumulate_query_rows(
             {enough, ResultAcc#{
                 cursor => NCursor,
                 count => Count + length(SubRows),
+                total => TotalAcc,
                 rows => [{Node, SubRows} | RowsAcc]
             }}
     end.
@@ -426,10 +487,13 @@ is_fuzzy_key(_) ->
 format_query_result(_FmtFun, _Meta, Error = {error, _Node, _Reason}) ->
     Error;
 format_query_result(
-    FmtFun, Meta, _ResultAcc = #{count := _Count, cursor := Cursor, rows := RowsAcc}
+    FmtFun, Meta, _ResultAcc = #{total := TotalAcc, rows := RowsAcc}
 ) ->
+    Total = lists:foldr(fun({_Node, T}, N) -> N + T end, 0, TotalAcc),
     #{
-        meta => Meta#{count => Cursor},
+        %% The `count` is used in HTTP API to indicate the total number of
+        %% queries that can be read
+        meta => Meta#{count => Total},
         data => lists:flatten(
             lists:foldr(
                 fun({Node, Rows}, Acc) ->
@@ -500,6 +564,11 @@ to_ip_port(IPAddress) ->
     Port = list_to_integer(Port0),
     {IP, Port}.
 
+b2i(Bin) when is_binary(Bin) ->
+    binary_to_integer(Bin);
+b2i(Any) ->
+    Any.
+
 %%--------------------------------------------------------------------
 %% EUnits
 %%--------------------------------------------------------------------
@@ -544,8 +613,3 @@ params2qs_test() ->
     {0, {[], []}} = parse_qstring([{not_a_predefined_params, val}], QSchema).
 
 -endif.
-
-b2i(Bin) when is_binary(Bin) ->
-    binary_to_integer(Bin);
-b2i(Any) ->
-    Any.

From 8f7337c9d26098ba735803375cff53359cab73df Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 16 Nov 2022 17:46:30 +0800
Subject: [PATCH 52/69] chore: return undefined fuzzy searching func

---
 apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
index e4678d641..cb88742c0 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
@@ -211,6 +211,8 @@ update_ms(share_group, X, {{Pid, Topic}, Opts}) ->
 update_ms(qos, X, {{Pid, Topic}, Opts}) ->
     {{Pid, Topic}, Opts#{qos => X}}.
 
+fuzzy_filter_fun([]) ->
+    undefined;
 fuzzy_filter_fun(Fuzzy) ->
     fun(MsRaws) when is_list(MsRaws) ->
         lists:filter(

From 9c7bf9d601bd54edb7f284d110d91442cf066211 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 16 Nov 2022 18:08:05 +0800
Subject: [PATCH 53/69] chore: update app.src

---
 apps/emqx_authz/src/emqx_authz.app.src     | 2 +-
 apps/emqx_modules/src/emqx_modules.app.src | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src
index 6ec14ac3b..e98e8c6b3 100644
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.7"},
+    {vsn, "0.1.8"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [
diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src
index e2a142a99..29a7d0362 100644
--- a/apps/emqx_modules/src/emqx_modules.app.src
+++ b/apps/emqx_modules/src/emqx_modules.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_modules, [
     {description, "EMQX Modules"},
-    {vsn, "5.0.6"},
+    {vsn, "5.0.7"},
     {modules, []},
     {applications, [kernel, stdlib, emqx]},
     {mod, {emqx_modules_app, []}},

From 09958d9a33ce2ad8a320a45ea0ad6fe53acf32e8 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Wed, 16 Nov 2022 20:46:41 +0800
Subject: [PATCH 54/69] chore: fix diaylzer warnings

---
 apps/emqx_management/src/emqx_mgmt_api.erl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index da5a75c62..bbac80c1d 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -122,7 +122,9 @@ limit(Params) ->
 
 -type query_params() :: list() | map().
 
--type query_schema() :: [{Key :: binary(), Type :: atom | integer | timestamp | ip | ip_port}].
+-type query_schema() :: [
+    {Key :: binary(), Type :: atom | binary | integer | timestamp | ip | ip_port}
+].
 
 -type query_to_match_spec_fun() ::
     fun((list(), list()) -> {ets:match_spec(), fun()}).

From 8a0c468b0183f1b8ee446eea2183f94121c1c3e7 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 17 Nov 2022 12:43:55 +0800
Subject: [PATCH 55/69] test: refine tests for lots of List HTTP API

---
 .../test/emqx_authn_mnesia_SUITE.erl          |  2 +-
 apps/emqx_management/src/emqx_mgmt_api.erl    | 83 ++++++++++---------
 .../test/emqx_mgmt_api_subscription_SUITE.erl |  4 +-
 .../src/emqx_rule_engine_api.erl              | 66 +++++++++------
 .../test/emqx_rule_engine_api_SUITE.erl       | 12 +--
 5 files changed, 96 insertions(+), 71 deletions(-)

diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
index 2ab9efb1d..d9fa225bb 100644
--- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
@@ -213,7 +213,7 @@ t_list_users(_) ->
 
     #{
         data := [#{is_superuser := false, user_id := <<"u3">>}],
-        meta := #{page := 1, limit := 20, count := 1}
+        meta := #{page := 1, limit := 20, count := 0}
     } = emqx_authn_mnesia:list_users(
         #{
             <<"page">> => 1,
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index bbac80c1d..ee733e2dc 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -21,6 +21,7 @@
 -elvis([{elvis_style, dont_repeat_yourself, #{min_complexity => 100}}]).
 
 -define(FRESH_SELECT, fresh_select).
+-define(LONG_QUERY_TIMEOUT, 50000).
 
 -export([
     paginate/3,
@@ -35,6 +36,7 @@
 ]).
 
 -export([do_query/5]).
+-export([parse_qstring/2]).
 
 paginate(Tables, Params, {Module, FormatFun}) ->
     Qh = query_handle(Tables),
@@ -236,25 +238,30 @@ do_cluster_query(
 
 maybe_collect_total_from_tail_nodes([], _Tab, _QString, _MsFun, ResultAcc) ->
     ResultAcc;
-maybe_collect_total_from_tail_nodes(Nodes, Tab, QString, MsFun, ResultAcc = #{total := TotalAcc}) ->
+maybe_collect_total_from_tail_nodes(Nodes, Tab, QString, MsFun, ResultAcc) ->
     {Ms, FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
-    case is_countable_total(Ms, FuzzyFun) of
-        true ->
-            %% XXX: badfun risk? if the FuzzyFun is an anonumous func in local node
-            case rpc:multicall(Nodes, ?MODULE, apply_total_query, [Tab, Ms, FuzzyFun]) of
-                {_, [Node | _]} ->
-                    {error, Node, {badrpc, badnode}};
-                {ResL0, []} ->
-                    ResL = lists:zip(Nodes, ResL0),
-                    case lists:filter(fun({_, I}) -> not is_integer(I) end, ResL) of
-                        [{Node, {badrpc, Reason}} | _] ->
-                            {error, Node, {badrpc, Reason}};
-                        [] ->
-                            ResultAcc#{total => ResL ++ TotalAcc}
-                    end
-            end;
+    case counting_total_fun(Ms, FuzzyFun) of
         false ->
-            ResultAcc
+            ResultAcc;
+        _Fun ->
+            collect_total_from_tail_nodes(Nodes, Tab, Ms, FuzzyFun, ResultAcc)
+    end.
+
+collect_total_from_tail_nodes(Nodes, Tab, Ms, FuzzyFun, ResultAcc = #{total := TotalAcc}) ->
+    %% XXX: badfun risk? if the FuzzyFun is an anonumous func in local node
+    case
+        rpc:multicall(Nodes, ?MODULE, apply_total_query, [Tab, Ms, FuzzyFun], ?LONG_QUERY_TIMEOUT)
+    of
+        {_, [Node | _]} ->
+            {error, Node, {badrpc, badnode}};
+        {ResL0, []} ->
+            ResL = lists:zip(Nodes, ResL0),
+            case lists:filter(fun({_, I}) -> not is_integer(I) end, ResL) of
+                [{Node, {badrpc, Reason}} | _] ->
+                    {error, Node, {badrpc, Reason}};
+                [] ->
+                    ResultAcc#{total => ResL ++ TotalAcc}
+            end
     end.
 
 %%--------------------------------------------------------------------
@@ -286,7 +293,7 @@ do_query(Node, Tab, QString, MsFun, QueryState) when is_function(MsFun) ->
             ?MODULE,
             do_query,
             [Node, Tab, QString, MsFun, QueryState],
-            50000
+            ?LONG_QUERY_TIMEOUT
         )
     of
         {badrpc, _} = R -> {error, R};
@@ -329,26 +336,31 @@ maybe_apply_total_query(Node, Tab, Ms, FuzzyFun, QueryState = #{total := TotalAc
             QueryState
     end.
 
-%% XXX: Calculating the total number of data that match a certain condition under a large table
-%% is very expensive because the entire ETS table needs to be scanned.
 apply_total_query(Tab, Ms, FuzzyFun) ->
-    case is_countable_total(Ms, FuzzyFun) of
-        true ->
-            ets:info(Tab, size);
+    case counting_total_fun(Ms, FuzzyFun) of
         false ->
             %% return a fake total number if the query have any conditions
-            0
+            0;
+        Fun ->
+            Fun(Tab)
     end.
 
-is_countable_total(Ms, FuzzyFun) ->
-    FuzzyFun =:= undefined andalso is_non_conditions_match_spec(Ms).
-
-is_non_conditions_match_spec([{_MatchHead, _Conds = [], _Return} | More]) ->
-    is_non_conditions_match_spec(More);
-is_non_conditions_match_spec([{_MatchHead, Conds, _Return} | _More]) when length(Conds) =/= 0 ->
-    false;
-is_non_conditions_match_spec([]) ->
-    true.
+counting_total_fun(Ms, undefined) ->
+    %% XXX: Calculating the total number of data that match a certain
+    %% condition under a large table is very expensive because the
+    %% entire ETS table needs to be scanned.
+    %%
+    %% XXX: How to optimize it? i.e, using:
+    %% `fun(Tab) -> ets:info(Tab, size) end`
+    [{MatchHead, Conditions, _Return}] = Ms,
+    CountingMs = [{MatchHead, Conditions, [true]}],
+    fun(Tab) ->
+        ets:select_count(Tab, CountingMs)
+    end;
+counting_total_fun(_Ms, FuzzyFun) when is_function(FuzzyFun) ->
+    %% XXX: Calculating the total number for a fuzzy searching is very very expensive
+    %% so it is not supported now
+    false.
 
 %% ResultAcc :: #{count := integer(),
 %%                cursor := integer(),
@@ -387,10 +399,6 @@ accumulate_query_rows(
             }}
     end.
 
-%%--------------------------------------------------------------------
-%% Table Select
-%%--------------------------------------------------------------------
-
 %%--------------------------------------------------------------------
 %% Internal Functions
 %%--------------------------------------------------------------------
@@ -402,6 +410,7 @@ parse_qstring(QString, QSchema) ->
     {length(NQString) + length(FuzzyQString), {NQString, FuzzyQString}}.
 
 do_parse_qstring([], _, Acc1, Acc2) ->
+    %% remove fuzzy keys if present in accurate query
     NAcc2 = [E || E <- Acc2, not lists:keymember(element(1, E), 1, Acc1)],
     {lists:reverse(Acc1), lists:reverse(NAcc2)};
 do_parse_qstring([{Key, Value} | RestQString], QSchema, Acc1, Acc2) ->
diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
index d1cf4e418..9a6de9938 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
@@ -93,6 +93,7 @@ t_subscription_api(_) ->
         {"match_topic", "t/#"}
     ]),
     Headers = emqx_mgmt_api_test_util:auth_header_(),
+
     {ok, ResponseTopic2} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers),
     DataTopic2 = emqx_json:decode(ResponseTopic2, [return_maps]),
     Meta2 = maps:get(<<"meta">>, DataTopic2),
@@ -114,7 +115,8 @@ t_subscription_api(_) ->
     MatchMeta = maps:get(<<"meta">>, MatchData),
     ?assertEqual(1, maps:get(<<"page">>, MatchMeta)),
     ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta)),
-    ?assertEqual(1, maps:get(<<"count">>, MatchMeta)),
+    %% count equals 0 in fuzzy searching
+    ?assertEqual(0, maps:get(<<"count">>, MatchMeta)),
     MatchSubs = maps:get(<<"data">>, MatchData),
     ?assertEqual(1, length(MatchSubs)),
 
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index f2d5914d4..ad0018c0c 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -554,34 +554,46 @@ filter_out_request_body(Conf) ->
     maps:without(ExtraConfs, Conf).
 
 qs2ms(_Tab, {Qs, Fuzzy}) ->
-    Ms = qs2ms(),
-    {Ms, fuzzy_match_fun(Qs, Ms, Fuzzy)}.
-
-%% rule is not a record, so everything is fuzzy filter.
-qs2ms() ->
-    [{'_', [], ['$_']}].
-
-fuzzy_match_fun(Qs, Ms, Fuzzy) ->
-    MsC = ets:match_spec_compile(Ms),
-    fun(Rows) ->
-        Ls = ets:match_spec_run(Rows, MsC),
-        lists:filter(
-            fun(E) ->
-                run_qs_match(E, Qs) andalso
-                    run_fuzzy_match(E, Fuzzy)
-            end,
-            Ls
-        )
+    case lists:keytake(from, 1, Qs) of
+        false ->
+            {generate_match_spec(Qs), fuzzy_match_fun(Fuzzy)};
+        {value, {from, '=:=', From}, Ls} ->
+            {generate_match_spec(Ls), fuzzy_match_fun([{from, '=:=', From} | Fuzzy])}
     end.
 
-run_qs_match(_, []) ->
-    true;
-run_qs_match(E = {_Id, #{enable := Enable}}, [{enable, '=:=', Pattern} | Qs]) ->
-    Enable =:= Pattern andalso run_qs_match(E, Qs);
-run_qs_match(E = {_Id, #{from := From}}, [{from, '=:=', Pattern} | Qs]) ->
-    lists:member(Pattern, From) andalso run_qs_match(E, Qs);
-run_qs_match(E, [_ | Qs]) ->
-    run_qs_match(E, Qs).
+generate_match_spec(Qs) ->
+    {MtchHead, Conds} = generate_match_spec(Qs, 2, {#{}, []}),
+    [{{'_', MtchHead}, Conds, ['$_']}].
+
+generate_match_spec([], _, {MtchHead, Conds}) ->
+    {MtchHead, lists:reverse(Conds)};
+generate_match_spec([Qs | Rest], N, {MtchHead, Conds}) ->
+    Holder = binary_to_atom(iolist_to_binary(["$", integer_to_list(N)]), utf8),
+    NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(element(1, Qs), Holder)),
+    NConds = put_conds(Qs, Holder, Conds),
+    generate_match_spec(Rest, N + 1, {NMtchHead, NConds}).
+
+put_conds({_, Op, V}, Holder, Conds) ->
+    [{Op, Holder, V} | Conds];
+put_conds({_, Op1, V1, Op2, V2}, Holder, Conds) ->
+    [
+        {Op2, Holder, V2},
+        {Op1, Holder, V1}
+        | Conds
+    ].
+
+ms(enable, X) ->
+    #{enable => X}.
+
+fuzzy_match_fun([]) ->
+    undefined;
+fuzzy_match_fun(Fuzzy) ->
+    fun(MsRaws) when is_list(MsRaws) ->
+        lists:filter(
+            fun(E) -> run_fuzzy_match(E, Fuzzy) end,
+            MsRaws
+        )
+    end.
 
 run_fuzzy_match(_, []) ->
     true;
@@ -589,6 +601,8 @@ run_fuzzy_match(E = {Id, _}, [{id, like, Pattern} | Fuzzy]) ->
     binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
 run_fuzzy_match(E = {_Id, #{description := Desc}}, [{description, like, Pattern} | Fuzzy]) ->
     binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = {_, #{from := Topics}}, [{from, '=:=', Pattern} | Fuzzy]) ->
+    lists:member(Pattern, Topics) /= false andalso run_fuzzy_match(E, Fuzzy);
 run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) ->
     lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) andalso
         run_fuzzy_match(E, Fuzzy);
diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl
index da4e299f9..93912dd6c 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl
@@ -133,23 +133,23 @@ t_list_rule_api(_Config) ->
 
     QueryStr2 = #{query_string => #{<<"like_description">> => <<"也能"/utf8>>}},
     {200, Result2} = emqx_rule_engine_api:'/rules'(get, QueryStr2),
-    ?assertEqual(Result1, Result2),
+    ?assertEqual(maps:get(data, Result1), maps:get(data, Result2)),
 
     QueryStr3 = #{query_string => #{<<"from">> => <<"t/1">>}},
-    {200, #{meta := #{count := Count3}}} = emqx_rule_engine_api:'/rules'(get, QueryStr3),
-    ?assertEqual(19, Count3),
+    {200, #{data := Data3}} = emqx_rule_engine_api:'/rules'(get, QueryStr3),
+    ?assertEqual(19, length(Data3)),
 
     QueryStr4 = #{query_string => #{<<"like_from">> => <<"t/1/+">>}},
     {200, Result4} = emqx_rule_engine_api:'/rules'(get, QueryStr4),
-    ?assertEqual(Result1, Result4),
+    ?assertEqual(maps:get(data, Result1), maps:get(data, Result4)),
 
     QueryStr5 = #{query_string => #{<<"match_from">> => <<"t/+/+">>}},
     {200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5),
-    ?assertEqual(Result1, Result5),
+    ?assertEqual(maps:get(data, Result1), maps:get(data, Result5)),
 
     QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}},
     {200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6),
-    ?assertEqual(Result1, Result6),
+    ?assertEqual(maps:get(data, Result1), maps:get(data, Result6)),
 
     %% clean up
     lists:foreach(

From 79a2682fd3d9d87615b8ffe829635d96170f8a5a Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 17 Nov 2022 18:36:01 +0800
Subject: [PATCH 56/69] chore: improve the no-conditions query

---
 apps/emqx/test/emqx_bpapi_static_checks.erl |   2 +-
 apps/emqx_management/src/emqx_mgmt_api.erl  | 100 ++++++++++----------
 2 files changed, 52 insertions(+), 50 deletions(-)

diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl
index d87258201..7849cd617 100644
--- a/apps/emqx/test/emqx_bpapi_static_checks.erl
+++ b/apps/emqx/test/emqx_bpapi_static_checks.erl
@@ -62,7 +62,7 @@
 %% List of business-layer functions that are exempt from the checks:
 %% erlfmt-ignore
 -define(EXEMPTIONS,
-    "emqx_mgmt_api:do_query/6"  % Reason: legacy code. A fun and a QC query are
+    "emqx_mgmt_api:do_query/2"  % Reason: legacy code. A fun and a QC query are
                                 % passed in the args, it's futile to try to statically
                                 % check it
 ).
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index ee733e2dc..9f757b0cc 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -35,8 +35,7 @@
     b2i/1
 ]).
 
--export([do_query/5]).
--export([parse_qstring/2]).
+-export([do_query/2, apply_total_query/1]).
 
 paginate(Tables, Params, {Module, FormatFun}) ->
     Qh = query_handle(Tables),
@@ -152,9 +151,9 @@ node_query(Node, Tab, QString, QSchema, MsFun, FmtFun) ->
         Meta ->
             {_CodCnt, NQString} = parse_qstring(QString, QSchema),
             ResultAcc = init_query_result(),
-            QueryState = init_query_state(Meta),
+            QueryState = init_query_state(Tab, NQString, MsFun, Meta),
             NResultAcc = do_node_query(
-                Node, Tab, NQString, MsFun, QueryState, ResultAcc
+                Node, QueryState, ResultAcc
             ),
             format_query_result(FmtFun, Meta, NResultAcc)
     end.
@@ -162,13 +161,10 @@ node_query(Node, Tab, QString, QSchema, MsFun, FmtFun) ->
 %% @private
 do_node_query(
     Node,
-    Tab,
-    QString,
-    MsFun,
     QueryState,
     ResultAcc
 ) ->
-    case do_query(Node, Tab, QString, MsFun, QueryState) of
+    case do_query(Node, QueryState) of
         {error, {badrpc, R}} ->
             {error, Node, {badrpc, R}};
         {Rows, NQueryState = #{continuation := ?FRESH_SELECT}} ->
@@ -179,7 +175,7 @@ do_node_query(
                 {enough, NResultAcc} ->
                     NResultAcc;
                 {more, NResultAcc} ->
-                    do_node_query(Node, Tab, QString, MsFun, NQueryState, NResultAcc)
+                    do_node_query(Node, NQueryState, NResultAcc)
             end
     end.
 
@@ -201,57 +197,51 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) ->
             {_CodCnt, NQString} = parse_qstring(QString, QSchema),
             Nodes = mria_mnesia:running_nodes(),
             ResultAcc = init_query_result(),
-            QueryState = init_query_state(Meta),
+            QueryState = init_query_state(Tab, NQString, MsFun, Meta),
             NResultAcc = do_cluster_query(
-                Nodes, Tab, NQString, MsFun, QueryState, ResultAcc
+                Nodes, QueryState, ResultAcc
             ),
             format_query_result(FmtFun, Meta, NResultAcc)
     end.
 
 %% @private
-do_cluster_query([], _Tab, _QString, _MsFun, _QueryState, ResultAcc) ->
+do_cluster_query([], _QueryState, ResultAcc) ->
     ResultAcc;
 do_cluster_query(
     [Node | Tail] = Nodes,
-    Tab,
-    QString,
-    MsFun,
     QueryState,
     ResultAcc
 ) ->
-    case do_query(Node, Tab, QString, MsFun, QueryState) of
+    case do_query(Node, QueryState) of
         {error, {badrpc, R}} ->
             {error, Node, {badrpc, R}};
         {Rows, NQueryState} ->
             case accumulate_query_rows(Node, Rows, NQueryState, ResultAcc) of
                 {enough, NResultAcc} ->
-                    maybe_collect_total_from_tail_nodes(Tail, Tab, QString, MsFun, NResultAcc);
+                    maybe_collect_total_from_tail_nodes(Tail, NQueryState, NResultAcc);
                 {more, NResultAcc} ->
                     NextNodes =
                         case NQueryState of
                             #{continuation := ?FRESH_SELECT} -> Tail;
                             _ -> Nodes
                         end,
-                    do_cluster_query(NextNodes, Tab, QString, MsFun, NQueryState, NResultAcc)
+                    do_cluster_query(NextNodes, NQueryState, NResultAcc)
             end
     end.
 
-maybe_collect_total_from_tail_nodes([], _Tab, _QString, _MsFun, ResultAcc) ->
+maybe_collect_total_from_tail_nodes([], _QueryState, ResultAcc) ->
     ResultAcc;
-maybe_collect_total_from_tail_nodes(Nodes, Tab, QString, MsFun, ResultAcc) ->
-    {Ms, FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
-    case counting_total_fun(Ms, FuzzyFun) of
+maybe_collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc) ->
+    case counting_total_fun(QueryState) of
         false ->
             ResultAcc;
         _Fun ->
-            collect_total_from_tail_nodes(Nodes, Tab, Ms, FuzzyFun, ResultAcc)
+            collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc)
     end.
 
-collect_total_from_tail_nodes(Nodes, Tab, Ms, FuzzyFun, ResultAcc = #{total := TotalAcc}) ->
+collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc}) ->
     %% XXX: badfun risk? if the FuzzyFun is an anonumous func in local node
-    case
-        rpc:multicall(Nodes, ?MODULE, apply_total_query, [Tab, Ms, FuzzyFun], ?LONG_QUERY_TIMEOUT)
-    of
+    case rpc:multicall(Nodes, ?MODULE, apply_total_query, [QueryState], ?LONG_QUERY_TIMEOUT) of
         {_, [Node | _]} ->
             {error, Node, {badrpc, badnode}};
         {ResL0, []} ->
@@ -272,27 +262,35 @@ collect_total_from_tail_nodes(Nodes, Tab, Ms, FuzzyFun, ResultAcc = #{total := T
 %%  #{continuation := ets:continuation(),
 %%    page := pos_integer(),
 %%    limit := pos_integer(),
-%%    total := [{node(), non_neg_integer()}]
+%%    total := [{node(), non_neg_integer()}],
+%%    table := atom(),
+%%    qs := {Qs, Fuzzy}   %% parsed query params
+%%    msfun := query_to_match_spec_fun()
 %%    }
-init_query_state(_Meta = #{page := Page, limit := Limit}) ->
+init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) ->
+    {Ms, FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
     #{
-        continuation => ?FRESH_SELECT,
         page => Page,
         limit => Limit,
-        total => []
+        table => Tab,
+        qs => QString,
+        msfun => MsFun,
+        mactch_spec => Ms,
+        fuzzy_fun => FuzzyFun,
+        total => [],
+        continuation => ?FRESH_SELECT
     }.
 
 %% @private This function is exempt from BPAPI
-do_query(Node, Tab, QString, MsFun, QueryState) when Node =:= node(), is_function(MsFun) ->
-    {Ms, FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
-    do_select(Node, Tab, Ms, FuzzyFun, QueryState);
-do_query(Node, Tab, QString, MsFun, QueryState) when is_function(MsFun) ->
+do_query(Node, QueryState) when Node =:= node() ->
+    do_select(Node, QueryState);
+do_query(Node, QueryState) ->
     case
         rpc:call(
             Node,
             ?MODULE,
             do_query,
-            [Node, Tab, QString, MsFun, QueryState],
+            [Node, QueryState],
             ?LONG_QUERY_TIMEOUT
         )
     of
@@ -302,18 +300,21 @@ do_query(Node, Tab, QString, MsFun, QueryState) when is_function(MsFun) ->
 
 do_select(
     Node,
-    Tab,
-    Ms,
-    FuzzyFun,
-    QueryState0 = #{continuation := Continuation, limit := Limit}
+    QueryState0 = #{
+        table := Tab,
+        mactch_spec := Ms,
+        fuzzy_fun := FuzzyFun,
+        continuation := Continuation,
+        limit := Limit
+    }
 ) ->
-    QueryState = maybe_apply_total_query(Node, Tab, Ms, FuzzyFun, QueryState0),
+    QueryState = maybe_apply_total_query(Node, QueryState0),
     Result =
         case Continuation of
             ?FRESH_SELECT ->
                 ets:select(Tab, Ms, Limit);
             _ ->
-                ets:select(ets:repair_continuation(Continuation, Ms))
+                ets:select(Continuation)
         end,
     case Result of
         '$end_of_table' ->
@@ -327,17 +328,17 @@ do_select(
             {NRows, QueryState#{continuation => NContinuation}}
     end.
 
-maybe_apply_total_query(Node, Tab, Ms, FuzzyFun, QueryState = #{total := TotalAcc}) ->
+maybe_apply_total_query(Node, QueryState = #{total := TotalAcc}) ->
     case proplists:get_value(Node, TotalAcc, undefined) of
         undefined ->
-            Total = apply_total_query(Tab, Ms, FuzzyFun),
+            Total = apply_total_query(QueryState),
             QueryState#{total := [{Node, Total} | TotalAcc]};
         _ ->
             QueryState
     end.
 
-apply_total_query(Tab, Ms, FuzzyFun) ->
-    case counting_total_fun(Ms, FuzzyFun) of
+apply_total_query(QueryState = #{table := Tab}) ->
+    case counting_total_fun(QueryState) of
         false ->
             %% return a fake total number if the query have any conditions
             0;
@@ -345,19 +346,20 @@ apply_total_query(Tab, Ms, FuzzyFun) ->
             Fun(Tab)
     end.
 
-counting_total_fun(Ms, undefined) ->
+counting_total_fun(_QueryState = #{qs := {[], []}}) ->
+    fun(Tab) -> ets:info(Tab, size) end;
+counting_total_fun(_QueryState = #{mactch_spec := Ms, fuzzy_fun := undefined}) ->
     %% XXX: Calculating the total number of data that match a certain
     %% condition under a large table is very expensive because the
     %% entire ETS table needs to be scanned.
     %%
     %% XXX: How to optimize it? i.e, using:
-    %% `fun(Tab) -> ets:info(Tab, size) end`
     [{MatchHead, Conditions, _Return}] = Ms,
     CountingMs = [{MatchHead, Conditions, [true]}],
     fun(Tab) ->
         ets:select_count(Tab, CountingMs)
     end;
-counting_total_fun(_Ms, FuzzyFun) when is_function(FuzzyFun) ->
+counting_total_fun(_QueryState = #{fuzzy_fun := FuzzyFun}) when is_function(FuzzyFun) ->
     %% XXX: Calculating the total number for a fuzzy searching is very very expensive
     %% so it is not supported now
     false.

From 0c1412315cb24d6042073b9163c9ca4b0abd4389 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 17 Nov 2022 21:43:30 +0800
Subject: [PATCH 57/69] chore(bpapi): ignore
 emqx_mgmt_api:collect_total_from_tail_nodes/3

---
 apps/emqx/test/emqx_bpapi_static_checks.erl                | 7 ++++---
 .../test/emqx_enhanced_authn_scram_mnesia_SUITE.erl        | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl
index 7849cd617..69cede2bc 100644
--- a/apps/emqx/test/emqx_bpapi_static_checks.erl
+++ b/apps/emqx/test/emqx_bpapi_static_checks.erl
@@ -62,9 +62,10 @@
 %% List of business-layer functions that are exempt from the checks:
 %% erlfmt-ignore
 -define(EXEMPTIONS,
-    "emqx_mgmt_api:do_query/2"  % Reason: legacy code. A fun and a QC query are
-                                % passed in the args, it's futile to try to statically
-                                % check it
+    % Reason: legacy code. A fun and a QC query are
+    % passed in the args, it's futile to try to statically
+    % check it
+    "emqx_mgmt_api:do_query/2, emqx_mgmt_api:collect_total_from_tail_nodes/3"
 ).
 
 -define(XREF, myxref).
diff --git a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl
index 41fa5a38c..a0b5ce980 100644
--- a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl
@@ -319,7 +319,7 @@ t_list_users(_) ->
                 is_superuser := _
             }
         ],
-        meta := #{page := 1, limit := 3, count := 1}
+        meta := #{page := 1, limit := 3, count := 0}
     } = emqx_enhanced_authn_scram_mnesia:list_users(
         #{
             <<"page">> => 1,

From b9c5a5f82265660c356b27a22aa64116e5fe7a56 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Fri, 18 Nov 2022 13:28:59 +0800
Subject: [PATCH 58/69] fix(delayed): return correct node name

---
 .../test/emqx_mgmt_api_SUITE.erl              | 42 +++++++++++++++++++
 .../test/emqx_mgmt_api_topics_SUITE.erl       | 24 ++++++++++-
 apps/emqx_modules/src/emqx_delayed.erl        | 23 ++++++----
 .../src/emqx_rule_engine_api.erl              |  8 +---
 4 files changed, 79 insertions(+), 18 deletions(-)
 create mode 100644 apps/emqx_management/test/emqx_mgmt_api_SUITE.erl

diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
new file mode 100644
index 000000000..c9abef375
--- /dev/null
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -0,0 +1,42 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_mgmt_api_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+%%--------------------------------------------------------------------
+%% setup
+%%--------------------------------------------------------------------
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_mgmt_api_test_util:init_suite(),
+    Config.
+
+end_per_suite(_) ->
+    emqx_mgmt_api_test_util:end_suite().
+
+%%--------------------------------------------------------------------
+%% cases
+%%--------------------------------------------------------------------
+
+t_cluster_query(_Config) ->
+    ok.
diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
index a06154400..419c17adf 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
@@ -31,6 +31,7 @@ end_per_suite(_) ->
     emqx_mgmt_api_test_util:end_suite().
 
 t_nodes_api(_) ->
+    Node = atom_to_binary(node(), utf8),
     Topic = <<"test_topic">>,
     {ok, Client} = emqtt:start_link(#{
         username => <<"routes_username">>, clientid => <<"routes_cid">>
@@ -49,11 +50,30 @@ t_nodes_api(_) ->
     Data = maps:get(<<"data">>, RoutesData),
     Route = erlang:hd(Data),
     ?assertEqual(Topic, maps:get(<<"topic">>, Route)),
-    ?assertEqual(atom_to_binary(node(), utf8), maps:get(<<"node">>, Route)),
+    ?assertEqual(Node, maps:get(<<"node">>, Route)),
+
+    %% exact match
+    Topic2 = <<"test_topic_2">>,
+    {ok, _, _} = emqtt:subscribe(Client, Topic2),
+    QS = uri_string:compose_query([
+        {"topic", Topic2},
+        {"node", atom_to_list(node())}
+    ]),
+    Headers = emqx_mgmt_api_test_util:auth_header_(),
+    {ok, MatchResponse} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers),
+    MatchData = emqx_json:decode(MatchResponse, [return_maps]),
+    ?assertMatch(
+        #{<<"count">> := 1, <<"page">> := 1, <<"limit">> := 100},
+        maps:get(<<"meta">>, MatchData)
+    ),
+    ?assertMatch(
+        [#{<<"topic">> := Topic2, <<"node">> := Node}],
+        maps:get(<<"data">>, MatchData)
+    ),
 
     %% get topics/:topic
     RoutePath = emqx_mgmt_api_test_util:api_path(["topics", Topic]),
     {ok, RouteResponse} = emqx_mgmt_api_test_util:request_api(get, RoutePath),
     RouteData = emqx_json:decode(RouteResponse, [return_maps]),
     ?assertEqual(Topic, maps:get(<<"topic">>, RouteData)),
-    ?assertEqual(atom_to_binary(node(), utf8), maps:get(<<"node">>, RouteData)).
+    ?assertEqual(Node, maps:get(<<"node">>, RouteData)).
diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl
index 0d83e65f1..c96e5a452 100644
--- a/apps/emqx_modules/src/emqx_delayed.erl
+++ b/apps/emqx_modules/src/emqx_delayed.erl
@@ -59,15 +59,17 @@
     cluster_list/1
 ]).
 
-%% internal exports
--export([qs2ms/2]).
+%% exports for query
+-export([
+    qs2ms/2,
+    format_delayed/1,
+    format_delayed/2
+]).
 
 -export([
     post_config_update/5
 ]).
 
--export([format_delayed/1]).
-
 %% exported for `emqx_telemetry'
 -export([get_basic_usage_info/0]).
 
@@ -168,13 +170,12 @@ list(Params) ->
     emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN).
 
 cluster_list(Params) ->
-    %% FIXME: why cluster_query???
     emqx_mgmt_api:cluster_query(
         ?TAB,
         Params,
         [],
         fun ?MODULE:qs2ms/2,
-        fun ?MODULE:format_delayed/1
+        fun ?MODULE:format_delayed/2
     ).
 
 -spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
@@ -182,9 +183,13 @@ qs2ms(_Table, {_Qs, _Fuzzy}) ->
     {[{'$1', [], ['$1']}], undefined}.
 
 format_delayed(Delayed) ->
-    format_delayed(Delayed, false).
+    format_delayed(node(), Delayed).
+
+format_delayed(WhichNode, Delayed) ->
+    format_delayed(WhichNode, Delayed, false).
 
 format_delayed(
+    WhichNode,
     #delayed_message{
         key = {ExpectTimeStamp, Id},
         delayed = Delayed,
@@ -204,7 +209,7 @@ format_delayed(
     RemainingTime = ExpectTimeStamp - ?NOW,
     Result = #{
         msgid => emqx_guid:to_hexstr(Id),
-        node => node(),
+        node => WhichNode,
         publish_at => PublishTime,
         delayed_interval => Delayed,
         delayed_remaining => RemainingTime div 1000,
@@ -231,7 +236,7 @@ get_delayed_message(Id) ->
             {error, not_found};
         Rows ->
             Message = hd(Rows),
-            {ok, format_delayed(Message, true)}
+            {ok, format_delayed(node(), Message, true)}
     end.
 
 get_delayed_message(Node, Id) when Node =:= node() ->
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index ad0018c0c..b34d0ceae 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -574,13 +574,7 @@ generate_match_spec([Qs | Rest], N, {MtchHead, Conds}) ->
     generate_match_spec(Rest, N + 1, {NMtchHead, NConds}).
 
 put_conds({_, Op, V}, Holder, Conds) ->
-    [{Op, Holder, V} | Conds];
-put_conds({_, Op1, V1, Op2, V2}, Holder, Conds) ->
-    [
-        {Op2, Holder, V2},
-        {Op1, Holder, V1}
-        | Conds
-    ].
+    [{Op, Holder, V} | Conds].
 
 ms(enable, X) ->
     #{enable => X}.

From 36de83a50a8489ecc12e782da16f1dff96a5c3b9 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Fri, 18 Nov 2022 18:20:34 +0800
Subject: [PATCH 59/69] feat(cm): change emqx_channel_info to ordered_set table

---
 apps/emqx/src/emqx_cm.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl
index d39f43686..483c51f26 100644
--- a/apps/emqx/src/emqx_cm.erl
+++ b/apps/emqx/src/emqx_cm.erl
@@ -650,7 +650,7 @@ init([]) ->
     TabOpts = [public, {write_concurrency, true}],
     ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]),
     ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]),
-    ok = emqx_tables:new(?CHAN_INFO_TAB, [set, compressed | TabOpts]),
+    ok = emqx_tables:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]),
     ok = emqx_tables:new(?CHAN_LIVE_TAB, [set, {write_concurrency, true} | TabOpts]),
     ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0),
     State = #{chan_pmon => emqx_pmon:new()},

From 6d9e1e0d7ac4025ba8c562dedd5024b0298b8a77 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Fri, 18 Nov 2022 18:21:03 +0800
Subject: [PATCH 60/69] test(mgmt): cover emqx_mgmt_api:cluster_query

---
 apps/emqx/test/emqx_common_test_helpers.erl   |  37 +++++-
 apps/emqx_management/src/emqx_mgmt_api.erl    |   6 +-
 .../test/emqx_mgmt_api_SUITE.erl              | 105 +++++++++++++++++-
 3 files changed, 142 insertions(+), 6 deletions(-)

diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index f195e083c..2d51f6f14 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -519,21 +519,51 @@ ensure_quic_listener(Name, UdpPort) ->
 %% Clusterisation and multi-node testing
 %%
 
+-type cluster_spec() :: [node_spec()].
+-type node_spec() :: role() | {role(), shortname()} | {role(), shortname(), node_opts()}.
+-type role() :: core | replicant.
+-type shortname() :: atom().
+-type nodename() :: atom().
+-type node_opts() :: #{
+    %% Need to loaded apps. These apps will be loaded once the node started
+    load_apps => list(),
+    %% Need to started apps. It is the first arg passed to emqx_common_test_helpers:start_apps/2
+    apps => list(),
+    %% Extras app starting handler. It is the second arg passed to emqx_common_test_helpers:start_apps/2
+    env_handler => fun((AppName :: atom()) -> term()),
+    %% Application env preset before calling `emqx_common_test_helpers:start_apps/2`
+    env => {AppName :: atom(), Key :: atom(), Val :: term()},
+    %% Whether to execute `emqx_config:init_load(SchemaMod)`
+    %% default: true
+    load_schema => boolean(),
+    %% Eval by emqx_config:put/2
+    conf => [{KeyPath :: list(), Val :: term()}],
+    %% Fast option to config listener port
+    %% default rule:
+    %% - tcp: base_port
+    %% - ssl: base_port + 1
+    %% - ws : base_port + 3
+    %% - wss: base_port + 4
+    listener_ports => [{Type :: tcp | ssl | ws | wss, inet:port_number()}]
+}.
+
+-spec emqx_cluster(cluster_spec()) -> [{shortname(), node_opts()}].
 emqx_cluster(Specs) ->
     emqx_cluster(Specs, #{}).
 
+-spec emqx_cluster(cluster_spec(), node_opts()) -> [{shortname(), node_opts()}].
 emqx_cluster(Specs, CommonOpts) when is_list(CommonOpts) ->
     emqx_cluster(Specs, maps:from_list(CommonOpts));
 emqx_cluster(Specs0, CommonOpts) ->
     Specs1 = lists:zip(Specs0, lists:seq(1, length(Specs0))),
     Specs = expand_node_specs(Specs1, CommonOpts),
-    CoreNodes = [node_name(Name) || {{core, Name, _}, _} <- Specs],
-    %% Assign grpc ports:
+    %% Assign grpc ports
     GenRpcPorts = maps:from_list([
         {node_name(Name), {tcp, gen_rpc_port(base_port(Num))}}
      || {{_, Name, _}, Num} <- Specs
     ]),
     %% Set the default node of the cluster:
+    CoreNodes = [node_name(Name) || {{core, Name, _}, _} <- Specs],
     JoinTo =
         case CoreNodes of
             [First | _] -> First;
@@ -554,6 +584,8 @@ emqx_cluster(Specs0, CommonOpts) ->
     ].
 
 %% Lower level starting API
+
+-spec start_slave(shortname(), node_opts()) -> nodename().
 start_slave(Name, Opts) ->
     {ok, Node} = ct_slave:start(
         list_to_atom(atom_to_list(Name) ++ "@" ++ host()),
@@ -590,6 +622,7 @@ epmd_path() ->
 
 %% Node initialization
 
+-spec setup_node(nodename(), node_opts()) -> ok.
 setup_node(Node, Opts) when is_list(Opts) ->
     setup_node(Node, maps:from_list(Opts));
 setup_node(Node, Opts) when is_map(Opts) ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 9f757b0cc..9646eb747 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -314,7 +314,9 @@ do_select(
             ?FRESH_SELECT ->
                 ets:select(Tab, Ms, Limit);
             _ ->
-                ets:select(Continuation)
+                %% XXX: Repair is necessary because we pass Continuation back
+                %% and forth through the nodes in the `do_cluster_query`
+                ets:select(ets:repair_continuation(Continuation, Ms))
         end,
     case Result of
         '$end_of_table' ->
@@ -508,7 +510,7 @@ format_query_result(
         %% queries that can be read
         meta => Meta#{count => Total},
         data => lists:flatten(
-            lists:foldr(
+            lists:foldl(
                 fun({Node, Rows}, Acc) ->
                     [lists:map(fun(Row) -> exec_format_fun(FmtFun, Node, Row) end, Rows) | Acc]
                 end,
diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
index c9abef375..a065b9c83 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -28,15 +28,116 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_mgmt_api_test_util:init_suite(),
     Config.
 
 end_per_suite(_) ->
-    emqx_mgmt_api_test_util:end_suite().
+    ok.
 
 %%--------------------------------------------------------------------
 %% cases
 %%--------------------------------------------------------------------
 
 t_cluster_query(_Config) ->
+    net_kernel:start(['master@127.0.0.1', longnames]),
+    ct:timetrap({seconds, 120}),
+    snabbkaffe:fix_ct_logging(),
+    [{Name, Opts}, {Name1, Opts1}] = cluster_specs(),
+    Node1 = emqx_common_test_helpers:start_slave(Name, Opts),
+    Node2 = emqx_common_test_helpers:start_slave(Name1, Opts1),
+    try
+        process_flag(trap_exit, true),
+        ClientLs1 = [start_emqtt_client(Node1, I, 2883) || I <- lists:seq(1, 10)],
+        ClientLs2 = [start_emqtt_client(Node2, I, 3883) || I <- lists:seq(1, 10)],
+
+        %% returned list should be the same regardless of which node is requested
+        {200, ClientsAll} = query_clients(Node1, #{}),
+        ?assertEqual({200, ClientsAll}, query_clients(Node2, #{})),
+        ?assertMatch(
+            #{page := 1, limit := 100, count := 20},
+            maps:get(meta, ClientsAll)
+        ),
+        ?assertMatch(20, length(maps:get(data, ClientsAll))),
+        %% query the first page, counting in entire cluster
+        {200, ClientsPage1} = query_clients(Node1, #{<<"limit">> => 5}),
+        ?assertMatch(
+            #{page := 1, limit := 5, count := 20},
+            maps:get(meta, ClientsPage1)
+        ),
+        ?assertMatch(5, length(maps:get(data, ClientsPage1))),
+
+        %% assert: AllPage = Page1 + Page2 + Page3 + Page4
+        %% !!!Note: this equation requires that the queried tables must be ordered_set
+        {200, ClientsPage2} = query_clients(Node1, #{<<"page">> => 2, <<"limit">> => 5}),
+        {200, ClientsPage3} = query_clients(Node2, #{<<"page">> => 3, <<"limit">> => 5}),
+        {200, ClientsPage4} = query_clients(Node1, #{<<"page">> => 4, <<"limit">> => 5}),
+        GetClientIds = fun(L) -> lists:map(fun(#{clientid := Id}) -> Id end, L) end,
+        ?assertEqual(
+            GetClientIds(maps:get(data, ClientsAll)),
+            GetClientIds(
+                maps:get(data, ClientsPage1) ++ maps:get(data, ClientsPage2) ++
+                    maps:get(data, ClientsPage3) ++ maps:get(data, ClientsPage4)
+            )
+        ),
+
+        %% exact match can return non-zero total
+        {200, ClientsNode1} = query_clients(Node2, #{<<"username">> => <<"corenode1@127.0.0.1">>}),
+        ?assertMatch(
+            #{count := 10},
+            maps:get(meta, ClientsNode1)
+        ),
+
+        %% fuzzy searching can't return total
+        {200, ClientsNode2} = query_clients(Node2, #{<<"like_username">> => <<"corenode2">>}),
+        ?assertMatch(
+            #{count := 0},
+            maps:get(meta, ClientsNode2)
+        ),
+        ?assertMatch(10, length(maps:get(data, ClientsNode2))),
+
+        _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1),
+        _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs2)
+    after
+        emqx_common_test_helpers:stop_slave(Node1),
+        emqx_common_test_helpers:stop_slave(Node2)
+    end,
     ok.
+
+%%--------------------------------------------------------------------
+%% helpers
+%%--------------------------------------------------------------------
+
+cluster_specs() ->
+    Specs =
+        %% default listeners port
+        [
+            {core, corenode1, #{listener_ports => [{tcp, 2883}]}},
+            {core, corenode2, #{listener_ports => [{tcp, 3883}]}}
+        ],
+    CommOpts =
+        [
+            {env, [{emqx, boot_modules, all}]},
+            {apps, []},
+            {conf, [
+                {[listeners, ssl, default, enabled], false},
+                {[listeners, ws, default, enabled], false},
+                {[listeners, wss, default, enabled], false}
+            ]}
+        ],
+    emqx_common_test_helpers:emqx_cluster(
+        Specs,
+        CommOpts
+    ).
+
+start_emqtt_client(Node0, N, Port) ->
+    Node = atom_to_binary(Node0),
+    ClientId = iolist_to_binary([Node, "-", integer_to_binary(N)]),
+    {ok, C} = emqtt:start_link([{clientid, ClientId}, {username, Node}, {port, Port}]),
+    {ok, _} = emqtt:connect(C),
+    C.
+
+query_clients(Node, Qs0) ->
+    Qs = maps:merge(
+        #{<<"page">> => 1, <<"limit">> => 100},
+        Qs0
+    ),
+    rpc:call(Node, emqx_mgmt_api_clients, clients, [get, #{query_string => Qs}]).

From ffb3f2d2d2c420becf5e25da8996410d77af6a33 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@163.com>
Date: Mon, 21 Nov 2022 17:09:27 +0800
Subject: [PATCH 61/69] chore: change emqx_live_connection tab type to
 ordered_set

Co-authored-by: Zaiming (Stone) Shi <zmstone@gmail.com>
---
 apps/emqx/src/emqx_cm.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl
index 483c51f26..5adc7a811 100644
--- a/apps/emqx/src/emqx_cm.erl
+++ b/apps/emqx/src/emqx_cm.erl
@@ -651,7 +651,7 @@ init([]) ->
     ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]),
     ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]),
     ok = emqx_tables:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]),
-    ok = emqx_tables:new(?CHAN_LIVE_TAB, [set, {write_concurrency, true} | TabOpts]),
+    ok = emqx_tables:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]),
     ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0),
     State = #{chan_pmon => emqx_pmon:new()},
     {ok, State}.

From 0122d3900f66b4d920644c90ce98c1a74c200a7b Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 22 Nov 2022 14:59:48 +0800
Subject: [PATCH 62/69] chore: update changes

---
 changes/v5.0.11-en.md | 2 ++
 changes/v5.0.11-zh.md | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/changes/v5.0.11-en.md b/changes/v5.0.11-en.md
index 5329be024..4b35aa5f4 100644
--- a/changes/v5.0.11-en.md
+++ b/changes/v5.0.11-en.md
@@ -27,6 +27,8 @@
 
 - Support message properties in `/publish` API [#9401](https://github.com/emqx/emqx/pull/9401).
 
+- Optimize client query performance for HTTP APIs [#9374](https://github.com/emqx/emqx/pull/9374).
+
 ## Bug fixes
 
 - Fix `ssl.existingName` option of  helm chart not working [#9307](https://github.com/emqx/emqx/issues/9307).
diff --git a/changes/v5.0.11-zh.md b/changes/v5.0.11-zh.md
index e0c82b022..ab9f5a0d4 100644
--- a/changes/v5.0.11-zh.md
+++ b/changes/v5.0.11-zh.md
@@ -25,6 +25,8 @@
 
 - 支持在 /publish API 中添加消息属性 [#9401](https://github.com/emqx/emqx/pull/9401)。
 
+- 优化查询客户端列表的 HTTP API 性能 [#9374](https://github.com/emqx/emqx/pull/9374)。
+
 ## 修复
 
 - 修复 helm chart 的 `ssl.existingName` 选项不起作用 [#9307](https://github.com/emqx/emqx/issues/9307)。

From 9786a6c26713ea38612306fc0c590a0a7e9af46a Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 22 Nov 2022 15:33:42 +0800
Subject: [PATCH 63/69] refactor(mgmt): convert fuzzy filter func to named func

---
 .../emqx_enhanced_authn_scram_mnesia.erl      |  8 ++----
 .../src/simple_authn/emqx_authn_mnesia.erl    |  8 ++----
 apps/emqx_authz/src/emqx_authz_api_mnesia.erl |  8 ++----
 .../src/emqx_gateway_api_clients.erl          |  8 ++----
 apps/emqx_management/src/emqx_mgmt_api.erl    | 26 +++++++++++++++----
 .../src/emqx_mgmt_api_clients.erl             |  8 ++----
 .../src/emqx_mgmt_api_subscriptions.erl       |  8 ++----
 .../src/emqx_rule_engine_api.erl              |  9 ++-----
 8 files changed, 35 insertions(+), 48 deletions(-)

diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
index dfa2f32ef..66e19efd2 100644
--- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
+++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
@@ -48,6 +48,7 @@
 
 -export([
     qs2ms/2,
+    run_fuzzy_filter/2,
     format_user_info/1,
     group_match_spec/1
 ]).
@@ -281,12 +282,7 @@ qs2ms(_Tab, {QString, Fuzzy}) ->
 fuzzy_filter_fun([]) ->
     undefined;
 fuzzy_filter_fun(Fuzzy) ->
-    fun(MsRaws) when is_list(MsRaws) ->
-        lists:filter(
-            fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
-            MsRaws
-        )
-    end.
+    {fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
 
 run_fuzzy_filter(_, []) ->
     true;
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
index 50edb6612..930a03b6a 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
@@ -50,6 +50,7 @@
 
 -export([
     qs2ms/2,
+    run_fuzzy_filter/2,
     format_user_info/1,
     group_match_spec/1
 ]).
@@ -307,12 +308,7 @@ qs2ms(_Tab, {QString, FuzzyQString}) ->
 fuzzy_filter_fun([]) ->
     undefined;
 fuzzy_filter_fun(Fuzzy) ->
-    fun(MsRaws) when is_list(MsRaws) ->
-        lists:filter(
-            fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
-            MsRaws
-        )
-    end.
+    {fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
 
 run_fuzzy_filter(_, []) ->
     true;
diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
index 1ab7feea8..f480934a0 100644
--- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
@@ -51,6 +51,7 @@
 -export([
     query_username/2,
     query_clientid/2,
+    run_fuzzy_filter/2,
     format_result/1
 ]).
 
@@ -589,12 +590,7 @@ query_clientid(_Tab, {_QString, FuzzyQString}) ->
 fuzzy_filter_fun([]) ->
     undefined;
 fuzzy_filter_fun(Fuzzy) ->
-    fun(MsRaws) when is_list(MsRaws) ->
-        lists:filter(
-            fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
-            MsRaws
-        )
-    end.
+    {fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
 
 run_fuzzy_filter(_, []) ->
     true;
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
index ba0f34206..646e6b6fd 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
@@ -56,6 +56,7 @@
 %% internal exports (for client query)
 -export([
     qs2ms/2,
+    run_fuzzy_filter/2,
     format_channel_info/1,
     format_channel_info/2
 ]).
@@ -327,12 +328,7 @@ ms(lifetime, X) ->
 fuzzy_filter_fun([]) ->
     undefined;
 fuzzy_filter_fun(Fuzzy) ->
-    fun(MsRaws) when is_list(MsRaws) ->
-        lists:filter(
-            fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
-            MsRaws
-        )
-    end.
+    {fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
 
 run_fuzzy_filter(_, []) ->
     true;
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 9646eb747..ff738a15a 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -128,7 +128,9 @@ limit(Params) ->
 ].
 
 -type query_to_match_spec_fun() ::
-    fun((list(), list()) -> {ets:match_spec(), fun()}).
+    fun((list(), list()) -> {ets:match_spec(), fuzzy_filter_fun()}).
+
+-type fuzzy_filter_fun() :: undefined | {fun(), list()}.
 
 -type format_result_fun() ::
     fun((node(), term()) -> term())
@@ -269,6 +271,15 @@ collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc
 %%    }
 init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) ->
     {Ms, FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
+    %% assert FuzzyFun type
+    _ =
+        case FuzzyFun of
+            undefined ->
+                ok;
+            {NamedFun, Args} ->
+                true = is_list(Args),
+                {type, external} = erlang:fun_info(NamedFun, type)
+        end,
     #{
         page => Page,
         limit => Limit,
@@ -323,9 +334,14 @@ do_select(
             {[], QueryState#{continuation => ?FRESH_SELECT}};
         {Rows, NContinuation} ->
             NRows =
-                case is_function(FuzzyFun) of
-                    true -> FuzzyFun(Rows);
-                    false -> Rows
+                case FuzzyFun of
+                    undefined ->
+                        Rows;
+                    {FilterFun, Args0} when is_function(FilterFun), is_list(Args0) ->
+                        lists:filter(
+                            fun(E) -> erlang:apply(FilterFun, [E | Args0]) end,
+                            Rows
+                        )
                 end,
             {NRows, QueryState#{continuation => NContinuation}}
     end.
@@ -361,7 +377,7 @@ counting_total_fun(_QueryState = #{mactch_spec := Ms, fuzzy_fun := undefined}) -
     fun(Tab) ->
         ets:select_count(Tab, CountingMs)
     end;
-counting_total_fun(_QueryState = #{fuzzy_fun := FuzzyFun}) when is_function(FuzzyFun) ->
+counting_total_fun(_QueryState = #{fuzzy_fun := FuzzyFun}) when FuzzyFun =/= undefined ->
     %% XXX: Calculating the total number for a fuzzy searching is very very expensive
     %% so it is not supported now
     false.
diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index cec9b9889..1f6928f67 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -47,6 +47,7 @@
 
 -export([
     qs2ms/2,
+    run_fuzzy_filter/2,
     format_channel_info/1,
     format_channel_info/2
 ]).
@@ -841,12 +842,7 @@ ms(created_at, X) ->
 fuzzy_filter_fun([]) ->
     undefined;
 fuzzy_filter_fun(Fuzzy) ->
-    fun(MsRaws) when is_list(MsRaws) ->
-        lists:filter(
-            fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
-            MsRaws
-        )
-    end.
+    {fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
 
 run_fuzzy_filter(_, []) ->
     true;
diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
index cb88742c0..d39f34207 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
@@ -33,6 +33,7 @@
 
 -export([
     qs2ms/2,
+    run_fuzzy_filter/2,
     format/2
 ]).
 
@@ -214,12 +215,7 @@ update_ms(qos, X, {{Pid, Topic}, Opts}) ->
 fuzzy_filter_fun([]) ->
     undefined;
 fuzzy_filter_fun(Fuzzy) ->
-    fun(MsRaws) when is_list(MsRaws) ->
-        lists:filter(
-            fun(E) -> run_fuzzy_filter(E, Fuzzy) end,
-            MsRaws
-        )
-    end.
+    {fun ?MODULE:run_fuzzy_filter/2, [Fuzzy]}.
 
 run_fuzzy_filter(_, []) ->
     true;
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index b34d0ceae..b4901c1a0 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -34,7 +34,7 @@
 -export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]).
 
 %% query callback
--export([qs2ms/2, format_rule_resp/1]).
+-export([qs2ms/2, run_fuzzy_match/2, format_rule_resp/1]).
 
 -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))).
 -define(ERR_BADARGS(REASON), begin
@@ -582,12 +582,7 @@ ms(enable, X) ->
 fuzzy_match_fun([]) ->
     undefined;
 fuzzy_match_fun(Fuzzy) ->
-    fun(MsRaws) when is_list(MsRaws) ->
-        lists:filter(
-            fun(E) -> run_fuzzy_match(E, Fuzzy) end,
-            MsRaws
-        )
-    end.
+    {fun ?MODULE:run_fuzzy_match/2, [Fuzzy]}.
 
 run_fuzzy_match(_, []) ->
     true;

From edb35c08a8732442742377c7491a25112ee7a2aa Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 24 Nov 2022 15:02:16 +0800
Subject: [PATCH 64/69] chore: refactor ms2qs function type

---
 .../enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl   | 7 +++++--
 apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl    | 7 +++++--
 apps/emqx_gateway/src/emqx_gateway_api_clients.erl        | 4 ++--
 apps/emqx_management/src/emqx_mgmt_api.erl                | 7 ++++---
 apps/emqx_management/src/emqx_mgmt_api_alarms.erl         | 4 ++--
 apps/emqx_management/src/emqx_mgmt_api_clients.erl        | 7 +++++--
 apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl  | 4 ++--
 apps/emqx_management/src/emqx_mgmt_api_topics.erl         | 6 +++++-
 apps/emqx_modules/src/emqx_delayed.erl                    | 7 +++++--
 apps/emqx_rule_engine/src/emqx_rule_engine_api.erl        | 8 ++++++--
 10 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
index 66e19efd2..ddc1bb464 100644
--- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
+++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl
@@ -274,9 +274,12 @@ list_users(QueryString, #{user_group := UserGroup}) ->
 %%--------------------------------------------------------------------
 %% QueryString to MatchSpec
 
--spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 qs2ms(_Tab, {QString, Fuzzy}) ->
-    {ms_from_qstring(QString), fuzzy_filter_fun(Fuzzy)}.
+    #{
+        match_spec => ms_from_qstring(QString),
+        fuzzy_fun => fuzzy_filter_fun(Fuzzy)
+    }.
 
 %% Fuzzy username funcs
 fuzzy_filter_fun([]) ->
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
index 930a03b6a..415d23c25 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl
@@ -300,9 +300,12 @@ list_users(QueryString, #{user_group := UserGroup}) ->
 %%--------------------------------------------------------------------
 %% QueryString to MatchSpec
 
--spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 qs2ms(_Tab, {QString, FuzzyQString}) ->
-    {ms_from_qstring(QString), fuzzy_filter_fun(FuzzyQString)}.
+    #{
+        match_spec => ms_from_qstring(QString),
+        fuzzy_fun => fuzzy_filter_fun(FuzzyQString)
+    }.
 
 %% Fuzzy username funcs
 fuzzy_filter_fun([]) ->
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
index 646e6b6fd..d219d07cd 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
@@ -268,9 +268,9 @@ extra_sub_props(Props) ->
 %%--------------------------------------------------------------------
 %% QueryString to MatchSpec
 
--spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 qs2ms(_Tab, {Qs, Fuzzy}) ->
-    {qs2ms(Qs), fuzzy_filter_fun(Fuzzy)}.
+    #{match_spec => qs2ms(Qs), fuzzy_fun => fuzzy_filter_fun(Fuzzy)}.
 
 qs2ms(Qs) ->
     {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}),
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index ff738a15a..5f8e39f13 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -127,8 +127,9 @@ limit(Params) ->
     {Key :: binary(), Type :: atom | binary | integer | timestamp | ip | ip_port}
 ].
 
--type query_to_match_spec_fun() ::
-    fun((list(), list()) -> {ets:match_spec(), fuzzy_filter_fun()}).
+-type query_to_match_spec_fun() :: fun((list(), list()) -> match_spec_and_filter()).
+
+-type match_spec_and_filter() :: #{match_spec := ets:match_spec(), fuzzy_fun := fuzzy_filter_fun()}.
 
 -type fuzzy_filter_fun() :: undefined | {fun(), list()}.
 
@@ -270,7 +271,7 @@ collect_total_from_tail_nodes(Nodes, QueryState, ResultAcc = #{total := TotalAcc
 %%    msfun := query_to_match_spec_fun()
 %%    }
 init_query_state(Tab, QString, MsFun, _Meta = #{page := Page, limit := Limit}) ->
-    {Ms, FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
+    #{match_spec := Ms, fuzzy_fun := FuzzyFun} = erlang:apply(MsFun, [Tab, QString]),
     %% assert FuzzyFun type
     _ =
         case FuzzyFun of
diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl
index 895a5bcf8..80f93ffe4 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl
@@ -136,9 +136,9 @@ alarms(delete, _Params) ->
 %%%==============================================================================================
 %% internal
 
--spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 qs2ms(_Tab, {_Qs, _Fuzzy}) ->
-    {[{'$1', [], ['$1']}], undefined}.
+    #{match_spec => [{'$1', [], ['$1']}], fuzzy_fun => undefined}.
 
 format_alarm(WhichNode, Alarm) ->
     emqx_alarm:format(WhichNode, Alarm).
diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index 1f6928f67..588031fc1 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -786,9 +786,12 @@ do_unsubscribe(ClientID, Topic) ->
 %%--------------------------------------------------------------------
 %% QueryString to Match Spec
 
--spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 qs2ms(_Tab, {QString, FuzzyQString}) ->
-    {qs2ms(QString), fuzzy_filter_fun(FuzzyQString)}.
+    #{
+        match_spec => qs2ms(QString),
+        fuzzy_fun => fuzzy_filter_fun(FuzzyQString)
+    }.
 
 -spec qs2ms(list()) -> ets:match_spec().
 qs2ms(Qs) ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
index d39f34207..c05878c6d 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
@@ -190,9 +190,9 @@ get_topic(Topic, _) ->
 %% QueryString to MatchSpec
 %%--------------------------------------------------------------------
 
--spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 qs2ms(_Tab, {Qs, Fuzzy}) ->
-    {gen_match_spec(Qs), fuzzy_filter_fun(Fuzzy)}.
+    #{match_spec => gen_match_spec(Qs), fuzzy_fun => fuzzy_filter_fun(Fuzzy)}.
 
 gen_match_spec(Qs) ->
     MtchHead = gen_match_spec(Qs, {{'_', '_'}, #{}}),
diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl
index ca707ba20..5e2bd1ea0 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl
@@ -143,8 +143,12 @@ generate_topic(Params = #{topic := Topic}) ->
 generate_topic(Params) ->
     Params.
 
+-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 qs2ms(_Tab, {Qs, _}) ->
-    {gen_match_spec(Qs, [{{route, '_', '_'}, [], ['$_']}]), undefined}.
+    #{
+        match_spec => gen_match_spec(Qs, [{{route, '_', '_'}, [], ['$_']}]),
+        fuzzy_fun => undefined
+    }.
 
 gen_match_spec([], Res) ->
     Res;
diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl
index c96e5a452..241ac5a8a 100644
--- a/apps/emqx_modules/src/emqx_delayed.erl
+++ b/apps/emqx_modules/src/emqx_delayed.erl
@@ -178,9 +178,12 @@ cluster_list(Params) ->
         fun ?MODULE:format_delayed/2
     ).
 
--spec qs2ms(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 qs2ms(_Table, {_Qs, _Fuzzy}) ->
-    {[{'$1', [], ['$1']}], undefined}.
+    #{
+        match_spec => [{'$1', [], ['$1']}],
+        fuzzy_fun => undefined
+    }.
 
 format_delayed(Delayed) ->
     format_delayed(node(), Delayed).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index b4901c1a0..01d819a69 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -553,12 +553,16 @@ filter_out_request_body(Conf) ->
     ],
     maps:without(ExtraConfs, Conf).
 
+-spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 qs2ms(_Tab, {Qs, Fuzzy}) ->
     case lists:keytake(from, 1, Qs) of
         false ->
-            {generate_match_spec(Qs), fuzzy_match_fun(Fuzzy)};
+            #{match_spec => generate_match_spec(Qs), fuzzy_fun => fuzzy_match_fun(Fuzzy)};
         {value, {from, '=:=', From}, Ls} ->
-            {generate_match_spec(Ls), fuzzy_match_fun([{from, '=:=', From} | Fuzzy])}
+            #{
+                match_spec => generate_match_spec(Ls),
+                fuzzy_fun => fuzzy_match_fun([{from, '=:=', From} | Fuzzy])
+            }
     end.
 
 generate_match_spec(Qs) ->

From dbdb78d38abf81da00aea801680da680a768f622 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Thu, 24 Nov 2022 15:24:55 +0800
Subject: [PATCH 65/69] chore: clarify the case when count returns zero

---
 apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
index 5ea2e0773..5af1aee89 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
@@ -138,7 +138,12 @@ fields(limit) ->
     Meta = #{in => query, desc => Desc, default => ?DEFAULT_ROW, example => 50},
     [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}];
 fields(count) ->
-    Meta = #{desc => <<"Results count.">>, required => true},
+    Desc = <<
+        "Total number of records counted.<br/>"
+        "Note: this field is <code>0</code> when the queryed table is empty, "
+        "or if the query can not be optimized and requires a full table scan."
+    >>,
+    Meta = #{desc => Desc, required => true},
     [{count, hoconsc:mk(non_neg_integer(), Meta)}];
 fields(meta) ->
     fields(page) ++ fields(limit) ++ fields(count).

From 6ee475d9b12a311da2ff693e4e80532a77355e6f Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 24 Nov 2022 17:56:14 +0100
Subject: [PATCH 66/69] fix(emqx_authz_api_mnesia): return the right matchers

---
 apps/emqx_authz/src/emqx_authz_api_mnesia.erl | 14 +++++--
 apps/emqx_management/src/emqx_mgmt_api.erl    | 40 ++++++++++---------
 2 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
index f480934a0..090f9d1e2 100644
--- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl
@@ -578,13 +578,19 @@ purge(delete, _) ->
 %%--------------------------------------------------------------------
 %% QueryString to MatchSpec
 
--spec query_username(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+-spec query_username(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 query_username(_Tab, {_QString, FuzzyQString}) ->
-    {emqx_authz_mnesia:list_username_rules(), fuzzy_filter_fun(FuzzyQString)}.
+    #{
+        match_spec => emqx_authz_mnesia:list_username_rules(),
+        fuzzy_fun => fuzzy_filter_fun(FuzzyQString)
+    }.
 
--spec query_clientid(atom(), {list(), list()}) -> {ets:match_spec(), fun() | undefined}.
+-spec query_clientid(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
 query_clientid(_Tab, {_QString, FuzzyQString}) ->
-    {emqx_authz_mnesia:list_clientid_rules(), fuzzy_filter_fun(FuzzyQString)}.
+    #{
+        match_spec => emqx_authz_mnesia:list_clientid_rules(),
+        fuzzy_fun => fuzzy_filter_fun(FuzzyQString)
+    }.
 
 %% Fuzzy username funcs
 fuzzy_filter_fun([]) ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 5f8e39f13..9f69ed3f0 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -35,6 +35,28 @@
     b2i/1
 ]).
 
+-export_type([
+    match_spec_and_filter/0
+]).
+
+-type query_params() :: list() | map().
+
+-type query_schema() :: [
+    {Key :: binary(), Type :: atom | binary | integer | timestamp | ip | ip_port}
+].
+
+-type query_to_match_spec_fun() :: fun((list(), list()) -> match_spec_and_filter()).
+
+-type match_spec_and_filter() :: #{match_spec := ets:match_spec(), fuzzy_fun := fuzzy_filter_fun()}.
+
+-type fuzzy_filter_fun() :: undefined | {fun(), list()}.
+
+-type format_result_fun() ::
+    fun((node(), term()) -> term())
+    | fun((term()) -> term()).
+
+-type query_return() :: #{meta := map(), data := [term()]}.
+
 -export([do_query/2, apply_total_query/1]).
 
 paginate(Tables, Params, {Module, FormatFun}) ->
@@ -121,24 +143,6 @@ limit(Params) ->
 %% Node Query
 %%--------------------------------------------------------------------
 
--type query_params() :: list() | map().
-
--type query_schema() :: [
-    {Key :: binary(), Type :: atom | binary | integer | timestamp | ip | ip_port}
-].
-
--type query_to_match_spec_fun() :: fun((list(), list()) -> match_spec_and_filter()).
-
--type match_spec_and_filter() :: #{match_spec := ets:match_spec(), fuzzy_fun := fuzzy_filter_fun()}.
-
--type fuzzy_filter_fun() :: undefined | {fun(), list()}.
-
--type format_result_fun() ::
-    fun((node(), term()) -> term())
-    | fun((term()) -> term()).
-
--type query_return() :: #{meta := map(), data := [term()]}.
-
 -spec node_query(
     node(),
     atom(),

From 970b3eb1a076117ce6fb111eed2168774ded23f8 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sat, 26 Nov 2022 15:51:38 +0100
Subject: [PATCH 67/69] chore: bump to version v5.0.11-rc.1

---
 apps/emqx/include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index 1335ed700..66b81d352 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.10").
+-define(EMQX_RELEASE_CE, "5.0.11-rc.1").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.0.0-alpha.1").

From 9dcec9cd23860794afe225bbacbf7be8b2e83d6d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sun, 27 Nov 2022 09:24:37 +0100
Subject: [PATCH 68/69] chore: fix chart version

---
 deploy/charts/emqx/Chart.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml
index b42ca84d0..f3f33c984 100644
--- a/deploy/charts/emqx/Chart.yaml
+++ b/deploy/charts/emqx/Chart.yaml
@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.0.10
+version: 5.0.11
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.0.10
+appVersion: 5.0.11

From 05719daff296b21bcc503f1e74bd318967470bd3 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sun, 27 Nov 2022 11:15:28 +0100
Subject: [PATCH 69/69] chore: bump version to v5.0.11

---
 apps/emqx/include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index 66b81d352..1dfb1bd61 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.11-rc.1").
+-define(EMQX_RELEASE_CE, "5.0.11").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.0.0-alpha.1").