From 651f72eabcfcde7d4d60956eb159d95d59c91e31 Mon Sep 17 00:00:00 2001
From: Kinplemelon <wujiu529@gmail.com>
Date: Thu, 16 Mar 2023 11:39:29 +0800
Subject: [PATCH 01/52] docs: correct the chinese label for the
 `socket_nodelay` field

---
 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf
index c41b95c3a..716ddb13a 100644
--- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf
@@ -224,7 +224,7 @@ emqx_ee_bridge_kafka {
         }
         label {
             en: "No Delay"
-            zh: "是否延迟发送"
+            zh: "是否关闭延迟发送"
         }
     }
     kafka_topic {

From 20414d737323aa6a35906f49697fd72fbe990ef5 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 16 Mar 2023 13:33:52 -0300
Subject: [PATCH 02/52] fix(buffer_worker): check request timeout and health
 check interval

Fixes https://emqx.atlassian.net/browse/EMQX-9099

The default value for `request_timeout` is 15 seconds, and the default
resume interval is also 15 seconds (the health check timeout, if
`resume_interval` is not explicitly given).  This means that, in
practice, if a buffer worker ever gets into the blocked state, then
almost all requests will timeout.

Proposed improvement:

- `request_timeout` should by default be twice as much as
health_check_interval.
- Emit a alarm if `request_timeout` is not greater than
`health_check_interval`.
---
 .../i18n/emqx_resource_schema_i18n.conf       |   4 +-
 apps/emqx_resource/include/emqx_resource.hrl  |   5 +-
 .../src/emqx_resource_buffer_worker.erl       |  38 ++++++
 .../src/emqx_resource_manager.erl             |   1 +
 .../test/emqx_resource_SUITE.erl              | 126 ++++++++++++++++++
 changes/ce/fix-10154.en.md                    |   7 +
 6 files changed, 178 insertions(+), 3 deletions(-)
 create mode 100644 changes/ce/fix-10154.en.md

diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index fb6b2eb06..2e5cf96e8 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -102,8 +102,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise
 
   request_timeout {
     desc {
-      en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired."""
-      zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。"""
+      en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.  We recommend setting this timeout to be at least twice the health check interval, so that the buffer has the chance to recover if too many requests get enqueued."""
+      zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。我们建议将这个超时设置为健康检查间隔的至少两倍，这样，如果有太多的请求被排队，缓冲区就有机会恢复。"""
     }
     label {
       en: """Request Expiry"""
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index be570e694..8033ed660 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -91,7 +91,10 @@
 -define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024).
 -define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>).
 
--define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)).
+%% Note: this should be greater than the health check timeout;
+%% otherwise, if the buffer worker is ever blocked, than all queued
+%% requests will basically fail without being attempted.
+-define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(30)).
 
 %% count
 -define(DEFAULT_BATCH_SIZE, 1).
diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
index 8bfd77e61..05622bdd7 100644
--- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
@@ -56,6 +56,8 @@
 
 -export([clear_disk_queue_dir/2]).
 
+-export([deactivate_bad_request_timeout_alarm/1]).
+
 -elvis([{elvis_style, dont_repeat_yourself, disable}]).
 
 -define(COLLECT_REQ_LIMIT, 1000).
@@ -88,6 +90,8 @@
 -type queue_query() :: ?QUERY(reply_fun(), request(), HasBeenSent :: boolean(), expire_at()).
 -type request() :: term().
 -type request_from() :: undefined | gen_statem:from().
+-type request_timeout() :: infinity | timer:time().
+-type health_check_interval() :: timer:time().
 -type state() :: blocked | running.
 -type inflight_key() :: integer().
 -type data() :: #{
@@ -199,6 +203,7 @@ init({Id, Index, Opts}) ->
     RequestTimeout = maps:get(request_timeout, Opts, ?DEFAULT_REQUEST_TIMEOUT),
     BatchTime0 = maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
     BatchTime = adjust_batch_time(Id, RequestTimeout, BatchTime0),
+    maybe_toggle_bad_request_timeout_alarm(Id, RequestTimeout, HealthCheckInterval),
     Data = #{
         id => Id,
         index => Index,
@@ -1679,6 +1684,39 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) ->
     end,
     BatchTime.
 
+%% The request timeout should be greater than the health check
+%% timeout, health timeout defines how often the buffer worker tries
+%% to unblock. If request timeout is <= health check timeout and the
+%% buffer worker is ever blocked, than all queued requests will
+%% basically fail without being attempted.
+-spec maybe_toggle_bad_request_timeout_alarm(
+    resource_id(), request_timeout(), health_check_interval()
+) -> ok.
+maybe_toggle_bad_request_timeout_alarm(Id, _RequestTimeout = infinity, _HealthCheckInterval) ->
+    deactivate_bad_request_timeout_alarm(Id),
+    ok;
+maybe_toggle_bad_request_timeout_alarm(Id, RequestTimeout, HealthCheckInterval) ->
+    case RequestTimeout > HealthCheckInterval of
+        true ->
+            deactivate_bad_request_timeout_alarm(Id),
+            ok;
+        false ->
+            _ = emqx_alarm:activate(
+                bad_request_timeout_alarm_id(Id),
+                #{resource_id => Id, reason => bad_request_timeout},
+                <<"Request timeout should be greater than health check timeout: ", Id/binary>>
+            ),
+            ok
+    end.
+
+-spec deactivate_bad_request_timeout_alarm(resource_id()) -> ok.
+deactivate_bad_request_timeout_alarm(Id) ->
+    _ = emqx_alarm:ensure_deactivated(bad_request_timeout_alarm_id(Id)),
+    ok.
+
+bad_request_timeout_alarm_id(Id) ->
+    <<"bad_request_timeout:", Id/binary>>.
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 adjust_batch_time_test_() ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 40f9fe1ab..2bdc67a4d 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -506,6 +506,7 @@ handle_remove_event(From, ClearMetrics, Data) ->
         true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id);
         false -> ok
     end,
+    emqx_resource_buffer_worker:deactivate_bad_request_timeout_alarm(Data#data.id),
     {stop_and_reply, {shutdown, removed}, [{reply, From, ok}]}.
 
 start_resource(Data, From) ->
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index ff7e1d347..25f4a6d77 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -2554,6 +2554,132 @@ do_t_recursive_flush() ->
     ),
     ok.
 
+%% Check that we raise an alarm if a bad request timeout config is
+%% issued.  Request timeout should be greater than health check
+%% timeout.
+t_bad_request_timeout_alarm(_Config) ->
+    emqx_connector_demo:set_callback_mode(async_if_possible),
+
+    %% 1) Same values.
+    {ok, _} = emqx_resource:create(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource},
+        #{
+            query_mode => async,
+            request_timeout => 1_000,
+            health_check_interval => 1_000,
+            worker_pool_size => 2
+        }
+    ),
+    ExpectedMessage =
+        <<"Request timeout should be greater than health check timeout: ", ?ID/binary>>,
+    ?assertMatch(
+        [
+            #{
+                message := ExpectedMessage,
+                details := #{reason := bad_request_timeout, resource_id := ?ID},
+                deactivate_at := infinity
+            }
+        ],
+        emqx_alarm:get_alarms(activated)
+    ),
+    %% The unexpected termination of one of the buffer workers should
+    %% not turn the alarm off.
+    [Pid, _ | _] = emqx_resource_buffer_worker_sup:worker_pids(?ID),
+    MRef = monitor(process, Pid),
+    exit(Pid, kill),
+    receive
+        {'DOWN', MRef, process, Pid, _} ->
+            ok
+    after 300 ->
+        ct:fail("buffer worker didn't die")
+    end,
+    ?assertMatch(
+        [
+            #{
+                message := ExpectedMessage,
+                details := #{reason := bad_request_timeout, resource_id := ?ID},
+                deactivate_at := infinity
+            }
+        ],
+        emqx_alarm:get_alarms(activated)
+    ),
+    ok = emqx_resource:remove_local(?ID),
+    ?assertEqual([], emqx_alarm:get_alarms(activated)),
+
+    %% 2) Request timeout < health check interval.
+    {ok, _} = emqx_resource:create(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource},
+        #{
+            query_mode => async,
+            request_timeout => 999,
+            health_check_interval => 1_000,
+            worker_pool_size => 2
+        }
+    ),
+    ?assertMatch(
+        [
+            #{
+                message := ExpectedMessage,
+                details := #{reason := bad_request_timeout, resource_id := ?ID},
+                deactivate_at := infinity
+            }
+        ],
+        emqx_alarm:get_alarms(activated)
+    ),
+    ok = emqx_resource:remove_local(?ID),
+    ?assertEqual([], emqx_alarm:get_alarms(activated)),
+
+    %% 2) Request timeout < health check interval.
+    {ok, _} = emqx_resource:create(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource},
+        #{
+            query_mode => async,
+            request_timeout => 999,
+            health_check_interval => 1_000,
+            worker_pool_size => 2
+        }
+    ),
+    ?assertMatch(
+        [
+            #{
+                message := ExpectedMessage,
+                details := #{reason := bad_request_timeout, resource_id := ?ID},
+                deactivate_at := infinity
+            }
+        ],
+        emqx_alarm:get_alarms(activated)
+    ),
+    ok = emqx_resource:remove_local(?ID),
+    ?assertEqual([], emqx_alarm:get_alarms(activated)),
+
+    %% 3) Request timeout > health check interval.
+    {ok, _} = emqx_resource:create(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource},
+        #{
+            query_mode => async,
+            request_timeout => 1_001,
+            health_check_interval => 1_000,
+            worker_pool_size => 2
+        }
+    ),
+    ?assertEqual([], emqx_alarm:get_alarms(activated)),
+    ok = emqx_resource:remove_local(?ID),
+    ?assertEqual([], emqx_alarm:get_alarms(activated)),
+
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------
diff --git a/changes/ce/fix-10154.en.md b/changes/ce/fix-10154.en.md
new file mode 100644
index 000000000..83a729360
--- /dev/null
+++ b/changes/ce/fix-10154.en.md
@@ -0,0 +1,7 @@
+Change the default `request_timeout` for bridges and connectors to be
+twice the default `health_check_interval`.
+
+Before this change, the default values for those two options meant
+that, if a buffer ever got blocked due to resource errors or high
+message volumes, then, by the time the buffer would try to resume its
+normal operations, almost all requests would have timed out.

From 39f22585d00adb15e2ba59073df6c8a541e6a17d Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.dev>
Date: Sun, 19 Mar 2023 01:16:49 +0200
Subject: [PATCH 03/52] chore(mix): fix mix builds

---
 Makefile                      |  1 +
 apps/emqx/rebar.config        | 10 +++++-
 apps/emqx/rebar.config.script | 22 ++++++------
 build                         | 25 ++-----------
 mix.exs                       | 68 +++++++++++++++++++++++++++++++----
 5 files changed, 85 insertions(+), 41 deletions(-)

diff --git a/Makefile b/Makefile
index fe4e6fc68..babd66b85 100644
--- a/Makefile
+++ b/Makefile
@@ -152,6 +152,7 @@ $(PROFILES:%=clean-%):
 .PHONY: clean-all
 clean-all:
 	@rm -f rebar.lock
+	@rm -rf deps
 	@rm -rf _build
 
 .PHONY: deps-all
diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 229979f6c..cbd0da109 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -59,4 +59,12 @@
     {statistics, true}
 ]}.
 
-{project_plugins, [erlfmt]}.
+{project_plugins, [
+    {erlfmt, [
+        {files, [
+            "{src,include,test}/*.{hrl,erl,app.src}",
+            "rebar.config",
+            "rebar.config.script"
+        ]}
+    ]}
+]}.
diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index 0827570ff..4639cc0db 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -27,17 +27,17 @@ Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.
 Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.113"}}}.
 
 Dialyzer = fun(Config) ->
-                   {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
-                   {plt_extra_apps, OldExtra} = lists:keyfind(plt_extra_apps, 1, OldDialyzerConfig),
-                   Extra = OldExtra ++ [quicer || IsQuicSupp()],
-                   NewDialyzerConfig = [{plt_extra_apps, Extra} | OldDialyzerConfig],
-                   lists:keystore(
-                     dialyzer,
-                     1,
-                     Config,
-                     {dialyzer, NewDialyzerConfig}
-                    )
-           end.
+    {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
+    {plt_extra_apps, OldExtra} = lists:keyfind(plt_extra_apps, 1, OldDialyzerConfig),
+    Extra = OldExtra ++ [quicer || IsQuicSupp()],
+    NewDialyzerConfig = [{plt_extra_apps, Extra} | OldDialyzerConfig],
+    lists:keystore(
+        dialyzer,
+        1,
+        Config,
+        {dialyzer, NewDialyzerConfig}
+    )
+end.
 
 ExtraDeps = fun(C) ->
     {deps, Deps0} = lists:keyfind(deps, 1, C),
diff --git a/build b/build
index 76298f1ab..3c558c19a 100755
--- a/build
+++ b/build
@@ -147,7 +147,7 @@ make_rel() {
 
 make_elixir_rel() {
   ./scripts/pre-compile.sh "$PROFILE"
-  export_release_vars "$PROFILE"
+  export_elixir_release_vars "$PROFILE"
   # for some reason, this has to be run outside "do"...
   mix local.rebar --if-missing --force
   # shellcheck disable=SC1010
@@ -362,7 +362,7 @@ function join {
 
 # used to control the Elixir Mix Release output
 # see docstring in `mix.exs`
-export_release_vars() {
+export_elixir_release_vars() {
   local profile="$1"
   case "$profile" in
     emqx|emqx-enterprise)
@@ -376,27 +376,6 @@ export_release_vars() {
       exit 1
   esac
   export MIX_ENV="$profile"
-
-  local erl_opts=()
-
-  case "$(is_enterprise "$profile")" in
-    'yes')
-      erl_opts+=( "{d, 'EMQX_RELEASE_EDITION', ee}" )
-      ;;
-    'no')
-      erl_opts+=( "{d, 'EMQX_RELEASE_EDITION', ce}" )
-      ;;
-  esac
-
-  # At this time, Mix provides no easy way to pass `erl_opts' to
-  # dependencies.  The workaround is to set this variable before
-  # compiling the project, so that `emqx_release.erl' picks up
-  # `emqx_vsn' as if it was compiled by rebar3.
-  erl_opts+=( "{compile_info,[{emqx_vsn,\"${PKG_VSN}\"}]}" )
-  erl_opts+=( "{d,snk_kind,msg}" )
-
-  ERL_COMPILER_OPTIONS="[$(join , "${erl_opts[@]}")]"
-  export ERL_COMPILER_OPTIONS
 }
 
 log "building artifact=$ARTIFACT for profile=$PROFILE"
diff --git a/mix.exs b/mix.exs
index 42354f8dc..4a16d9cc5 100644
--- a/mix.exs
+++ b/mix.exs
@@ -31,16 +31,17 @@ defmodule EMQXUmbrella.MixProject do
 
   def project() do
     profile_info = check_profile!()
+    version = pkg_vsn()
 
     [
       app: :emqx_mix,
-      version: pkg_vsn(),
-      deps: deps(profile_info),
+      version: version,
+      deps: deps(profile_info, version),
       releases: releases()
     ]
   end
 
-  defp deps(profile_info) do
+  defp deps(profile_info, version) do
     # we need several overrides here because dependencies specify
     # other exact versions, and not ranges.
     [
@@ -61,7 +62,9 @@ defmodule EMQXUmbrella.MixProject do
       {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true},
       {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
-      {:emqtt, github: "emqx/emqtt", tag: "1.8.5", override: true},
+      # maybe forbid to fetch quicer
+      {:emqtt,
+       github: "emqx/emqtt", tag: "1.8.5", override: true, system_env: maybe_no_quic_env()},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
       {:observer_cli, "1.7.1"},
       {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
@@ -92,11 +95,15 @@ defmodule EMQXUmbrella.MixProject do
       {:gpb, "4.19.5", override: true, runtime: false},
       {:hackney, github: "benoitc/hackney", tag: "1.18.1", override: true}
     ] ++
-      umbrella_apps() ++
-      enterprise_apps(profile_info) ++
+      emqx_apps(profile_info, version) ++
       enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
   end
 
+  defp emqx_apps(profile_info, version) do
+    apps = umbrella_apps() ++ enterprise_apps(profile_info)
+    set_emqx_app_system_env(apps, profile_info, version)
+  end
+
   defp umbrella_apps() do
     "apps/*"
     |> Path.wildcard()
@@ -145,6 +152,46 @@ defmodule EMQXUmbrella.MixProject do
     []
   end
 
+  defp set_emqx_app_system_env(apps, profile_info, version) do
+    system_env = emqx_app_system_env(profile_info, version) ++ maybe_no_quic_env()
+
+    Enum.map(
+      apps,
+      fn {app, opts} ->
+        {app,
+         Keyword.update(
+           opts,
+           :system_env,
+           system_env,
+           &Keyword.merge(&1, system_env)
+         )}
+      end
+    )
+  end
+
+  def emqx_app_system_env(profile_info, version) do
+    erlc_options(profile_info, version)
+    |> dump_as_erl()
+    |> then(&[{"ERL_COMPILER_OPTIONS", &1}])
+  end
+
+  defp erlc_options(%{edition_type: edition_type}, version) do
+    [
+      :debug_info,
+      {:compile_info, [{:emqx_vsn, String.to_charlist(version)}]},
+      {:d, :EMQX_RELEASE_EDITION, erlang_edition(edition_type)},
+      {:d, :snk_kind, :msg}
+    ]
+  end
+
+  def maybe_no_quic_env() do
+    if not enable_quicer?() do
+      [{"BUILD_WITHOUT_QUIC", "true"}]
+    else
+      []
+    end
+  end
+
   defp releases() do
     [
       emqx: fn ->
@@ -804,4 +851,13 @@ defmodule EMQXUmbrella.MixProject do
         |> List.first()
     end
   end
+
+  defp dump_as_erl(term) do
+    term
+    |> then(&:io_lib.format("~0p", [&1]))
+    |> :erlang.iolist_to_binary()
+  end
+
+  defp erlang_edition(:community), do: :ce
+  defp erlang_edition(:enterprise), do: :ee
 end

From a2ee85acb8cad1b2747ffa1c1c486fb5703bc373 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 20 Mar 2023 13:26:00 +0100
Subject: [PATCH 04/52] build: add script to delete old change log files

---
 scripts/rel/delete-old-changelog.sh | 39 +++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100755 scripts/rel/delete-old-changelog.sh

diff --git a/scripts/rel/delete-old-changelog.sh b/scripts/rel/delete-old-changelog.sh
new file mode 100755
index 000000000..4b0f4db2f
--- /dev/null
+++ b/scripts/rel/delete-old-changelog.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+[ "${DEBUG:-0}" = 1 ] && set -x
+
+top_dir="$(git rev-parse --show-toplevel)"
+prev_ce_tag="$("$top_dir"/scripts/find-prev-rel-tag.sh 'emqx')"
+prev_ee_tag="$("$top_dir"/scripts/find-prev-rel-tag.sh 'emqx-enterprise')"
+
+## check if a file is included in the previous release
+is_released() {
+    file="$1"
+    prev_tag="$2"
+    # check if file exists in the previous release
+    if git show "$prev_tag:$file" >/dev/null 2>&1; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+## loop over files in $top_dir/changes/ce
+## and delete the ones that are included in the previous ce and ee releases
+while read -r file; do
+    if is_released "$file" "$prev_ce_tag" && is_released "$file" "$prev_ee_tag"; then
+        echo "deleting $file, released in $prev_ce_tag and $prev_ee_tag"
+        rm -f "$file"
+    fi
+done < <(find "$top_dir/changes/ce" -type f -name '*.md')
+
+## loop over files in $top_dir/changes/ee
+## and delete the ones taht are included in the previous ee release
+while read -r file; do
+    if is_released "$file" "$prev_ee_tag"; then
+        echo "deleting $file, released in $prev_ee_tag"
+        rm -f "$file"
+    fi
+done < <(find "$top_dir/changes/ee" -type f -name '*.md')

From 2c6f977cac5c2f8fe0c09d21818b7090d6656534 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 20 Mar 2023 13:32:28 +0100
Subject: [PATCH 05/52] chore: delete old changelogs

---
 changes/ce/feat-10019.en.md |  1 -
 changes/ce/feat-10019.zh.md |  1 -
 changes/ce/feat-10022.en.md |  1 -
 changes/ce/feat-10022.zh.md |  1 -
 changes/ce/feat-10059.en.md |  1 -
 changes/ce/feat-10059.zh.md |  1 -
 changes/ce/feat-10139.en.md |  3 ---
 changes/ce/feat-10139.zh.md |  3 ---
 changes/ce/feat-9213.en.md  |  1 -
 changes/ce/feat-9213.zh.md  |  1 -
 changes/ce/feat-9893.en.md  |  2 --
 changes/ce/feat-9893.zh.md  |  2 --
 changes/ce/feat-9949.en.md  |  2 --
 changes/ce/feat-9949.zh.md  |  1 -
 changes/ce/feat-9986.en.md  |  1 -
 changes/ce/feat-9986.zh.md  |  1 -
 changes/ce/fix-10009.en.md  |  1 -
 changes/ce/fix-10009.zh.md  |  1 -
 changes/ce/fix-10013.en.md  |  1 -
 changes/ce/fix-10013.zh.md  |  1 -
 changes/ce/fix-10014.en.md  |  1 -
 changes/ce/fix-10014.zh.md  |  1 -
 changes/ce/fix-10015.en.md  |  7 -------
 changes/ce/fix-10015.zh.md  |  4 ----
 changes/ce/fix-10020.en.md  |  1 -
 changes/ce/fix-10020.zh.md  |  1 -
 changes/ce/fix-10021.en.md  |  1 -
 changes/ce/fix-10021.zh.md  |  1 -
 changes/ce/fix-10027.en.md  |  2 --
 changes/ce/fix-10027.zh.md  |  2 --
 changes/ce/fix-10032.en.md  |  1 -
 changes/ce/fix-10032.zh.md  |  1 -
 changes/ce/fix-10037.en.md  |  2 --
 changes/ce/fix-10037.zh.md  |  2 --
 changes/ce/fix-10041.en.md  |  2 --
 changes/ce/fix-10041.zh.md  |  2 --
 changes/ce/fix-10042.en.md  |  5 -----
 changes/ce/fix-10042.zh.md  |  6 ------
 changes/ce/fix-10043.en.md  |  3 ---
 changes/ce/fix-10043.zh.md  |  3 ---
 changes/ce/fix-10044.en.md  |  1 -
 changes/ce/fix-10044.zh.md  |  1 -
 changes/ce/fix-10050.en.md  |  1 -
 changes/ce/fix-10050.zh.md  |  1 -
 changes/ce/fix-10052.en.md  | 12 ------------
 changes/ce/fix-10052.zh.md  | 11 -----------
 changes/ce/fix-10054.en.md  |  1 -
 changes/ce/fix-10054.zh.md  |  1 -
 changes/ce/fix-10055.en.md  |  1 -
 changes/ce/fix-10055.zh.md  |  1 -
 changes/ce/fix-10056.en.md  |  3 ---
 changes/ce/fix-10056.zh.md  |  3 ---
 changes/ce/fix-10058.en.md  |  7 -------
 changes/ce/fix-10058.zh.md  |  8 --------
 changes/ce/fix-10066.en.md  |  1 -
 changes/ce/fix-10066.zh.md  |  1 -
 changes/ce/fix-10074.en.md  |  1 -
 changes/ce/fix-10074.zh.md  |  1 -
 changes/ce/fix-10076.en.md  |  2 --
 changes/ce/fix-10076.zh.md  |  2 --
 changes/ce/fix-10078.en.md  |  2 --
 changes/ce/fix-10078.zh.md  |  2 --
 changes/ce/fix-10079.en.md  |  1 -
 changes/ce/fix-10079.zh.md  |  2 --
 changes/ce/fix-10084.en.md  |  3 ---
 changes/ce/fix-10084.zh.md  |  3 ---
 changes/ce/fix-10085.en.md  |  1 -
 changes/ce/fix-10085.zh.md  |  1 -
 changes/ce/fix-10086.en.md  |  4 ----
 changes/ce/fix-10086.zh.md  |  3 ---
 changes/ce/fix-10098.en.md  |  1 -
 changes/ce/fix-10098.zh.md  |  1 -
 changes/ce/fix-10100.en.md  |  2 --
 changes/ce/fix-10100.zh.md  |  2 --
 changes/ce/fix-10107.en.md  |  9 ---------
 changes/ce/fix-10107.zh.md  |  8 --------
 changes/ce/fix-10117.en.md  |  2 --
 changes/ce/fix-10118.en.md  |  4 ----
 changes/ce/fix-10118.zh.md  |  4 ----
 changes/ce/fix-10119.en.md  |  1 -
 changes/ce/fix-10119.zh.md  |  1 -
 changes/ce/fix-10124.en.md  |  1 -
 changes/ce/fix-10124.zh.md  |  1 -
 changes/ce/fix-10130.en.md  |  3 ---
 changes/ce/fix-10130.zh.md  |  3 ---
 changes/ce/fix-10132.en.md  |  1 -
 changes/ce/fix-10132.zh.md  |  1 -
 changes/ce/fix-10144.en.md  |  1 -
 changes/ce/fix-9939.en.md   |  3 ---
 changes/ce/fix-9939.zh.md   |  2 --
 changes/ce/fix-9958.en.md   |  1 -
 changes/ce/fix-9958.zh.md   |  1 -
 changes/ce/fix-9961.en.md   |  1 -
 changes/ce/fix-9961.zh.md   |  1 -
 changes/ce/fix-9974.en.md   |  2 --
 changes/ce/fix-9974.zh.md   |  2 --
 changes/ce/fix-9978.en.md   |  2 --
 changes/ce/fix-9978.zh.md   |  2 --
 changes/ce/fix-9997.en.md   |  1 -
 changes/ce/fix-9997.zh.md   |  1 -
 changes/ce/perf-9967.en.md  |  1 -
 changes/ce/perf-9967.zh.md  |  1 -
 changes/ce/perf-9998.en.md  |  1 -
 changes/ce/perf-9998.zh.md  |  1 -
 changes/ee/feat-10083.en.md |  1 -
 changes/ee/feat-10083.zh.md |  1 -
 changes/ee/feat-9564.en.md  |  2 --
 changes/ee/feat-9564.zh.md  |  2 --
 changes/ee/feat-9881.en.md  |  4 ----
 changes/ee/feat-9881.zh.md  |  3 ---
 changes/ee/feat-9932.en.md  |  1 -
 changes/ee/feat-9932.zh.md  |  1 -
 changes/ee/fix-10007.en.md  |  5 -----
 changes/ee/fix-10007.zh.md  |  3 ---
 changes/ee/fix-10087.en.md  |  2 --
 changes/ee/fix-10087.zh.md  |  2 --
 changes/ee/fix-10095.en.md  |  3 ---
 changes/ee/fix-10095.zh.md  |  1 -
 118 files changed, 257 deletions(-)
 delete mode 100644 changes/ce/feat-10019.en.md
 delete mode 100644 changes/ce/feat-10019.zh.md
 delete mode 100644 changes/ce/feat-10022.en.md
 delete mode 100644 changes/ce/feat-10022.zh.md
 delete mode 100644 changes/ce/feat-10059.en.md
 delete mode 100644 changes/ce/feat-10059.zh.md
 delete mode 100644 changes/ce/feat-10139.en.md
 delete mode 100644 changes/ce/feat-10139.zh.md
 delete mode 100644 changes/ce/feat-9213.en.md
 delete mode 100644 changes/ce/feat-9213.zh.md
 delete mode 100644 changes/ce/feat-9893.en.md
 delete mode 100644 changes/ce/feat-9893.zh.md
 delete mode 100644 changes/ce/feat-9949.en.md
 delete mode 100644 changes/ce/feat-9949.zh.md
 delete mode 100644 changes/ce/feat-9986.en.md
 delete mode 100644 changes/ce/feat-9986.zh.md
 delete mode 100644 changes/ce/fix-10009.en.md
 delete mode 100644 changes/ce/fix-10009.zh.md
 delete mode 100644 changes/ce/fix-10013.en.md
 delete mode 100644 changes/ce/fix-10013.zh.md
 delete mode 100644 changes/ce/fix-10014.en.md
 delete mode 100644 changes/ce/fix-10014.zh.md
 delete mode 100644 changes/ce/fix-10015.en.md
 delete mode 100644 changes/ce/fix-10015.zh.md
 delete mode 100644 changes/ce/fix-10020.en.md
 delete mode 100644 changes/ce/fix-10020.zh.md
 delete mode 100644 changes/ce/fix-10021.en.md
 delete mode 100644 changes/ce/fix-10021.zh.md
 delete mode 100644 changes/ce/fix-10027.en.md
 delete mode 100644 changes/ce/fix-10027.zh.md
 delete mode 100644 changes/ce/fix-10032.en.md
 delete mode 100644 changes/ce/fix-10032.zh.md
 delete mode 100644 changes/ce/fix-10037.en.md
 delete mode 100644 changes/ce/fix-10037.zh.md
 delete mode 100644 changes/ce/fix-10041.en.md
 delete mode 100644 changes/ce/fix-10041.zh.md
 delete mode 100644 changes/ce/fix-10042.en.md
 delete mode 100644 changes/ce/fix-10042.zh.md
 delete mode 100644 changes/ce/fix-10043.en.md
 delete mode 100644 changes/ce/fix-10043.zh.md
 delete mode 100644 changes/ce/fix-10044.en.md
 delete mode 100644 changes/ce/fix-10044.zh.md
 delete mode 100644 changes/ce/fix-10050.en.md
 delete mode 100644 changes/ce/fix-10050.zh.md
 delete mode 100644 changes/ce/fix-10052.en.md
 delete mode 100644 changes/ce/fix-10052.zh.md
 delete mode 100644 changes/ce/fix-10054.en.md
 delete mode 100644 changes/ce/fix-10054.zh.md
 delete mode 100644 changes/ce/fix-10055.en.md
 delete mode 100644 changes/ce/fix-10055.zh.md
 delete mode 100644 changes/ce/fix-10056.en.md
 delete mode 100644 changes/ce/fix-10056.zh.md
 delete mode 100644 changes/ce/fix-10058.en.md
 delete mode 100644 changes/ce/fix-10058.zh.md
 delete mode 100644 changes/ce/fix-10066.en.md
 delete mode 100644 changes/ce/fix-10066.zh.md
 delete mode 100644 changes/ce/fix-10074.en.md
 delete mode 100644 changes/ce/fix-10074.zh.md
 delete mode 100644 changes/ce/fix-10076.en.md
 delete mode 100644 changes/ce/fix-10076.zh.md
 delete mode 100644 changes/ce/fix-10078.en.md
 delete mode 100644 changes/ce/fix-10078.zh.md
 delete mode 100644 changes/ce/fix-10079.en.md
 delete mode 100644 changes/ce/fix-10079.zh.md
 delete mode 100644 changes/ce/fix-10084.en.md
 delete mode 100644 changes/ce/fix-10084.zh.md
 delete mode 100644 changes/ce/fix-10085.en.md
 delete mode 100644 changes/ce/fix-10085.zh.md
 delete mode 100644 changes/ce/fix-10086.en.md
 delete mode 100644 changes/ce/fix-10086.zh.md
 delete mode 100644 changes/ce/fix-10098.en.md
 delete mode 100644 changes/ce/fix-10098.zh.md
 delete mode 100644 changes/ce/fix-10100.en.md
 delete mode 100644 changes/ce/fix-10100.zh.md
 delete mode 100644 changes/ce/fix-10107.en.md
 delete mode 100644 changes/ce/fix-10107.zh.md
 delete mode 100644 changes/ce/fix-10117.en.md
 delete mode 100644 changes/ce/fix-10118.en.md
 delete mode 100644 changes/ce/fix-10118.zh.md
 delete mode 100644 changes/ce/fix-10119.en.md
 delete mode 100644 changes/ce/fix-10119.zh.md
 delete mode 100644 changes/ce/fix-10124.en.md
 delete mode 100644 changes/ce/fix-10124.zh.md
 delete mode 100644 changes/ce/fix-10130.en.md
 delete mode 100644 changes/ce/fix-10130.zh.md
 delete mode 100644 changes/ce/fix-10132.en.md
 delete mode 100644 changes/ce/fix-10132.zh.md
 delete mode 100644 changes/ce/fix-10144.en.md
 delete mode 100644 changes/ce/fix-9939.en.md
 delete mode 100644 changes/ce/fix-9939.zh.md
 delete mode 100644 changes/ce/fix-9958.en.md
 delete mode 100644 changes/ce/fix-9958.zh.md
 delete mode 100644 changes/ce/fix-9961.en.md
 delete mode 100644 changes/ce/fix-9961.zh.md
 delete mode 100644 changes/ce/fix-9974.en.md
 delete mode 100644 changes/ce/fix-9974.zh.md
 delete mode 100644 changes/ce/fix-9978.en.md
 delete mode 100644 changes/ce/fix-9978.zh.md
 delete mode 100644 changes/ce/fix-9997.en.md
 delete mode 100644 changes/ce/fix-9997.zh.md
 delete mode 100644 changes/ce/perf-9967.en.md
 delete mode 100644 changes/ce/perf-9967.zh.md
 delete mode 100644 changes/ce/perf-9998.en.md
 delete mode 100644 changes/ce/perf-9998.zh.md
 delete mode 100644 changes/ee/feat-10083.en.md
 delete mode 100644 changes/ee/feat-10083.zh.md
 delete mode 100644 changes/ee/feat-9564.en.md
 delete mode 100644 changes/ee/feat-9564.zh.md
 delete mode 100644 changes/ee/feat-9881.en.md
 delete mode 100644 changes/ee/feat-9881.zh.md
 delete mode 100644 changes/ee/feat-9932.en.md
 delete mode 100644 changes/ee/feat-9932.zh.md
 delete mode 100644 changes/ee/fix-10007.en.md
 delete mode 100644 changes/ee/fix-10007.zh.md
 delete mode 100644 changes/ee/fix-10087.en.md
 delete mode 100644 changes/ee/fix-10087.zh.md
 delete mode 100644 changes/ee/fix-10095.en.md
 delete mode 100644 changes/ee/fix-10095.zh.md

diff --git a/changes/ce/feat-10019.en.md b/changes/ce/feat-10019.en.md
deleted file mode 100644
index b6cc0381c..000000000
--- a/changes/ce/feat-10019.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Add low level tuning settings for QUIC listeners.
diff --git a/changes/ce/feat-10019.zh.md b/changes/ce/feat-10019.zh.md
deleted file mode 100644
index b0eb2a673..000000000
--- a/changes/ce/feat-10019.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-为 QUIC 监听器添加更多底层调优选项。
diff --git a/changes/ce/feat-10022.en.md b/changes/ce/feat-10022.en.md
deleted file mode 100644
index 61d027aa2..000000000
--- a/changes/ce/feat-10022.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Start releasing Rocky Linux 9 (compatible with Enterprise Linux 9) and MacOS 12 packages
diff --git a/changes/ce/feat-10022.zh.md b/changes/ce/feat-10022.zh.md
deleted file mode 100644
index 970704f55..000000000
--- a/changes/ce/feat-10022.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-开始发布Rocky Linux 9（与Enterprise Linux 9兼容）和MacOS 12软件包。
diff --git a/changes/ce/feat-10059.en.md b/changes/ce/feat-10059.en.md
deleted file mode 100644
index 2c4de015c..000000000
--- a/changes/ce/feat-10059.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Errors returned by rule engine API are formatted in a more human readable way rather than dumping the raw error including the stacktrace.
diff --git a/changes/ce/feat-10059.zh.md b/changes/ce/feat-10059.zh.md
deleted file mode 100644
index 99f8fe8ee..000000000
--- a/changes/ce/feat-10059.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-规则引擎 API 返回用户可读的错误信息而不是原始的栈追踪信息。
diff --git a/changes/ce/feat-10139.en.md b/changes/ce/feat-10139.en.md
deleted file mode 100644
index 3c49ceac3..000000000
--- a/changes/ce/feat-10139.en.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Add `extraVolumeMounts` to EMQX Helm Chart, it will have the ability to mount the user-own files into the EMQX instance, for example, ACL rule files as mentioned in [#9052](https://github.com/emqx/emqx/issues/9052)
-
-Done of [#10116](https://github.com/emqx/emqx/issues/10116)
diff --git a/changes/ce/feat-10139.zh.md b/changes/ce/feat-10139.zh.md
deleted file mode 100644
index f7920a127..000000000
--- a/changes/ce/feat-10139.zh.md
+++ /dev/null
@@ -1,3 +0,0 @@
-将 `extraVolumeMounts` 添加到 EMQX Helm Chart 中，它将能够挂载用户自己的文件到 EMQX 实例中，例如在 [#9052](https://github.com/emqx/emqx/issues/9052) 中提到的 ACL 规则文件。
-
-完成了 [#10116](https://github.com/emqx/emqx/issues/10116)
diff --git a/changes/ce/feat-9213.en.md b/changes/ce/feat-9213.en.md
deleted file mode 100644
index 3266ed836..000000000
--- a/changes/ce/feat-9213.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Add pod disruption budget to helm chart
diff --git a/changes/ce/feat-9213.zh.md b/changes/ce/feat-9213.zh.md
deleted file mode 100644
index 66cb2693e..000000000
--- a/changes/ce/feat-9213.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-在 Helm chart 中添加干扰预算 (disruption budget)。
diff --git a/changes/ce/feat-9893.en.md b/changes/ce/feat-9893.en.md
deleted file mode 100644
index 343c3794f..000000000
--- a/changes/ce/feat-9893.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-When connecting with the flag `clean_start=false`, EMQX will filter out messages that published by banned clients.
-Previously, the messages sent by banned clients may still be delivered to subscribers in this scenario.
diff --git a/changes/ce/feat-9893.zh.md b/changes/ce/feat-9893.zh.md
deleted file mode 100644
index 426439c3e..000000000
--- a/changes/ce/feat-9893.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-当使用 `clean_start=false` 标志连接时，EMQX 将会从消息队列中过滤出被封禁客户端发出的消息，使它们不能被下发给订阅者。
-此前被封禁客户端发出的消息仍可能在这一场景下被下发给订阅者。
diff --git a/changes/ce/feat-9949.en.md b/changes/ce/feat-9949.en.md
deleted file mode 100644
index 3ed9c30b2..000000000
--- a/changes/ce/feat-9949.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-QUIC transport Multistreams support and QUIC TLS cacert support.
-
diff --git a/changes/ce/feat-9949.zh.md b/changes/ce/feat-9949.zh.md
deleted file mode 100644
index 6efabac3f..000000000
--- a/changes/ce/feat-9949.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-QUIC 传输多流支持和 QUIC TLS cacert 支持。
diff --git a/changes/ce/feat-9986.en.md b/changes/ce/feat-9986.en.md
deleted file mode 100644
index ee7a6be71..000000000
--- a/changes/ce/feat-9986.en.md
+++ /dev/null
@@ -1 +0,0 @@
-For helm charts, add MQTT ingress bridge; and removed stale `mgmt` references.
diff --git a/changes/ce/feat-9986.zh.md b/changes/ce/feat-9986.zh.md
deleted file mode 100644
index a7f418587..000000000
--- a/changes/ce/feat-9986.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-在 helm chart 中新增了 MQTT 桥接 ingress 的配置参数；并删除了旧版本遗留的 `mgmt` 配置。
diff --git a/changes/ce/fix-10009.en.md b/changes/ce/fix-10009.en.md
deleted file mode 100644
index 37f33a958..000000000
--- a/changes/ce/fix-10009.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Validate `bytes` param to `GET /trace/:name/log` to not exceed signed 32bit integer.
diff --git a/changes/ce/fix-10009.zh.md b/changes/ce/fix-10009.zh.md
deleted file mode 100644
index bb55ea5b9..000000000
--- a/changes/ce/fix-10009.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-验证 `GET /trace/:name/log` 的 `bytes` 参数，使其不超过有符号的32位整数。
diff --git a/changes/ce/fix-10013.en.md b/changes/ce/fix-10013.en.md
deleted file mode 100644
index ed7fa21eb..000000000
--- a/changes/ce/fix-10013.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix return type structure for error case in API schema for `/gateways/:name/clients`.
diff --git a/changes/ce/fix-10013.zh.md b/changes/ce/fix-10013.zh.md
deleted file mode 100644
index 171b79538..000000000
--- a/changes/ce/fix-10013.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修复 API `/gateways/:name/clients` 返回值的类型结构错误。
diff --git a/changes/ce/fix-10014.en.md b/changes/ce/fix-10014.en.md
deleted file mode 100644
index d52452bf9..000000000
--- a/changes/ce/fix-10014.en.md
+++ /dev/null
@@ -1 +0,0 @@
-In dashboard API for `/monitor(_current)/nodes/:node` return `404` instead of `400` if node does not exist.
diff --git a/changes/ce/fix-10014.zh.md b/changes/ce/fix-10014.zh.md
deleted file mode 100644
index 5e6a1660f..000000000
--- a/changes/ce/fix-10014.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-如果 API 查询的节点不存在，将会返回 404 而不再是 400。
diff --git a/changes/ce/fix-10015.en.md b/changes/ce/fix-10015.en.md
deleted file mode 100644
index 5727a52cd..000000000
--- a/changes/ce/fix-10015.en.md
+++ /dev/null
@@ -1,7 +0,0 @@
-To prevent errors caused by an incorrect EMQX node cookie provided from an environment variable,
-we have implemented a fail-fast mechanism.
-Previously, when an incorrect cookie was provided, the command would still attempt to ping the node,
-leading to the error message 'Node xxx not responding to pings'.
-With the new implementation, if a mismatched cookie is detected,
-a message will be logged to indicate that the cookie is incorrect,
-and the command will terminate with an error code of 1 without trying to ping the node.
diff --git a/changes/ce/fix-10015.zh.md b/changes/ce/fix-10015.zh.md
deleted file mode 100644
index 0f58fa99c..000000000
--- a/changes/ce/fix-10015.zh.md
+++ /dev/null
@@ -1,4 +0,0 @@
-在 cookie 给错时，快速失败。
-在此修复前，即使 cookie 配置错误，emqx 命令仍然会尝试去 ping EMQX 节点，
-并得到一个 "Node xxx not responding to pings" 的错误。
-修复后，如果发现 cookie 不一致，立即打印不一致的错误信息并退出。
diff --git a/changes/ce/fix-10020.en.md b/changes/ce/fix-10020.en.md
deleted file mode 100644
index 73615804b..000000000
--- a/changes/ce/fix-10020.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix bridge metrics when running in async mode with batching enabled (`batch_size` > 1).
diff --git a/changes/ce/fix-10020.zh.md b/changes/ce/fix-10020.zh.md
deleted file mode 100644
index 2fce853e3..000000000
--- a/changes/ce/fix-10020.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修复使用异步和批量配置的桥接计数不准确的问题。
diff --git a/changes/ce/fix-10021.en.md b/changes/ce/fix-10021.en.md
deleted file mode 100644
index 28302da70..000000000
--- a/changes/ce/fix-10021.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix error message when the target node of `emqx_ctl cluster join` command is not running.
diff --git a/changes/ce/fix-10021.zh.md b/changes/ce/fix-10021.zh.md
deleted file mode 100644
index 6df64b76d..000000000
--- a/changes/ce/fix-10021.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修正当`emqx_ctl cluster join`命令的目标节点未运行时的错误信息。
diff --git a/changes/ce/fix-10027.en.md b/changes/ce/fix-10027.en.md
deleted file mode 100644
index 531da1c50..000000000
--- a/changes/ce/fix-10027.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Allow setting node name from `EMQX_NODE__NAME` when running in docker.
-Prior to this fix, only `EMQX_NODE_NAME` is allowed.
diff --git a/changes/ce/fix-10027.zh.md b/changes/ce/fix-10027.zh.md
deleted file mode 100644
index ee7055d6c..000000000
--- a/changes/ce/fix-10027.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-在 docker 中启动时，允许使用 `EMQX_NODE__NAME` 环境变量来配置节点名。
-在此修复前，只能使 `EMQX_NODE_NAME`。
diff --git a/changes/ce/fix-10032.en.md b/changes/ce/fix-10032.en.md
deleted file mode 100644
index bd730c96c..000000000
--- a/changes/ce/fix-10032.en.md
+++ /dev/null
@@ -1 +0,0 @@
-When resources on some nodes in the cluster are still in the 'initializing/connecting' state, the `bridges/` API will crash due to missing Metrics information for those resources. This fix will ignore resources that do not have Metrics information.
diff --git a/changes/ce/fix-10032.zh.md b/changes/ce/fix-10032.zh.md
deleted file mode 100644
index fc1fb38b6..000000000
--- a/changes/ce/fix-10032.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-当集群中某些节点上的资源仍处于 '初始化/连接中' 状态时，`bridges/` API 将由于缺少这些资源的 Metrics 信息而崩溃。此修复后将忽略没有 Metrics 信息的资源。
diff --git a/changes/ce/fix-10037.en.md b/changes/ce/fix-10037.en.md
deleted file mode 100644
index 73c92d69d..000000000
--- a/changes/ce/fix-10037.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix Swagger API doc rendering crash.
-In version 5.0.18, a bug was introduced that resulted in duplicated field names in the configuration schema. This, in turn, caused the Swagger schema generated to become invalid.
diff --git a/changes/ce/fix-10037.zh.md b/changes/ce/fix-10037.zh.md
deleted file mode 100644
index 5bd447c1f..000000000
--- a/changes/ce/fix-10037.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-修复 Swagger API 文档渲染崩溃。
-在版本 5.0.18 中，引入了一个错误，导致配置 schema 中出现了重复的配置名称，进而导致生成了无效的 Swagger spec。
diff --git a/changes/ce/fix-10041.en.md b/changes/ce/fix-10041.en.md
deleted file mode 100644
index c1aff24c2..000000000
--- a/changes/ce/fix-10041.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-For influxdb bridge, added integer value placeholder annotation hint to `write_syntax` documentation.
-Also supported setting a constant value for the `timestamp` field.
diff --git a/changes/ce/fix-10041.zh.md b/changes/ce/fix-10041.zh.md
deleted file mode 100644
index d197ea81f..000000000
--- a/changes/ce/fix-10041.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-为 influxdb 桥接的配置项 `write_syntax` 描述文档增加了类型标识符的提醒。
-另外在配置中支持 `timestamp` 使用一个常量。
diff --git a/changes/ce/fix-10042.en.md b/changes/ce/fix-10042.en.md
deleted file mode 100644
index af9213c06..000000000
--- a/changes/ce/fix-10042.en.md
+++ /dev/null
@@ -1,5 +0,0 @@
-Improve behavior of the `replicant` nodes when the `core` cluster becomes partitioned (for example when a core node leaves the cluster).
-Previously, the replicant nodes were unable to rebalance connections to the core nodes, until the core cluster became whole again.
-This was indicated by the error messages: `[error] line: 182, mfa: mria_lb:list_core_nodes/1, msg: mria_lb_core_discovery divergent cluster`.
-
-[Mria PR](https://github.com/emqx/mria/pull/123/files)
diff --git a/changes/ce/fix-10042.zh.md b/changes/ce/fix-10042.zh.md
deleted file mode 100644
index 80db204e2..000000000
--- a/changes/ce/fix-10042.zh.md
+++ /dev/null
@@ -1,6 +0,0 @@
-改进 `core` 集群被分割时 `replicant`节点的行为。
-修复前，如果 `core` 集群分裂成两个小集群（例如一个节点离开集群）时，`replicant` 节点无法重新平衡与核心节点的连接，直到核心集群再次变得完整。
-这种个问题会导致 replicant 节点出现如下日志：
-`[error] line: 182, mfa: mria_lb:list_core_nodes/1, msg: mria_lb_core_discovery divergent cluster`。
-
-[Mria PR](https://github.com/emqx/mria/pull/123/files)
diff --git a/changes/ce/fix-10043.en.md b/changes/ce/fix-10043.en.md
deleted file mode 100644
index 4fd46cb4e..000000000
--- a/changes/ce/fix-10043.en.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Fixed two bugs introduced in v5.0.18.
-* The environment varialbe `SSL_DIST_OPTFILE` was not set correctly for non-boot commands.
-* When cookie is overridden from environment variable, EMQX node is unable to start.
diff --git a/changes/ce/fix-10043.zh.md b/changes/ce/fix-10043.zh.md
deleted file mode 100644
index 6b150f6fb..000000000
--- a/changes/ce/fix-10043.zh.md
+++ /dev/null
@@ -1,3 +0,0 @@
-修复 v5.0.18 引入的 2 个bug。
-* 环境变量 `SSL_DIST_OPTFILE` 的值设置错误导致节点无法为 Erlang distribution 启用 SSL。
-* 当节点的 cookie 从环境变量重载 （而不是设置在配置文件中时），节点无法启动的问题。
diff --git a/changes/ce/fix-10044.en.md b/changes/ce/fix-10044.en.md
deleted file mode 100644
index 00668c5cb..000000000
--- a/changes/ce/fix-10044.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix node information formatter for stopped nodes in the cluster. The bug was introduced by v5.0.18.
diff --git a/changes/ce/fix-10044.zh.md b/changes/ce/fix-10044.zh.md
deleted file mode 100644
index 72759d707..000000000
--- a/changes/ce/fix-10044.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修复集群中已停止节点的信息序列化问题，该错误由 v5.0.18 引入。
diff --git a/changes/ce/fix-10050.en.md b/changes/ce/fix-10050.en.md
deleted file mode 100644
index c225c380d..000000000
--- a/changes/ce/fix-10050.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Ensure Bridge API returns `404` status code consistently for resources that don't exist.
diff --git a/changes/ce/fix-10050.zh.md b/changes/ce/fix-10050.zh.md
deleted file mode 100644
index d7faf9434..000000000
--- a/changes/ce/fix-10050.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-确保 Bridge API 对不存在的资源一致返回 `404` 状态代码。
diff --git a/changes/ce/fix-10052.en.md b/changes/ce/fix-10052.en.md
deleted file mode 100644
index f83c4d40c..000000000
--- a/changes/ce/fix-10052.en.md
+++ /dev/null
@@ -1,12 +0,0 @@
-Improve daemon mode startup failure logs.
-
-Before this change, it was difficult for users to understand the reason for EMQX 'start' command failed to boot the node.
-The only information they received was that the node did not start within the expected time frame,
-and they were instructed to boot the node with 'console' command in the hope of obtaining some logs.
-However, the node might actually be running, which could cause 'console' mode to fail for a different reason.
-
-With this new change, when daemon mode fails to boot, a diagnosis is issued. Here are the possible scenarios:
-
-* If the node cannot be found from `ps -ef`, the user is instructed to find information in log files `erlang.log.*`.
-* If the node is found to be running but not responding to pings, the user is advised to check if the host name is resolvable and reachable.
-* If the node is responding to pings, but the EMQX app is not running, it is likely a bug. In this case, the user is advised to report a Github issue.
diff --git a/changes/ce/fix-10052.zh.md b/changes/ce/fix-10052.zh.md
deleted file mode 100644
index 1c2eff342..000000000
--- a/changes/ce/fix-10052.zh.md
+++ /dev/null
@@ -1,11 +0,0 @@
-优化 EMQX daemon 模式启动启动失败的日志。
-
-在进行此更改之前，当 EMQX 用 `start` 命令启动失败时，用户很难理解出错的原因。
-所知道的仅仅是节点未能在预期时间内启动，然后被指示以 `console` 式引导节点以获取一些日志。
-然而，节点实际上可能正在运行，这可能会导致 `console` 模式因不同的原因而失败。
-
-此次修复后，启动脚本会发出诊断：
-
-* 如果无法从 `ps -ef` 中找到节点，则指示用户在 `erlang.log.*` 中查找信息。
-* 如果发现节点正在运行但不响应 ping，则建议用户检查节点主机名是否有效并可达。
-* 如果节点响应 ping 但 EMQX 应用程序未运行，则很可能是一个错误。在这种情况下，建议用户报告一个Github issue。
diff --git a/changes/ce/fix-10054.en.md b/changes/ce/fix-10054.en.md
deleted file mode 100644
index 5efa73314..000000000
--- a/changes/ce/fix-10054.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix the problem that the obfuscated password is used when using the `/bridges_probe` API to test the connection in Data-Bridge.
diff --git a/changes/ce/fix-10054.zh.md b/changes/ce/fix-10054.zh.md
deleted file mode 100644
index 45a80dc45..000000000
--- a/changes/ce/fix-10054.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修复数据桥接中使用 `/bridges_probe` API 进行测试连接时密码被混淆的问题。
diff --git a/changes/ce/fix-10055.en.md b/changes/ce/fix-10055.en.md
deleted file mode 100644
index 4ffaae195..000000000
--- a/changes/ce/fix-10055.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix `mqtt.max_awaiting_rel` change does not work.
diff --git a/changes/ce/fix-10055.zh.md b/changes/ce/fix-10055.zh.md
deleted file mode 100644
index 4da371c51..000000000
--- a/changes/ce/fix-10055.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修复 `mqtt.max_awaiting_rel` 更新不生效问题。
diff --git a/changes/ce/fix-10056.en.md b/changes/ce/fix-10056.en.md
deleted file mode 100644
index 55449294d..000000000
--- a/changes/ce/fix-10056.en.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix `/bridges` API status code.
-- Return `400` instead of `403` in case of removing a data bridge that is dependent on an active rule.
-- Return `400` instead of `403` in case of calling operations (start|stop|restart) when Data-Bridging is not enabled.
diff --git a/changes/ce/fix-10056.zh.md b/changes/ce/fix-10056.zh.md
deleted file mode 100644
index ec5982137..000000000
--- a/changes/ce/fix-10056.zh.md
+++ /dev/null
@@ -1,3 +0,0 @@
-修复 `/bridges` API 的 HTTP 状态码。
-- 当删除被活动中的规则依赖的数据桥接时，将返回 `400` 而不是 `403` 。
-- 当数据桥接未启用时，调用操作（启动|停止|重启）将返回 `400` 而不是 `403`。
diff --git a/changes/ce/fix-10058.en.md b/changes/ce/fix-10058.en.md
deleted file mode 100644
index 337ac5d47..000000000
--- a/changes/ce/fix-10058.en.md
+++ /dev/null
@@ -1,7 +0,0 @@
-Deprecate unused QUIC TLS options.
-Only following TLS options are kept for the QUIC listeners: 
-
-- cacertfile
-- certfile
-- keyfile
-- verify
diff --git a/changes/ce/fix-10058.zh.md b/changes/ce/fix-10058.zh.md
deleted file mode 100644
index d1dea37c3..000000000
--- a/changes/ce/fix-10058.zh.md
+++ /dev/null
@@ -1,8 +0,0 @@
-废弃未使用的 QUIC TLS 选项。
-QUIC 监听器只保留以下 TLS 选项:
-
-- cacertfile
-- certfile
-- keyfile
-- verify
-
diff --git a/changes/ce/fix-10066.en.md b/changes/ce/fix-10066.en.md
deleted file mode 100644
index 87e253aca..000000000
--- a/changes/ce/fix-10066.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Improve error messages for `/briges_probe` and `[/node/:node]/bridges/:id/:operation` API calls to make them more readable. And set HTTP status code to `400` instead of `500`.
diff --git a/changes/ce/fix-10066.zh.md b/changes/ce/fix-10066.zh.md
deleted file mode 100644
index e5e3c2113..000000000
--- a/changes/ce/fix-10066.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-改进 `/briges_probe` 和 `[/node/:node]/bridges/:id/:operation` API 调用的错误信息，使之更加易读。并将 HTTP 状态代码设置为 `400` 而不是 `500`。
diff --git a/changes/ce/fix-10074.en.md b/changes/ce/fix-10074.en.md
deleted file mode 100644
index 49c52b948..000000000
--- a/changes/ce/fix-10074.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Check if type in `PUT /authorization/sources/:type` matches `type` given in body of request.
diff --git a/changes/ce/fix-10074.zh.md b/changes/ce/fix-10074.zh.md
deleted file mode 100644
index 930840cdf..000000000
--- a/changes/ce/fix-10074.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-检查 `PUT /authorization/sources/:type` 中的类型是否与请求正文中的 `type` 相符。
diff --git a/changes/ce/fix-10076.en.md b/changes/ce/fix-10076.en.md
deleted file mode 100644
index 5bbbffa32..000000000
--- a/changes/ce/fix-10076.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix webhook bridge error handling: connection timeout should be a retriable error.
-Prior to this fix, connection timeout was classified as unrecoverable error and led to request being dropped.
diff --git a/changes/ce/fix-10076.zh.md b/changes/ce/fix-10076.zh.md
deleted file mode 100644
index 516345f92..000000000
--- a/changes/ce/fix-10076.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-修复 HTTP 桥接的一个异常处理：连接超时错误发生后，发生错误的请求可以被重试。
-在此修复前，连接超时后，被当作不可重试类型的错误处理，导致请求被丢弃。
diff --git a/changes/ce/fix-10078.en.md b/changes/ce/fix-10078.en.md
deleted file mode 100644
index afb7bcbe0..000000000
--- a/changes/ce/fix-10078.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix an issue that invalid QUIC listener setting could casue segfault. 
-
diff --git a/changes/ce/fix-10078.zh.md b/changes/ce/fix-10078.zh.md
deleted file mode 100644
index 47a774d1e..000000000
--- a/changes/ce/fix-10078.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-修复了无效的 QUIC 监听器设置可能导致 segfault 的问题。
-
diff --git a/changes/ce/fix-10079.en.md b/changes/ce/fix-10079.en.md
deleted file mode 100644
index 440351753..000000000
--- a/changes/ce/fix-10079.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix description of `shared_subscription_strategy`. 
diff --git a/changes/ce/fix-10079.zh.md b/changes/ce/fix-10079.zh.md
deleted file mode 100644
index ca2ab9173..000000000
--- a/changes/ce/fix-10079.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-修正对 `shared_subscription_strategy` 的描述。
-
diff --git a/changes/ce/fix-10084.en.md b/changes/ce/fix-10084.en.md
deleted file mode 100644
index 90da7d660..000000000
--- a/changes/ce/fix-10084.en.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix problem when joining core nodes running different EMQX versions into a cluster.
-
-[Mria PR](https://github.com/emqx/mria/pull/127)
diff --git a/changes/ce/fix-10084.zh.md b/changes/ce/fix-10084.zh.md
deleted file mode 100644
index dd44533cf..000000000
--- a/changes/ce/fix-10084.zh.md
+++ /dev/null
@@ -1,3 +0,0 @@
-修正将运行不同 EMQX 版本的核心节点加入集群的问题。
-
-[Mria PR](https://github.com/emqx/mria/pull/127)
diff --git a/changes/ce/fix-10085.en.md b/changes/ce/fix-10085.en.md
deleted file mode 100644
index e539a04b4..000000000
--- a/changes/ce/fix-10085.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Consistently return `404` for all requests on non existent source in `/authorization/sources/:source[/*]`.
diff --git a/changes/ce/fix-10085.zh.md b/changes/ce/fix-10085.zh.md
deleted file mode 100644
index 059680efa..000000000
--- a/changes/ce/fix-10085.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-如果向 `/authorization/sources/:source[/*]`  请求的 `source` 不存在，将一致地返回 `404`。
diff --git a/changes/ce/fix-10086.en.md b/changes/ce/fix-10086.en.md
deleted file mode 100644
index d337a57c7..000000000
--- a/changes/ce/fix-10086.en.md
+++ /dev/null
@@ -1,4 +0,0 @@
-Upgrade HTTP client ehttpc to `0.4.7`.
-Prior to this upgrade, HTTP clients for authentication, authorization and webhook may crash
-if `Body` is empty but `Content-Type` HTTP header is set.
-For more details see [ehttpc PR#44](https://github.com/emqx/ehttpc/pull/44).
diff --git a/changes/ce/fix-10086.zh.md b/changes/ce/fix-10086.zh.md
deleted file mode 100644
index c083d6055..000000000
--- a/changes/ce/fix-10086.zh.md
+++ /dev/null
@@ -1,3 +0,0 @@
-HTTP 客户端库 `ehttpc` 升级到 0.4.7。
-在升级前，如果 HTTP 客户端，例如 '认证'、'授权'、'WebHook' 等配置中使用了 `Content-Type` HTTP 头，但是没有配置 `Body`，则可能会发生异常。
-详情见 [ehttpc PR#44](https://github.com/emqx/ehttpc/pull/44)。
diff --git a/changes/ce/fix-10098.en.md b/changes/ce/fix-10098.en.md
deleted file mode 100644
index 61058da0a..000000000
--- a/changes/ce/fix-10098.en.md
+++ /dev/null
@@ -1 +0,0 @@
-A crash with an error in the log file that happened when the MongoDB authorization module queried the database has been fixed.
diff --git a/changes/ce/fix-10098.zh.md b/changes/ce/fix-10098.zh.md
deleted file mode 100644
index 6b6d86159..000000000
--- a/changes/ce/fix-10098.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-当MongoDB授权模块查询数据库时，在日志文件中发生的崩溃与错误已经被修复。
diff --git a/changes/ce/fix-10100.en.md b/changes/ce/fix-10100.en.md
deleted file mode 100644
index e16ee5efc..000000000
--- a/changes/ce/fix-10100.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix channel crash for slow clients with enhanced authentication.
-Previously, when the client was using enhanced authentication, but the Auth message was sent slowly or the Auth message was lost, the client process would crash.
diff --git a/changes/ce/fix-10100.zh.md b/changes/ce/fix-10100.zh.md
deleted file mode 100644
index ac2483a27..000000000
--- a/changes/ce/fix-10100.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-修复响应较慢的客户端在使用增强认证时可能出现崩溃的问题。
-此前，当客户端使用增强认证功能，但发送 Auth 报文较慢或 Auth 报文丢失时会导致客户端进程崩溃。
diff --git a/changes/ce/fix-10107.en.md b/changes/ce/fix-10107.en.md
deleted file mode 100644
index 1bcbbad60..000000000
--- a/changes/ce/fix-10107.en.md
+++ /dev/null
@@ -1,9 +0,0 @@
-For operations on `bridges API` if `bridge-id` is unknown we now return `404`
-instead of `400`. Also a bug was fixed that caused a crash if that was a node
-operation. Additionally we now also check if the given bridge is enabled when
-doing the cluster operation `start` . Affected endpoints:
- * [cluster] `/bridges/:id/:operation`,
- * [node] `/nodes/:node/bridges/:id/:operation`, where `operation` is one of
-`[start|stop|restart]`.
-Moreover, for a node operation, EMQX checks if node name is in our cluster and
-return `404` instead of `501`.
diff --git a/changes/ce/fix-10107.zh.md b/changes/ce/fix-10107.zh.md
deleted file mode 100644
index e541a834f..000000000
--- a/changes/ce/fix-10107.zh.md
+++ /dev/null
@@ -1,8 +0,0 @@
-现在对桥接的 API 进行调用时，如果 `bridge-id` 不存在，将会返回 `404`，而不再是`400`。
-然后，还修复了这种情况下，在节点级别上进行 API 调用时，可能导致崩溃的问题。
-另外，在启动某个桥接时，会先检查指定桥接是否已启用。
-受影响的接口有：
- * [cluster] `/bridges/:id/:operation`,
- * [node] `/nodes/:node/bridges/:id/:operation`,
-其中 `operation` 是 `[start|stop|restart]` 之一。
-此外，对于节点操作，EMQX 将检查节点是否存在于集群中，如果不在，则会返回`404`，而不再是`501`。
diff --git a/changes/ce/fix-10117.en.md b/changes/ce/fix-10117.en.md
deleted file mode 100644
index 711d739ca..000000000
--- a/changes/ce/fix-10117.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix an error occurring when a joining node doesn't have plugins that are installed on other nodes in the cluster.
-After this change, the joining node will copy all the necessary plugins from other nodes.
diff --git a/changes/ce/fix-10118.en.md b/changes/ce/fix-10118.en.md
deleted file mode 100644
index f6db758f3..000000000
--- a/changes/ce/fix-10118.en.md
+++ /dev/null
@@ -1,4 +0,0 @@
-Fix problems related to manual joining of EMQX replicant nodes to the cluster.
-Previously, after manually executing joining and then leaving the cluster, the `replicant` node can only run normally after restarting the node after joining the cluster again.
-
-[Mria PR](https://github.com/emqx/mria/pull/128)
diff --git a/changes/ce/fix-10118.zh.md b/changes/ce/fix-10118.zh.md
deleted file mode 100644
index a037215f0..000000000
--- a/changes/ce/fix-10118.zh.md
+++ /dev/null
@@ -1,4 +0,0 @@
-修复 `replicant` 节点因为手动加入 EMQX 集群导致的相关问题。
-此前，手动执行 `加入集群-离开集群` 后，`replicant` 节点再次加入集群后只有重启节点才能正常运行。
-
-[Mria PR](https://github.com/emqx/mria/pull/128)
diff --git a/changes/ce/fix-10119.en.md b/changes/ce/fix-10119.en.md
deleted file mode 100644
index c23a9dcdb..000000000
--- a/changes/ce/fix-10119.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix crash when `statsd.server` is set to an empty string.
diff --git a/changes/ce/fix-10119.zh.md b/changes/ce/fix-10119.zh.md
deleted file mode 100644
index c77b99025..000000000
--- a/changes/ce/fix-10119.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修复 `statsd.server` 配置为空字符串时启动崩溃的问题。
diff --git a/changes/ce/fix-10124.en.md b/changes/ce/fix-10124.en.md
deleted file mode 100644
index 1a4aca3d9..000000000
--- a/changes/ce/fix-10124.en.md
+++ /dev/null
@@ -1 +0,0 @@
-The default heartbeat period for MongoDB has been increased to reduce the risk of too excessive logging to the MongoDB log file.
diff --git a/changes/ce/fix-10124.zh.md b/changes/ce/fix-10124.zh.md
deleted file mode 100644
index 7605f2da3..000000000
--- a/changes/ce/fix-10124.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-增加了MongoDB的默认心跳周期，以减少对MongoDB日志文件的过多记录的风险。
diff --git a/changes/ce/fix-10130.en.md b/changes/ce/fix-10130.en.md
deleted file mode 100644
index 98484e38f..000000000
--- a/changes/ce/fix-10130.en.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix garbled config display in dashboard when the value is originally from environment variables.
-For example, `env EMQX_STATSD__SERVER='127.0.0.1:8124' . /bin/emqx start` results in unreadable string (not '127.0.0.1:8124') displayed in Dashboard's Statsd settings page.
-Related PR: [HOCON#234](https://github.com/emqx/hocon/pull/234).
diff --git a/changes/ce/fix-10130.zh.md b/changes/ce/fix-10130.zh.md
deleted file mode 100644
index 19c092fdf..000000000
--- a/changes/ce/fix-10130.zh.md
+++ /dev/null
@@ -1,3 +0,0 @@
-修复通过环境变量配置启动的 EMQX 节点无法通过HTTP API获取到正确的配置信息。
-比如：`EMQX_STATSD__SERVER='127.0.0.1:8124' ./bin/emqx start` 后通过 Dashboard看到的 Statsd 配置信息是乱码。
-相关 PR: [HOCON:234](https://github.com/emqx/hocon/pull/234).
diff --git a/changes/ce/fix-10132.en.md b/changes/ce/fix-10132.en.md
deleted file mode 100644
index ceb617d11..000000000
--- a/changes/ce/fix-10132.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix `systemctl stop emqx` command not stopping jq, os_mon application properly, generating some error logs.
diff --git a/changes/ce/fix-10132.zh.md b/changes/ce/fix-10132.zh.md
deleted file mode 100644
index 36811e1bf..000000000
--- a/changes/ce/fix-10132.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修复`systemctl stop emqx` 命令没有正常停止 jq, os_mon 组件，产生一些错误日志。
diff --git a/changes/ce/fix-10144.en.md b/changes/ce/fix-10144.en.md
deleted file mode 100644
index d5a84b24c..000000000
--- a/changes/ce/fix-10144.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Add -setcookie emulator flag when invoking emqx ctl to prevent problems with emqx cli when home directory is read only. Fixes [#10142](https://github.com/emqx/emqx/issues/10142)
diff --git a/changes/ce/fix-9939.en.md b/changes/ce/fix-9939.en.md
deleted file mode 100644
index 83e84c493..000000000
--- a/changes/ce/fix-9939.en.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Allow 'emqx ctl cluster' command to be issued before Mnesia starts.
-Prior to this change, EMQX `replicant` could not use `manual` discovery strategy.
-Now it's possible to join cluster using 'manual' strategy.
diff --git a/changes/ce/fix-9939.zh.md b/changes/ce/fix-9939.zh.md
deleted file mode 100644
index 4b150c5fc..000000000
--- a/changes/ce/fix-9939.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-允许 'emqx ctl cluster join' 命令在 Mnesia 启动前就可以调用。
-在此修复前， EMQX 的 `replicant` 类型节点无法使用 `manual` 集群发现策略。
diff --git a/changes/ce/fix-9958.en.md b/changes/ce/fix-9958.en.md
deleted file mode 100644
index 821934ad0..000000000
--- a/changes/ce/fix-9958.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix bad http response format when client ID is not found in `clients` APIs
diff --git a/changes/ce/fix-9958.zh.md b/changes/ce/fix-9958.zh.md
deleted file mode 100644
index a26fbb7fe..000000000
--- a/changes/ce/fix-9958.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修复 `clients` API 在 Client ID 不存在时返回的错误的 HTTP 应答格式。
diff --git a/changes/ce/fix-9961.en.md b/changes/ce/fix-9961.en.md
deleted file mode 100644
index 6185a64ea..000000000
--- a/changes/ce/fix-9961.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Avoid parsing config files for node name and cookie when executing non-boot commands in bin/emqx.
diff --git a/changes/ce/fix-9961.zh.md b/changes/ce/fix-9961.zh.md
deleted file mode 100644
index edd90b2ca..000000000
--- a/changes/ce/fix-9961.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-在 bin/emqx 脚本中，避免在运行非启动命令时解析 emqx.conf 来获取节点名称和 cookie。
diff --git a/changes/ce/fix-9974.en.md b/changes/ce/fix-9974.en.md
deleted file mode 100644
index 97223e03f..000000000
--- a/changes/ce/fix-9974.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Report memory usage to statsd and prometheus using the same data source as dashboard.
-Prior to this fix, the memory usage data source was collected from an outdated source which did not work well in containers.
diff --git a/changes/ce/fix-9974.zh.md b/changes/ce/fix-9974.zh.md
deleted file mode 100644
index 8358204f3..000000000
--- a/changes/ce/fix-9974.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Statsd 和 prometheus 使用跟 Dashboard 相同的内存用量数据源。
-在此修复前，内存的总量和用量统计使用了过时的（在容器环境中不准确）的数据源。
diff --git a/changes/ce/fix-9978.en.md b/changes/ce/fix-9978.en.md
deleted file mode 100644
index 6750d136f..000000000
--- a/changes/ce/fix-9978.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Fixed configuration issue when choosing to use SSL for a Postgres connection (`authn`, `authz` and bridge).
-The connection could fail to complete with a previously working configuration after an upgrade from 5.0.13 to newer EMQX versions.
diff --git a/changes/ce/fix-9978.zh.md b/changes/ce/fix-9978.zh.md
deleted file mode 100644
index 75eed3600..000000000
--- a/changes/ce/fix-9978.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-修正了在Postgres连接中选择使用SSL时的配置问题（`authn`, `authz` 和 bridge）。
-从5.0.13升级到较新的EMQX版本后，连接可能无法完成之前的配置。
diff --git a/changes/ce/fix-9997.en.md b/changes/ce/fix-9997.en.md
deleted file mode 100644
index be0344ec1..000000000
--- a/changes/ce/fix-9997.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix Swagger API schema generation. `deprecated` metadata field is now always boolean, as [Swagger specification](https://swagger.io/specification/) suggests.
diff --git a/changes/ce/fix-9997.zh.md b/changes/ce/fix-9997.zh.md
deleted file mode 100644
index 6f1a0b779..000000000
--- a/changes/ce/fix-9997.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-修复 Swagger API 生成时，`deprecated` 元数据字段未按照[标准](https://swagger.io/specification/)建议的那样始终为布尔值的问题。
diff --git a/changes/ce/perf-9967.en.md b/changes/ce/perf-9967.en.md
deleted file mode 100644
index fadba24c9..000000000
--- a/changes/ce/perf-9967.en.md
+++ /dev/null
@@ -1 +0,0 @@
-New common TLS option 'hibernate_after' to reduce memory footprint per idle connecion, default: 5s.
diff --git a/changes/ce/perf-9967.zh.md b/changes/ce/perf-9967.zh.md
deleted file mode 100644
index 7b73f9bd0..000000000
--- a/changes/ce/perf-9967.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-新的通用 TLS 选项 'hibernate_after'， 以减少空闲连接的内存占用，默认： 5s 。
diff --git a/changes/ce/perf-9998.en.md b/changes/ce/perf-9998.en.md
deleted file mode 100644
index e9e23a25e..000000000
--- a/changes/ce/perf-9998.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Redact the HTTP request body in the authentication error logs for security reasons.
diff --git a/changes/ce/perf-9998.zh.md b/changes/ce/perf-9998.zh.md
deleted file mode 100644
index 146eb858f..000000000
--- a/changes/ce/perf-9998.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-出于安全原因，在身份验证错误日志中模糊 HTTP 请求正文。
diff --git a/changes/ee/feat-10083.en.md b/changes/ee/feat-10083.en.md
deleted file mode 100644
index f4331faf9..000000000
--- a/changes/ee/feat-10083.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Add `DynamoDB` support for Data-Brdige.
diff --git a/changes/ee/feat-10083.zh.md b/changes/ee/feat-10083.zh.md
deleted file mode 100644
index 8274e62c2..000000000
--- a/changes/ee/feat-10083.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-为数据桥接增加 `DynamoDB` 支持。
diff --git a/changes/ee/feat-9564.en.md b/changes/ee/feat-9564.en.md
deleted file mode 100644
index 4405e3e07..000000000
--- a/changes/ee/feat-9564.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Implemented Kafka Consumer bridge.
-Now it's possible to consume messages from Kafka and publish them to MQTT topics.
diff --git a/changes/ee/feat-9564.zh.md b/changes/ee/feat-9564.zh.md
deleted file mode 100644
index 01a7ffe58..000000000
--- a/changes/ee/feat-9564.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-实现了 Kafka 消费者桥接。
-现在可以从 Kafka 消费消息并将其发布到 MQTT 主题。
diff --git a/changes/ee/feat-9881.en.md b/changes/ee/feat-9881.en.md
deleted file mode 100644
index 546178965..000000000
--- a/changes/ee/feat-9881.en.md
+++ /dev/null
@@ -1,4 +0,0 @@
-In this pull request, we have enhanced the error logs related to InfluxDB connectivity health checks.
-Previously, if InfluxDB failed to pass the health checks using the specified parameters, the only message provided was "timed out waiting for it to become healthy".
-With the updated implementation, the error message will be displayed in both the logs and the dashboard, enabling easier identification and resolution of the issue.
-
diff --git a/changes/ee/feat-9881.zh.md b/changes/ee/feat-9881.zh.md
deleted file mode 100644
index 9746a4c0a..000000000
--- a/changes/ee/feat-9881.zh.md
+++ /dev/null
@@ -1,3 +0,0 @@
-增强了与 InfluxDB 连接健康检查相关的错误日志。
-在此更改之前，如果使用配置的参数 InfluxDB 未能通过健康检查，用户仅能获得一个“超时”的信息。
-现在，详细的错误消息将显示在日志和控制台，从而让用户更容易地识别和解决问题。
diff --git a/changes/ee/feat-9932.en.md b/changes/ee/feat-9932.en.md
deleted file mode 100644
index f4f9ce40d..000000000
--- a/changes/ee/feat-9932.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Integrate `TDengine` into `bridges` as a new backend.
diff --git a/changes/ee/feat-9932.zh.md b/changes/ee/feat-9932.zh.md
deleted file mode 100644
index 1fbf7bf34..000000000
--- a/changes/ee/feat-9932.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-在 `桥接` 中集成 `TDengine`。
diff --git a/changes/ee/fix-10007.en.md b/changes/ee/fix-10007.en.md
deleted file mode 100644
index 1adab8e9b..000000000
--- a/changes/ee/fix-10007.en.md
+++ /dev/null
@@ -1,5 +0,0 @@
-Change Kafka bridge's config `memory_overload_protection` default value from `true` to `false`.
-EMQX logs cases when messages get dropped due to overload protection, and this is also reflected in counters.
-However, since there is by default no alerting based on the logs and counters,
-setting it to `true` may cause messages being dropped without noticing.
-At the time being, the better option is to let sysadmin set it explicitly so they are fully aware of the benefits and risks.
diff --git a/changes/ee/fix-10007.zh.md b/changes/ee/fix-10007.zh.md
deleted file mode 100644
index 0c08f20d0..000000000
--- a/changes/ee/fix-10007.zh.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Kafka 桥接的配置参数 `memory_overload_protection` 默认值从 `true` 改成了 `false`。
-尽管内存过载后消息被丢弃会产生日志和计数，如果没有基于这些日志或计数的告警，系统管理员可能无法及时发现消息被丢弃。
-当前更好的选择是：让管理员显式的配置该项，迫使他们理解这个配置的好处以及风险。
diff --git a/changes/ee/fix-10087.en.md b/changes/ee/fix-10087.en.md
deleted file mode 100644
index fd6e10b7b..000000000
--- a/changes/ee/fix-10087.en.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Use default template `${timestamp}` if the `timestamp` config is empty (undefined) when inserting data in InfluxDB.
-Prior to this change, InfluxDB bridge inserted a wrong timestamp when template is not provided.
diff --git a/changes/ee/fix-10087.zh.md b/changes/ee/fix-10087.zh.md
deleted file mode 100644
index e08e61f37..000000000
--- a/changes/ee/fix-10087.zh.md
+++ /dev/null
@@ -1,2 +0,0 @@
-在 InfluxDB 中插入数据时，如果时间戳为空（未定义），则使用默认的占位符 `${timestamp}`。
-在此修复前，如果时间戳字段没有设置，InfluxDB 桥接使用了一个错误的时间戳。
diff --git a/changes/ee/fix-10095.en.md b/changes/ee/fix-10095.en.md
deleted file mode 100644
index 49c588345..000000000
--- a/changes/ee/fix-10095.en.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Stop MySQL client from bombarding server repeatedly with unnecessary `PREPARE` queries on every batch, trashing the server and exhausting its internal limits. This was happening when the MySQL bridge was in the batch mode.
-
-Ensure safer and more careful escaping of strings and binaries in batch insert queries when the MySQL bridge is in the batch mode.
diff --git a/changes/ee/fix-10095.zh.md b/changes/ee/fix-10095.zh.md
deleted file mode 100644
index 5a62ccfca..000000000
--- a/changes/ee/fix-10095.zh.md
+++ /dev/null
@@ -1 +0,0 @@
-优化 MySQL 桥接在批量模式下能更高效的使用预处理语句 ，减少了对 MySQL 服务器的查询压力， 并确保对 SQL 语句进行更安全和谨慎的转义。

From f9044a09599019c1d957ae93b83df710cc428387 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 16 Mar 2023 14:03:37 -0300
Subject: [PATCH 06/52] docs(changelog): fix en changelog

---
 changes/ce/feat-10128.en.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/ce/feat-10128.en.md

diff --git a/changes/ce/feat-10128.en.md b/changes/ce/feat-10128.en.md
new file mode 100644
index 000000000..ab3e5ba3e
--- /dev/null
+++ b/changes/ce/feat-10128.en.md
@@ -0,0 +1 @@
+Add support for OCSP stapling for SSL MQTT listeners.

From 3a46681dde991f301d77adb4de3bb6447b682517 Mon Sep 17 00:00:00 2001
From: Serge Tupchii <serge.tupchii@protonmail.com>
Date: Thu, 16 Mar 2023 22:21:04 +0200
Subject: [PATCH 07/52] feat: handle escaped characters in InfluxDB data bridge
 write_syntax

Closes: EMQX-7834
---
 changes/ee/feat-10165.en.md                   |   2 +
 .../src/emqx_ee_bridge_influxdb.erl           | 184 +++++++---
 .../test/emqx_ee_bridge_influxdb_tests.erl    | 328 ++++++++++++++++++
 3 files changed, 471 insertions(+), 43 deletions(-)
 create mode 100644 changes/ee/feat-10165.en.md
 create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl

diff --git a/changes/ee/feat-10165.en.md b/changes/ee/feat-10165.en.md
new file mode 100644
index 000000000..199d45707
--- /dev/null
+++ b/changes/ee/feat-10165.en.md
@@ -0,0 +1,2 @@
+Support escaped special characters in InfluxDB data bridge write_syntax.
+This update allows to use escaped special characters in string elements in accordance with InfluxDB line protocol.
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 14f53b5e7..62a9b4e80 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -3,6 +3,7 @@
 %%--------------------------------------------------------------------
 -module(emqx_ee_bridge_influxdb).
 
+-include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("typerefl/include/types.hrl").
@@ -169,53 +170,150 @@ write_syntax(_) ->
     undefined.
 
 to_influx_lines(RawLines) ->
-    Lines = string:tokens(str(RawLines), "\n"),
-    lists:reverse(lists:foldl(fun converter_influx_line/2, [], Lines)).
-
-converter_influx_line(Line, AccIn) ->
-    case string:tokens(str(Line), " ") of
-        [MeasurementAndTags, Fields, Timestamp] ->
-            append_influx_item(MeasurementAndTags, Fields, Timestamp, AccIn);
-        [MeasurementAndTags, Fields] ->
-            append_influx_item(MeasurementAndTags, Fields, undefined, AccIn);
-        _ ->
-            throw("Bad InfluxDB Line Protocol schema")
+    try
+        influx_lines(str(RawLines), [])
+    catch
+        _:Reason:Stacktrace ->
+            Msg = lists:flatten(
+                io_lib:format("Unable to parse InfluxDB line protocol: ~p", [RawLines])
+            ),
+            ?SLOG(error, #{msg => Msg, error_reason => Reason, stacktrace => Stacktrace}),
+            throw(Msg)
     end.
 
-append_influx_item(MeasurementAndTags, Fields, Timestamp, Acc) ->
-    {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags),
-    [
-        #{
-            measurement => Measurement,
-            tags => kv_pairs(Tags),
-            fields => kv_pairs(string:tokens(Fields, ",")),
-            timestamp => Timestamp
-        }
-        | Acc
-    ].
+-define(MEASUREMENT_ESC_CHARS, [$,, $\s]).
+-define(TAG_FIELD_KEY_ESC_CHARS, [$,, $=, $\s]).
+-define(FIELD_VAL_ESC_CHARS, [$", $\\]).
+% Common separator for both tags and fields
+-define(SEP, $\s).
+-define(MEASUREMENT_TAG_SEP, $,).
+-define(KEY_SEP, $=).
+-define(VAL_SEP, $,).
+-define(NON_EMPTY, [_ | _]).
 
-split_measurement_and_tags(Subject) ->
-    case string:tokens(Subject, ",") of
-        [] ->
-            throw("Bad Measurement schema");
-        [Measurement] ->
-            {Measurement, []};
-        [Measurement | Tags] ->
-            {Measurement, Tags}
-    end.
+influx_lines([] = _RawLines, Acc) ->
+    ?NON_EMPTY = lists:reverse(Acc);
+influx_lines(RawLines, Acc) ->
+    {Acc1, RawLines1} = influx_line(string:trim(RawLines, leading, "\s\n"), Acc),
+    influx_lines(RawLines1, Acc1).
 
-kv_pairs(Pairs) ->
-    kv_pairs(Pairs, []).
-kv_pairs([], Acc) ->
-    lists:reverse(Acc);
-kv_pairs([Pair | Rest], Acc) ->
-    case string:tokens(Pair, "=") of
-        [K, V] ->
-            %% Reduplicated keys will be overwritten. Follows InfluxDB Line Protocol.
-            kv_pairs(Rest, [{K, V} | Acc]);
-        _ ->
-            throw(io_lib:format("Bad InfluxDB Line Protocol Key Value pair: ~p", Pair))
-    end.
+influx_line([], Acc) ->
+    {Acc, []};
+influx_line(Line, Acc) ->
+    {?NON_EMPTY = Measurement, Line1} = measurement(Line),
+    {Tags, Line2} = tags(Line1),
+    {?NON_EMPTY = Fields, Line3} = influx_fields(Line2),
+    {Timestamp, Line4} = timestamp(Line3),
+    {
+        [
+            #{
+                measurement => Measurement,
+                tags => Tags,
+                fields => Fields,
+                timestamp => Timestamp
+            }
+            | Acc
+        ],
+        Line4
+    }.
+
+measurement(Line) ->
+    unescape(?MEASUREMENT_ESC_CHARS, [?MEASUREMENT_TAG_SEP, ?SEP], Line, []).
+
+tags([?MEASUREMENT_TAG_SEP | Line]) ->
+    tags1(Line, []);
+tags(Line) ->
+    {[], Line}.
+
+%% Empty line is invalid as fields are required after tags,
+%% need to break recursion here and fail later on parsing fields
+tags1([] = Line, Acc) ->
+    {lists:reverse(Acc), Line};
+%% Matching non empty Acc treats lines like "m, field=field_val" invalid
+tags1([?SEP | _] = Line, ?NON_EMPTY = Acc) ->
+    {lists:reverse(Acc), Line};
+tags1(Line, Acc) ->
+    {Tag, Line1} = tag(Line),
+    tags1(Line1, [Tag | Acc]).
+
+tag(Line) ->
+    {?NON_EMPTY = Key, Line1} = key(Line),
+    {?NON_EMPTY = Val, Line2} = tag_val(Line1),
+    {{Key, Val}, Line2}.
+
+tag_val(Line) ->
+    {Val, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?VAL_SEP, ?SEP], Line, []),
+    {Val, strip_l(Line1, ?VAL_SEP)}.
+
+influx_fields([?SEP | Line]) ->
+    fields1(string:trim(Line, leading, "\s"), []).
+
+%% Timestamp is optional, so fields may be at the very end of the line
+fields1([Ch | _] = Line, Acc) when Ch =:= ?SEP; Ch =:= $\n ->
+    {lists:reverse(Acc), Line};
+fields1([] = Line, Acc) ->
+    {lists:reverse(Acc), Line};
+fields1(Line, Acc) ->
+    {Field, Line1} = field(Line),
+    fields1(Line1, [Field | Acc]).
+
+field(Line) ->
+    {?NON_EMPTY = Key, Line1} = key(Line),
+    {Val, Line2} = field_val(Line1),
+    {{Key, Val}, Line2}.
+
+field_val([$" | Line]) ->
+    {Val, [$" | Line1]} = unescape(?FIELD_VAL_ESC_CHARS, [$"], Line, []),
+    %% Quoted val can be empty
+    {Val, strip_l(Line1, ?VAL_SEP)};
+field_val(Line) ->
+    %% Unquoted value should not be un-escaped according to InfluxDB protocol,
+    %% as it can only hold float, integer, uinteger or boolean value.
+    %% However, as templates are possible, un-escaping is applied here,
+    %% which also helps to detect some invalid lines, e.g.: "m,tag=1 field= ${timestamp}"
+    {Val, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?VAL_SEP, ?SEP, $\n], Line, []),
+    {?NON_EMPTY = Val, strip_l(Line1, ?VAL_SEP)}.
+
+timestamp([?SEP | Line]) ->
+    Line1 = string:trim(Line, leading, "\s"),
+    %% Similarly to unquoted field value, un-escape a timestamp to validate and handle
+    %% potentially escaped characters in a template
+    {T, Line2} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?SEP, $\n], Line1, []),
+    {timestamp1(T), Line2};
+timestamp(Line) ->
+    {undefined, Line}.
+
+timestamp1(?NON_EMPTY = Ts) -> Ts;
+timestamp1(_Ts) -> undefined.
+
+%% Common for both tag and field keys
+key(Line) ->
+    {Key, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?KEY_SEP], Line, []),
+    {Key, strip_l(Line1, ?KEY_SEP)}.
+
+%% Only strip a character between pairs, don't strip it(and let it fail)
+%% if the char to be stripped is at the end, e.g.: m,tag=val, field=val
+strip_l([Ch, Ch1 | Str], Ch) when Ch1 =/= ?SEP ->
+    [Ch1 | Str];
+strip_l(Str, _Ch) ->
+    Str.
+
+unescape(EscapeChars, SepChars, [$\\, Char | T], Acc) ->
+    ShouldEscapeBackslash = lists:member($\\, EscapeChars),
+    Acc1 =
+        case lists:member(Char, EscapeChars) of
+            true -> [Char | Acc];
+            false when not ShouldEscapeBackslash -> [Char, $\\ | Acc]
+        end,
+    unescape(EscapeChars, SepChars, T, Acc1);
+unescape(EscapeChars, SepChars, [Char | T] = L, Acc) ->
+    IsEscapeChar = lists:member(Char, EscapeChars),
+    case lists:member(Char, SepChars) of
+        true -> {lists:reverse(Acc), L};
+        false when not IsEscapeChar -> unescape(EscapeChars, SepChars, T, [Char | Acc])
+    end;
+unescape(_EscapeChars, _SepChars, [] = L, Acc) ->
+    {lists:reverse(Acc), L}.
 
 str(A) when is_atom(A) ->
     atom_to_list(A);
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl
new file mode 100644
index 000000000..ce3a0b06f
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl
@@ -0,0 +1,328 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge_influxdb_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-import(emqx_ee_bridge_influxdb, [to_influx_lines/1]).
+
+-define(INVALID_LINES, [
+    "   ",
+    " \n",
+    "  \n\n\n  ",
+    "\n",
+    "  \n\n   \n  \n",
+    "measurement",
+    "measurement ",
+    "measurement,tag",
+    "measurement field",
+    "measurement,tag field",
+    "measurement,tag field ${timestamp}",
+    "measurement,tag=",
+    "measurement,tag=tag1",
+    "measurement,tag =",
+    "measurement field=",
+    "measurement field= ",
+    "measurement field = ",
+    "measurement, tag = field = ",
+    "measurement, tag = field = ",
+    "measurement, tag = tag_val field = field_val",
+    "measurement, tag = tag_val field = field_val ${timestamp}",
+    "measurement,= = ${timestamp}",
+    "measurement,t=a, f=a, ${timestamp}",
+    "measurement,t=a,t1=b, f=a,f1=b, ${timestamp}",
+    "measurement,t=a,t1=b, f=a,f1=b,",
+    "measurement,t=a, t1=b, f=a,f1=b,",
+    "measurement,t=a,,t1=b, f=a,f1=b,",
+    "measurement,t=a,,t1=b f=a,,f1=b",
+    "measurement,t=a,,t1=b f=a,f1=b ${timestamp}",
+    "measurement, f=a,f1=b",
+    "measurement, f=a,f1=b ${timestamp}",
+    "measurement,, f=a,f1=b ${timestamp}",
+    "measurement,, f=a,f1=b",
+    "measurement,, f=a,f1=b,, ${timestamp}",
+    "measurement f=a,f1=b,, ${timestamp}",
+    "measurement,t=a f=a,f1=b,, ${timestamp}",
+    "measurement,t=a f=a,f1=b,, ",
+    "measurement,t=a f=a,f1=b,,",
+    "measurement, t=a  f=a,f1=b",
+    "measurement,t=a f=a, f1=b",
+    "measurement,t=a f=a, f1=b ${timestamp}",
+    "measurement, t=a  f=a, f1=b ${timestamp}",
+    "measurement,t= a f=a,f1=b ${timestamp}",
+    "measurement,t= a f=a,f1 =b ${timestamp}",
+    "measurement, t = a f = a,f1 = b ${timestamp}",
+    "measurement,t=a f=a,f1=b \n ${timestamp}",
+    "measurement,t=a \n f=a,f1=b \n ${timestamp}",
+    "measurement,t=a \n f=a,f1=b \n ",
+    "\n measurement,t=a \n f=a,f1=b \n ${timestamp}",
+    "\n measurement,t=a \n f=a,f1=b \n",
+    %% not escaped backslash in a quoted field value is invalid
+    "measurement,tag=1 field=\"val\\1\""
+]).
+
+-define(VALID_LINE_PARSED_PAIRS, [
+    {"m1,tag=tag1 field=field1 ${timestamp1}", #{
+        measurement => "m1",
+        tags => [{"tag", "tag1"}],
+        fields => [{"field", "field1"}],
+        timestamp => "${timestamp1}"
+    }},
+    {"m2,tag=tag2 field=field2", #{
+        measurement => "m2",
+        tags => [{"tag", "tag2"}],
+        fields => [{"field", "field2"}],
+        timestamp => undefined
+    }},
+    {"m3 field=field3 ${timestamp3}", #{
+        measurement => "m3",
+        tags => [],
+        fields => [{"field", "field3"}],
+        timestamp => "${timestamp3}"
+    }},
+    {"m4 field=field4", #{
+        measurement => "m4",
+        tags => [],
+        fields => [{"field", "field4"}],
+        timestamp => undefined
+    }},
+    {"m5,tag=tag5,tag_a=tag5a,tag_b=tag5b field=field5,field_a=field5a,field_b=field5b ${timestamp5}",
+        #{
+            measurement => "m5",
+            tags => [{"tag", "tag5"}, {"tag_a", "tag5a"}, {"tag_b", "tag5b"}],
+            fields => [{"field", "field5"}, {"field_a", "field5a"}, {"field_b", "field5b"}],
+            timestamp => "${timestamp5}"
+        }},
+    {"m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=field6,field_a=field6a,field_b=field6b", #{
+        measurement => "m6",
+        tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}],
+        fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}],
+        timestamp => undefined
+    }},
+    {"m7,tag=tag7,tag_a=\"tag7a\",tag_b=tag7b field=\"field7\",field_a=field7a,field_b=\"field7b\"",
+        #{
+            measurement => "m7",
+            tags => [{"tag", "tag7"}, {"tag_a", "\"tag7a\""}, {"tag_b", "tag7b"}],
+            fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b"}],
+            timestamp => undefined
+        }},
+    {"m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a,field_b=\"field8b\" ${timestamp8}",
+        #{
+            measurement => "m8",
+            tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}],
+            fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "field8b"}],
+            timestamp => "${timestamp8}"
+        }},
+    {"m9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}",
+        #{
+            measurement => "m9",
+            tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}],
+            fields => [{"field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}],
+            timestamp => "${timestamp9}"
+        }},
+    {"m10 field=\"\" ${timestamp10}", #{
+        measurement => "m10",
+        tags => [],
+        fields => [{"field", ""}],
+        timestamp => "${timestamp10}"
+    }}
+]).
+
+-define(VALID_LINE_EXTRA_SPACES_PARSED_PAIRS, [
+    {"\n  m1,tag=tag1  field=field1  ${timestamp1} \n", #{
+        measurement => "m1",
+        tags => [{"tag", "tag1"}],
+        fields => [{"field", "field1"}],
+        timestamp => "${timestamp1}"
+    }},
+    {"  m2,tag=tag2  field=field2  ", #{
+        measurement => "m2",
+        tags => [{"tag", "tag2"}],
+        fields => [{"field", "field2"}],
+        timestamp => undefined
+    }},
+    {" m3  field=field3   ${timestamp3}  ", #{
+        measurement => "m3",
+        tags => [],
+        fields => [{"field", "field3"}],
+        timestamp => "${timestamp3}"
+    }},
+    {" \n m4  field=field4\n ", #{
+        measurement => "m4",
+        tags => [],
+        fields => [{"field", "field4"}],
+        timestamp => undefined
+    }},
+    {" \n m5,tag=tag5,tag_a=tag5a,tag_b=tag5b   field=field5,field_a=field5a,field_b=field5b    ${timestamp5}  \n",
+        #{
+            measurement => "m5",
+            tags => [{"tag", "tag5"}, {"tag_a", "tag5a"}, {"tag_b", "tag5b"}],
+            fields => [{"field", "field5"}, {"field_a", "field5a"}, {"field_b", "field5b"}],
+            timestamp => "${timestamp5}"
+        }},
+    {" m6,tag=tag6,tag_a=tag6a,tag_b=tag6b  field=field6,field_a=field6a,field_b=field6b\n  ", #{
+        measurement => "m6",
+        tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}],
+        fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}],
+        timestamp => undefined
+    }}
+]).
+
+-define(VALID_LINE_PARSED_ESCAPED_CHARS_PAIRS, [
+    {"m\\ =1\\,,\\,tag\\ \\==\\=tag\\ 1\\, \\,fie\\ ld\\ =\\ field\\,1 ${timestamp1}", #{
+        measurement => "m =1,",
+        tags => [{",tag =", "=tag 1,"}],
+        fields => [{",fie ld ", " field,1"}],
+        timestamp => "${timestamp1}"
+    }},
+    {"m2,tag=tag2 field=\"field \\\"2\\\",\n\"", #{
+        measurement => "m2",
+        tags => [{"tag", "tag2"}],
+        fields => [{"field", "field \"2\",\n"}],
+        timestamp => undefined
+    }},
+    {"m\\ 3 field=\"field3\" ${payload.timestamp\\ 3}", #{
+        measurement => "m 3",
+        tags => [],
+        fields => [{"field", "field3"}],
+        timestamp => "${payload.timestamp 3}"
+    }},
+    {"m4 field=\"\\\"field\\\\4\\\"\"", #{
+        measurement => "m4",
+        tags => [],
+        fields => [{"field", "\"field\\4\""}],
+        timestamp => undefined
+    }},
+    {"m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5,field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5}",
+        #{
+            measurement => "m5,mA",
+            tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}],
+            fields => [
+                {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"}
+            ],
+            timestamp => "${timestamp5}"
+        }},
+    {"m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=\"field6\",field_a=\"field6a\",field_b=\"field6b\"",
+        #{
+            measurement => "m6",
+            tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}],
+            fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}],
+            timestamp => undefined
+        }},
+    {"\\ \\ m7\\ \\ ,tag=\\ tag\\,7\\ ,tag_a=\"tag7a\",tag_b\\,tag1=tag7b field=\"field7\",field_a=field7a,field_b=\"field7b\\\\\n\"",
+        #{
+            measurement => "  m7  ",
+            tags => [{"tag", " tag,7 "}, {"tag_a", "\"tag7a\""}, {"tag_b,tag1", "tag7b"}],
+            fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b\\\n"}],
+            timestamp => undefined
+        }},
+    {"m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a,field_b=\"\\\"field\\\" = 8b\" ${timestamp8}",
+        #{
+            measurement => "m8",
+            tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}],
+            fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "\"field\" = 8b"}],
+            timestamp => "${timestamp8}"
+        }},
+    {"m\\9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field\\=field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}",
+        #{
+            measurement => "m\\9",
+            tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}],
+            fields => [{"field=field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}],
+            timestamp => "${timestamp9}"
+        }},
+    {"m\\,10 \"field\\\\\"=\"\" ${timestamp10}", #{
+        measurement => "m,10",
+        tags => [],
+        %% backslash should not be un-escaped in tag key
+        fields => [{"\"field\\\\\"", ""}],
+        timestamp => "${timestamp10}"
+    }}
+]).
+
+-define(VALID_LINE_PARSED_ESCAPED_CHARS_EXTRA_SPACES_PAIRS, [
+    {" \n m\\ =1\\,,\\,tag\\ \\==\\=tag\\ 1\\,   \\,fie\\ ld\\ =\\ field\\,1  ${timestamp1}  ", #{
+        measurement => "m =1,",
+        tags => [{",tag =", "=tag 1,"}],
+        fields => [{",fie ld ", " field,1"}],
+        timestamp => "${timestamp1}"
+    }},
+    {" m2,tag=tag2   field=\"field \\\"2\\\",\n\"  ", #{
+        measurement => "m2",
+        tags => [{"tag", "tag2"}],
+        fields => [{"field", "field \"2\",\n"}],
+        timestamp => undefined
+    }},
+    {"  m\\ 3   field=\"field3\"   ${payload.timestamp\\ 3}  ", #{
+        measurement => "m 3",
+        tags => [],
+        fields => [{"field", "field3"}],
+        timestamp => "${payload.timestamp 3}"
+    }},
+    {"   m4       field=\"\\\"field\\\\4\\\"\"    ", #{
+        measurement => "m4",
+        tags => [],
+        fields => [{"field", "\"field\\4\""}],
+        timestamp => undefined
+    }},
+    {" m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b   \\ field\\ =field5,field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b   ${timestamp5}    ",
+        #{
+            measurement => "m5,mA",
+            tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}],
+            fields => [
+                {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"}
+            ],
+            timestamp => "${timestamp5}"
+        }},
+    {"  m6,tag=tag6,tag_a=tag6a,tag_b=tag6b   field=\"field6\",field_a=\"field6a\",field_b=\"field6b\"  ",
+        #{
+            measurement => "m6",
+            tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}],
+            fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}],
+            timestamp => undefined
+        }}
+]).
+
+invalid_write_syntax_line_test_() ->
+    [?_assertThrow(_, to_influx_lines(L)) || L <- ?INVALID_LINES].
+
+invalid_write_syntax_multiline_test_() ->
+    LinesList = [
+        join("\n", ?INVALID_LINES),
+        join("\n\n\n", ?INVALID_LINES),
+        join("\n\n", lists:reverse(?INVALID_LINES))
+    ],
+    [?_assertThrow(_, to_influx_lines(Lines)) || Lines <- LinesList].
+
+valid_write_syntax_test_() ->
+    test_pairs(?VALID_LINE_PARSED_PAIRS).
+
+valid_write_syntax_with_extra_spaces_test_() ->
+    test_pairs(?VALID_LINE_EXTRA_SPACES_PARSED_PAIRS).
+
+valid_write_syntax_escaped_chars_test_() ->
+    test_pairs(?VALID_LINE_PARSED_ESCAPED_CHARS_PAIRS).
+
+valid_write_syntax_escaped_chars_with_extra_spaces_test_() ->
+    test_pairs(?VALID_LINE_PARSED_ESCAPED_CHARS_EXTRA_SPACES_PAIRS).
+
+test_pairs(PairsList) ->
+    {Lines, AllExpected} = lists:unzip(PairsList),
+    JoinedLines = join("\n", Lines),
+    JoinedLines1 = join("\n\n\n", Lines),
+    JoinedLines2 = join("\n\n", lists:reverse(Lines)),
+    SingleLineTests =
+        [
+            ?_assertEqual([Expected], to_influx_lines(Line))
+         || {Line, Expected} <- PairsList
+        ],
+    JoinedLinesTests =
+        [
+            ?_assertEqual(AllExpected, to_influx_lines(JoinedLines)),
+            ?_assertEqual(AllExpected, to_influx_lines(JoinedLines1)),
+            ?_assertEqual(lists:reverse(AllExpected), to_influx_lines(JoinedLines2))
+        ],
+    SingleLineTests ++ JoinedLinesTests.
+
+join(Sep, LinesList) ->
+    lists:flatten(lists:join(Sep, LinesList)).

From 08e249dc4d57b9e0288e07f9a9e5b12f3a4379a0 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Mon, 20 Mar 2023 16:53:55 +0100
Subject: [PATCH 08/52] docs: refine README.md for the emqx_rule_engine app

This commit adds more content to the readme file of the `emqx_rule_engine`
app. The aim of the changes are to make it easier to understand what the
rule engine is and what it does.

Fixes:
https://emqx.atlassian.net/browse/EMQX-9229
---
 apps/emqx_rule_engine/README.md | 77 ++++++++++++++++++++++++++-------
 1 file changed, 61 insertions(+), 16 deletions(-)

diff --git a/apps/emqx_rule_engine/README.md b/apps/emqx_rule_engine/README.md
index 2485ff534..f1533a0e3 100644
--- a/apps/emqx_rule_engine/README.md
+++ b/apps/emqx_rule_engine/README.md
@@ -1,23 +1,46 @@
 
-# emqx-rule-engine
+# Emqx Rule Engine
 
-IoT Rule Engine
+The rule engine's goal is to provide a simple and flexible way to transform and
+reroute the messages coming to the EMQX broker. For example, one message
+containing measurements from multiple sensors of different types can be
+transformed into multiple messages.
+
+
+## Concepts
+
+A rule is quite simple. A rule describes which messages it affects by
+specifying a topic filter and a set of conditions that need to be met. If a
+message matches the topic filter and all the conditions are met, the rule is
+triggered. The rule can then transform the message and route it to a different
+topic, or send it to another service (defined by an EMQX bridge). The rule
+engine's message data transformation is designed to work well with JSON
+formatted data.
+
+
+A rule consists of the three parts **MATCH**, **TRANSFORM** and **ACTIONS** that are
+described below:
+
+* **MATCH** - The rule's trigger condition. The rule is triggered when a message 
+  arrives that matches the topic filter and all the specified conditions are met.
+* **TRANSFORM** - The rule's data transformation. The rule can select data from the 
+  incoming message and transform it into a new message.
+* **ACTIONS** - The rule's action(s). The rule can have one or more actions. The 
+  actions are executed when the rule is triggered. The actions can be to route
+  the message to a different topic, or send it to another service (defined by
+  an EMQX bridge). 
 
-## Concept
 
-```
-iot rule "Rule Name"
-  when
-     match TopicFilters and Conditions
-  select
-    para1 = val1
-    para2 = val2
-  then
-    take action(#{para2 => val1, #para2 => val2})
-```
 
 ## Architecture
 
+The following diagram shows how the rule engine is integrated with the EMQX
+message broker. Incoming messages are checked against the rules, and if a rule
+matches, it is triggered with the message as input. The rule can then transform
+or split the message and/or route it to a different topic, or send it to another
+service (defined by an EMQX bridge).
+
+
 ```
           |-----------------|
  Pub ---->| Message Routing |----> Sub
@@ -28,11 +51,33 @@ iot rule "Rule Name"
           |   Rule Engine   |
           |-----------------|
                |      |
-        Backends Services Bridges
+           Services Bridges (defined by EMQX bridges)
 ```
 
-## SQL for Rule query statement
+## Domain Specific Language for Rules
+
+The **MATCH** and **TRANSFORM** parts of the rule are specified using a domain
+specific language that looks similar to SQL. The following is an example of a
+rule engine statement. The `from "topic/a"` part specifies the topic filter
+(only messages to the topic `topic/a` will be considered). The `where t > 50`
+part specifies the condition that needs to be met for the rule to be triggered.
+The `select id, time, temperature as t` part specifies the data transformation
+(the selected fields will remain in the transformed message payload). The `as
+t` part specifies that the `temperature` field name is changed to `t` in the
+output message. The name `t` can also be used in the where part of the rule as
+an alias for `t`.
+
 
 ```
-select id, time, temperature as t from "topic/a" where t > 50;
+select id, time, temperature as t from "topic/a" where t > 50
 ```
+
+ This just scratches the surface of what is possible with the rule engine. The
+ full documentation is available at [EMQX Rule
+ Engine](https://www.emqx.io/docs/en/v5.0/data-integration/rules.html). For
+ example, there are many built-in functions that can be used in the rule engine
+ language to help in doing transformations and matching. One of the [built-in
+ functions allows you to run JQ
+ queries](https://www.emqx.io/docs/en/v5.0/data-integration/rule-sql-jq.html)
+ which allows you to do complex transformations of the message.
+

From 6f71898546977403f4354a1ac025b0d2b1cd73a7 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 20 Mar 2023 12:48:41 +0100
Subject: [PATCH 09/52] fix: upgrade esockd from 5.9.4 to 5.9.6

changes in esockd are

* 5.9.5: added API to retrieve SNI
* 5.9.6: avoid error log if socket is closed before sending
         proxy protocol headers
---
 apps/emqx/rebar.config     | 2 +-
 changes/ce/fix-10174.en.md | 2 ++
 changes/ce/fix-10174.zh.md | 2 ++
 mix.exs                    | 2 +-
 rebar.config               | 2 +-
 5 files changed, 7 insertions(+), 3 deletions(-)
 create mode 100644 changes/ce/fix-10174.en.md
 create mode 100644 changes/ce/fix-10174.zh.md

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 229979f6c..bb66280f0 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -26,7 +26,7 @@
     {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}},
     {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
-    {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
+    {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
     {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.0"}}},
diff --git a/changes/ce/fix-10174.en.md b/changes/ce/fix-10174.en.md
new file mode 100644
index 000000000..213af19da
--- /dev/null
+++ b/changes/ce/fix-10174.en.md
@@ -0,0 +1,2 @@
+Upgrade library `esockd` from 5.9.4 to 5.9.6.
+Fix an unnecessary error level logging when a connection is closed before proxy protocol header is sent by the proxy.
diff --git a/changes/ce/fix-10174.zh.md b/changes/ce/fix-10174.zh.md
new file mode 100644
index 000000000..435056280
--- /dev/null
+++ b/changes/ce/fix-10174.zh.md
@@ -0,0 +1,2 @@
+依赖库 `esockd` 从 5.9.4 升级到 5.9.6。
+修复了一个不必要的错误日志。如果连接在 proxy protocol 包头还没有发送前就关闭了， 则不打印错误日志。
diff --git a/mix.exs b/mix.exs
index 42354f8dc..c4677dd1d 100644
--- a/mix.exs
+++ b/mix.exs
@@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do
       {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true},
       {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
       {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
-      {:esockd, github: "emqx/esockd", tag: "5.9.4", override: true},
+      {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true},
       {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true},
       {:ekka, github: "emqx/ekka", tag: "0.14.5", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
diff --git a/rebar.config b/rebar.config
index 5ce9138ce..62fadbe51 100644
--- a/rebar.config
+++ b/rebar.config
@@ -54,7 +54,7 @@
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
-    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}
+    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}
     , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}

From c739d9957c6257f5ec0e0d1da74dfc5120922a35 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 20 Mar 2023 18:25:35 +0100
Subject: [PATCH 10/52] docs: add readme for emqx_ctl app

---
 apps/emqx_ctl/README.md | 43 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 40 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_ctl/README.md b/apps/emqx_ctl/README.md
index a91342606..2638031e6 100644
--- a/apps/emqx_ctl/README.md
+++ b/apps/emqx_ctl/README.md
@@ -1,4 +1,41 @@
-emqx_ctl
-=====
+# emqx_ctl
 
-Backend module for `emqx_ctl` command.
+This application accepts dynamic `emqx ctl` command registrations so plugins can add their own commands.
+Please note that the 'proxy' command `emqx_ctl` is considered deprecated, going forward, please use `emqx ctl` instead.
+
+## Add a new command
+
+To add a new command, the application must implement a callback function to handle the command, and register the command with `emqx_ctl:register_command/2` API.
+
+### Register
+
+To add a new command which can be executed from `emqx ctl`, the application must call `emqx_ctl:register_command/2` API to register the command.
+
+For example, to add a new command `myplugin` which is to be executed as `emqx ctl myplugin`, the application must call `emqx_ctl:register_command/2` API as follows:
+
+```erlang
+emqx_ctl:register_command(mypluin, {myplugin_cli, cmd}).
+```
+
+### Callback
+
+The callback function must be exported by the application and must have the following signature:
+
+```erlang
+cmd([Arg1, Arg2, ...]) -> ok.
+```
+
+It must also implement a special clause to handle the `usage` argument:
+
+```erlang
+cmd([usage]) -> "myplugin [arg1] [arg2] ...";
+```
+
+### Utility
+
+The `emqx_ctl` application provides some utility functions which help to format the output of the command.
+For example `emqx_ctl:print/2` and `emqx_ctl:usage/1`.
+
+## Reference
+
+[emqx_management_cli](../emqx_management/src/emqx_mgmt_cli.erl) can be taken as a reference for how to implement a command.

From 7a71d64686ea989b4a221a10778d1d15ba837003 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Thu, 16 Mar 2023 13:58:52 -0300
Subject: [PATCH 11/52] feat(crl): add crl check support

---
 apps/emqx/i18n/emqx_schema_i18n.conf          |   50 +
 apps/emqx/src/emqx_crl_cache.erl              |  314 +++++
 apps/emqx/src/emqx_kernel_sup.erl             |    3 +-
 apps/emqx/src/emqx_listeners.erl              |   17 +-
 apps/emqx/src/emqx_schema.erl                 |   64 +-
 apps/emqx/src/emqx_ssl_crl_cache.erl          |  237 ++++
 apps/emqx/test/emqx_common_test_helpers.erl   |   15 +-
 apps/emqx/test/emqx_crl_cache_SUITE.erl       | 1057 +++++++++++++++++
 .../ca-chain.cert.pem                         |   68 ++
 .../client-no-dist-points.cert.pem            |   32 +
 .../client-no-dist-points.key.pem             |   28 +
 .../client-revoked.cert.pem                   |   32 +
 .../client-revoked.key.pem                    |   28 +
 .../emqx_crl_cache_SUITE_data/client.cert.pem |   32 +
 .../emqx_crl_cache_SUITE_data/client.key.pem  |   28 +
 .../client1.cert.pem                          |   32 +
 .../emqx_crl_cache_SUITE_data/client1.key.pem |   28 +
 .../client2.cert.pem                          |   32 +
 .../emqx_crl_cache_SUITE_data/client2.key.pem |   28 +
 .../client3.cert.pem                          |   32 +
 .../emqx_crl_cache_SUITE_data/client3.key.pem |   28 +
 .../test/emqx_crl_cache_SUITE_data/crl.pem    |   20 +
 .../test/emqx_crl_cache_SUITE_data/emqx.conf  |   12 +
 .../emqx_crl_cache_http_server.erl            |   67 ++
 .../emqx_just_verify.conf                     |   12 +
 .../intermediate-not-revoked.crl.pem          |   19 +
 .../intermediate-revoked-no-dp.crl.pem        |   19 +
 .../intermediate-revoked.crl.pem              |   20 +
 .../intermediate.crl.pem                      |   20 +
 .../emqx_crl_cache_SUITE_data/server.cert.pem |   35 +
 .../emqx_crl_cache_SUITE_data/server.key.pem  |   28 +
 apps/emqx/test/emqx_ocsp_cache_SUITE.erl      |    2 +-
 changes/ce/feat-10164.en.md                   |    1 +
 rebar.config                                  |    8 +-
 34 files changed, 2436 insertions(+), 12 deletions(-)
 create mode 100644 apps/emqx/src/emqx_crl_cache.erl
 create mode 100644 apps/emqx/src/emqx_ssl_crl_cache.erl
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE.erl
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/ca-chain.cert.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.cert.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.key.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.cert.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.key.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client.cert.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client.key.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client1.cert.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client1.key.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client2.cert.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client2.key.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client3.cert.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client3.key.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/crl.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_crl_cache_http_server.erl
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-not-revoked.crl.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked-no-dp.crl.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked.crl.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate.crl.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/server.cert.pem
 create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/server.key.pem
 create mode 100644 changes/ce/feat-10164.en.md

diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf
index 5a48c218a..e05ee4c35 100644
--- a/apps/emqx/i18n/emqx_schema_i18n.conf
+++ b/apps/emqx/i18n/emqx_schema_i18n.conf
@@ -1810,6 +1810,56 @@ server_ssl_opts_schema_ocsp_refresh_http_timeout {
     }
 }
 
+server_ssl_opts_schema_enable_crl_check {
+    desc {
+        en: "Whether to enable CRL verification for this listener."
+        zh: "是否为该监听器启用 CRL 检查。"
+    }
+    label: {
+        en: "Enable CRL Check"
+        zh: "启用 CRL 检查"
+    }
+}
+
+crl_cache_refresh_http_timeout {
+    desc {
+        en: "The timeout for the HTTP request when fetching CRLs.  This is"
+            " global for all listeners."
+        zh: "获取 CRLs 时 HTTP 请求的超时。 这对所有监听器来说是全局的。"
+    }
+    label: {
+        en: "CRL Cache Refresh HTTP Timeout"
+        zh: "CRL 缓存刷新 HTTP 超时"
+    }
+}
+
+crl_cache_refresh_interval {
+    desc {
+        en: "The period to refresh the CRLs from the servers.  This is global"
+            " for all URLs and listeners."
+        zh: "从服务器刷新CRL的周期。 这对所有URL和监听器来说是全局性的。"
+    }
+    label: {
+        en: "CRL Cache Refresh Interval"
+        zh: "CRL 缓存刷新间隔"
+    }
+}
+
+crl_cache_capacity {
+    desc {
+        en: "The maximum number of CRL URLs that can be held in cache.  If the cache is at"
+            " full capacity and a new URL must be fetched, then it'll evict the oldest"
+            " inserted URL in the cache."
+        zh: "缓存中可容纳的 CRL URL 的最大数量。"
+            " 如果缓存的容量已满，并且必须获取一个新的 URL，"
+            "那么它将驱逐缓存中插入的最老的 URL。"
+    }
+    label: {
+        en: "CRL Cache Capacity"
+        zh: "CRL 缓存容量"
+    }
+}
+
 fields_listeners_tcp {
     desc {
         en: """TCP listeners."""
diff --git a/apps/emqx/src/emqx_crl_cache.erl b/apps/emqx/src/emqx_crl_cache.erl
new file mode 100644
index 000000000..79e47a6dc
--- /dev/null
+++ b/apps/emqx/src/emqx_crl_cache.erl
@@ -0,0 +1,314 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% @doc EMQX CRL cache.
+%%--------------------------------------------------------------------
+
+-module(emqx_crl_cache).
+
+%% API
+-export([
+    start_link/0,
+    start_link/1,
+    register_der_crls/2,
+    refresh/1,
+    evict/1
+]).
+
+%% gen_server callbacks
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+%% internal exports
+-export([http_get/2]).
+
+-behaviour(gen_server).
+
+-include("logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-define(HTTP_TIMEOUT, timer:seconds(15)).
+-define(RETRY_TIMEOUT, 5_000).
+-ifdef(TEST).
+-define(MIN_REFRESH_PERIOD, timer:seconds(5)).
+-else.
+-define(MIN_REFRESH_PERIOD, timer:minutes(1)).
+-endif.
+-define(DEFAULT_REFRESH_INTERVAL, timer:minutes(15)).
+-define(DEFAULT_CACHE_CAPACITY, 100).
+
+-record(state, {
+    refresh_timers = #{} :: #{binary() => timer:tref()},
+    refresh_interval = timer:minutes(15) :: timer:time(),
+    http_timeout = ?HTTP_TIMEOUT :: timer:time(),
+    %% keeps track of URLs by insertion time
+    insertion_times = gb_trees:empty() :: gb_trees:tree(timer:time(), url()),
+    %% the set of cached URLs, for testing if an URL is already
+    %% registered.
+    cached_urls = sets:new([{version, 2}]) :: sets:set(url()),
+    cache_capacity = 100 :: pos_integer(),
+    %% for future use
+    extra = #{} :: map()
+}).
+-type url() :: uri_string:uri_string().
+-type state() :: #state{}.
+
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
+
+start_link() ->
+    Config = gather_config(),
+    start_link(Config).
+
+start_link(Config = #{cache_capacity := _, refresh_interval := _, http_timeout := _}) ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, Config, []).
+
+-spec refresh(url()) -> ok.
+refresh(URL) ->
+    gen_server:cast(?MODULE, {refresh, URL}).
+
+-spec evict(url()) -> ok.
+evict(URL) ->
+    gen_server:cast(?MODULE, {evict, URL}).
+
+%% Adds CRLs in DER format to the cache and register them for periodic
+%% refresh.
+-spec register_der_crls(url(), [public_key:der_encoded()]) -> ok.
+register_der_crls(URL, CRLs) when is_list(CRLs) ->
+    gen_server:cast(?MODULE, {register_der_crls, URL, CRLs}).
+
+%%--------------------------------------------------------------------
+%% gen_server behaviour
+%%--------------------------------------------------------------------
+
+init(Config) ->
+    #{
+        cache_capacity := CacheCapacity,
+        refresh_interval := RefreshIntervalMS,
+        http_timeout := HTTPTimeoutMS
+    } = Config,
+    State = #state{
+        cache_capacity = CacheCapacity,
+        refresh_interval = RefreshIntervalMS,
+        http_timeout = HTTPTimeoutMS
+    },
+    {ok, State}.
+
+handle_call(Call, _From, State) ->
+    {reply, {error, {bad_call, Call}}, State}.
+
+handle_cast({evict, URL}, State0 = #state{refresh_timers = RefreshTimers0}) ->
+    emqx_ssl_crl_cache:delete(URL),
+    MTimer = maps:get(URL, RefreshTimers0, undefined),
+    emqx_misc:cancel_timer(MTimer),
+    RefreshTimers = maps:without([URL], RefreshTimers0),
+    State = State0#state{refresh_timers = RefreshTimers},
+    ?tp(
+        crl_cache_evict,
+        #{url => URL}
+    ),
+    {noreply, State};
+handle_cast({register_der_crls, URL, CRLs}, State0) ->
+    handle_register_der_crls(State0, URL, CRLs);
+handle_cast({refresh, URL}, State0) ->
+    case do_http_fetch_and_cache(URL, State0#state.http_timeout) of
+        {error, Error} ->
+            ?tp(crl_refresh_failure, #{error => Error, url => URL}),
+            ?SLOG(error, #{
+                msg => "failed_to_fetch_crl_response",
+                url => URL,
+                error => Error
+            }),
+            {noreply, ensure_timer(URL, State0, ?RETRY_TIMEOUT)};
+        {ok, _CRLs} ->
+            ?SLOG(debug, #{
+                msg => "fetched_crl_response",
+                url => URL
+            }),
+            {noreply, ensure_timer(URL, State0)}
+    end;
+handle_cast(_Cast, State) ->
+    {noreply, State}.
+
+handle_info(
+    {timeout, TRef, {refresh, URL}},
+    State = #state{
+        refresh_timers = RefreshTimers,
+        http_timeout = HTTPTimeoutMS
+    }
+) ->
+    case maps:get(URL, RefreshTimers, undefined) of
+        TRef ->
+            ?tp(debug, crl_refresh_timer, #{url => URL}),
+            case do_http_fetch_and_cache(URL, HTTPTimeoutMS) of
+                {error, Error} ->
+                    ?SLOG(error, #{
+                        msg => "failed_to_fetch_crl_response",
+                        url => URL,
+                        error => Error
+                    }),
+                    {noreply, ensure_timer(URL, State, ?RETRY_TIMEOUT)};
+                {ok, _CRLs} ->
+                    ?tp(debug, crl_refresh_timer_done, #{url => URL}),
+                    {noreply, ensure_timer(URL, State)}
+            end;
+        _ ->
+            {noreply, State}
+    end;
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+%%--------------------------------------------------------------------
+%% internal functions
+%%--------------------------------------------------------------------
+
+http_get(URL, HTTPTimeout) ->
+    httpc:request(
+        get,
+        {URL, [{"connection", "close"}]},
+        [{timeout, HTTPTimeout}],
+        [{body_format, binary}]
+    ).
+
+do_http_fetch_and_cache(URL, HTTPTimeoutMS) ->
+    ?tp(crl_http_fetch, #{crl_url => URL}),
+    Resp = ?MODULE:http_get(URL, HTTPTimeoutMS),
+    case Resp of
+        {ok, {{_, 200, _}, _, Body}} ->
+            case parse_crls(Body) of
+                error ->
+                    {error, invalid_crl};
+                CRLs ->
+                    %% Note: must ensure it's a string and not a
+                    %% binary because that's what the ssl manager uses
+                    %% when doing lookups.
+                    emqx_ssl_crl_cache:insert(to_string(URL), {der, CRLs}),
+                    ?tp(crl_cache_insert, #{url => URL, crls => CRLs}),
+                    {ok, CRLs}
+            end;
+        {ok, {{_, Code, _}, _, Body}} ->
+            {error, {bad_response, #{code => Code, body => Body}}};
+        {error, Error} ->
+            {error, {http_error, Error}}
+    end.
+
+parse_crls(Bin) ->
+    try
+        [CRL || {'CertificateList', CRL, not_encrypted} <- public_key:pem_decode(Bin)]
+    catch
+        _:_ ->
+            error
+    end.
+
+ensure_timer(URL, State = #state{refresh_interval = Timeout}) ->
+    ensure_timer(URL, State, Timeout).
+
+ensure_timer(URL, State = #state{refresh_timers = RefreshTimers0}, Timeout) ->
+    ?tp(crl_cache_ensure_timer, #{url => URL, timeout => Timeout}),
+    MTimer = maps:get(URL, RefreshTimers0, undefined),
+    emqx_misc:cancel_timer(MTimer),
+    RefreshTimers = RefreshTimers0#{
+        URL => emqx_misc:start_timer(
+            Timeout,
+            {refresh, URL}
+        )
+    },
+    State#state{refresh_timers = RefreshTimers}.
+
+-spec gather_config() ->
+    #{
+        cache_capacity := pos_integer(),
+        refresh_interval := timer:time(),
+        http_timeout := timer:time()
+    }.
+gather_config() ->
+    %% TODO: add a config handler to refresh the config when those
+    %% globals change?
+    CacheCapacity = emqx_config:get([crl_cache, capacity], ?DEFAULT_CACHE_CAPACITY),
+    RefreshIntervalMS0 = emqx_config:get([crl_cache, refresh_interval], ?DEFAULT_REFRESH_INTERVAL),
+    MinimumRefreshInverval = ?MIN_REFRESH_PERIOD,
+    RefreshIntervalMS = max(RefreshIntervalMS0, MinimumRefreshInverval),
+    HTTPTimeoutMS = emqx_config:get([crl_cache, http_timeout], ?HTTP_TIMEOUT),
+    #{
+        cache_capacity => CacheCapacity,
+        refresh_interval => RefreshIntervalMS,
+        http_timeout => HTTPTimeoutMS
+    }.
+
+-spec handle_register_der_crls(state(), url(), [public_key:der_encoded()]) -> {noreply, state()}.
+handle_register_der_crls(State0, URL0, CRLs) ->
+    #state{cached_urls = CachedURLs0} = State0,
+    URL = to_string(URL0),
+    case sets:is_element(URL, CachedURLs0) of
+        true ->
+            {noreply, State0};
+        false ->
+            emqx_ssl_crl_cache:insert(URL, {der, CRLs}),
+            ?tp(debug, new_crl_url_inserted, #{url => URL}),
+            State1 = do_register_url(State0, URL),
+            State2 = handle_cache_overflow(State1),
+            State = ensure_timer(URL, State2),
+            {noreply, State}
+    end.
+
+-spec do_register_url(state(), url()) -> state().
+do_register_url(State0, URL) ->
+    #state{
+        cached_urls = CachedURLs0,
+        insertion_times = InsertionTimes0
+    } = State0,
+    Now = erlang:monotonic_time(),
+    CachedURLs = sets:add_element(URL, CachedURLs0),
+    InsertionTimes = gb_trees:enter(Now, URL, InsertionTimes0),
+    State0#state{
+        cached_urls = CachedURLs,
+        insertion_times = InsertionTimes
+    }.
+
+-spec handle_cache_overflow(state()) -> state().
+handle_cache_overflow(State0) ->
+    #state{
+        cached_urls = CachedURLs0,
+        insertion_times = InsertionTimes0,
+        cache_capacity = CacheCapacity,
+        refresh_timers = RefreshTimers0
+    } = State0,
+    case sets:size(CachedURLs0) > CacheCapacity of
+        false ->
+            State0;
+        true ->
+            {_Time, OldestURL, InsertionTimes} = gb_trees:take_smallest(InsertionTimes0),
+            emqx_ssl_crl_cache:delete(OldestURL),
+            MTimer = maps:get(OldestURL, RefreshTimers0, undefined),
+            emqx_misc:cancel_timer(MTimer),
+            RefreshTimers = maps:remove(OldestURL, RefreshTimers0),
+            CachedURLs = sets:del_element(OldestURL, CachedURLs0),
+            ?tp(debug, crl_cache_overflow, #{oldest_url => OldestURL}),
+            State0#state{
+                insertion_times = InsertionTimes,
+                cached_urls = CachedURLs,
+                refresh_timers = RefreshTimers
+            }
+    end.
+
+to_string(B) when is_binary(B) ->
+    binary_to_list(B);
+to_string(L) when is_list(L) ->
+    L.
diff --git a/apps/emqx/src/emqx_kernel_sup.erl b/apps/emqx/src/emqx_kernel_sup.erl
index 9d2f71068..1027ef639 100644
--- a/apps/emqx/src/emqx_kernel_sup.erl
+++ b/apps/emqx/src/emqx_kernel_sup.erl
@@ -36,7 +36,8 @@ init([]) ->
             child_spec(emqx_stats, worker),
             child_spec(emqx_metrics, worker),
             child_spec(emqx_authn_authz_metrics_sup, supervisor),
-            child_spec(emqx_ocsp_cache, worker)
+            child_spec(emqx_ocsp_cache, worker),
+            child_spec(emqx_crl_cache, worker)
         ]
     }}.
 
diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index 97bc15ad3..b351212a7 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -487,7 +487,8 @@ esockd_opts(ListenerId, Type, Opts0) ->
             tcp ->
                 Opts3#{tcp_options => tcp_opts(Opts0)};
             ssl ->
-                OptsWithSNI = inject_sni_fun(ListenerId, Opts0),
+                OptsWithCRL = inject_crl_config(Opts0),
+                OptsWithSNI = inject_sni_fun(ListenerId, OptsWithCRL),
                 SSLOpts = ssl_opts(OptsWithSNI),
                 Opts3#{ssl_options => SSLOpts, tcp_options => tcp_opts(Opts0)}
         end
@@ -794,3 +795,17 @@ inject_sni_fun(ListenerId, Conf = #{ssl_options := #{ocsp := #{enable_ocsp_stapl
     emqx_ocsp_cache:inject_sni_fun(ListenerId, Conf);
 inject_sni_fun(_ListenerId, Conf) ->
     Conf.
+
+inject_crl_config(
+    Conf = #{ssl_options := #{enable_crl_check := true} = SSLOpts}
+) ->
+    HTTPTimeout = emqx_config:get([crl_cache, http_timeout], timer:seconds(15)),
+    Conf#{
+        ssl_options := SSLOpts#{
+            %% `crl_check => true' doesn't work
+            crl_check => peer,
+            crl_cache => {emqx_ssl_crl_cache, {internal, [{http, HTTPTimeout}]}}
+        }
+    };
+inject_crl_config(Conf) ->
+    Conf.
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index b18534a42..25b1dd1d6 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -226,6 +226,11 @@ roots(low) ->
             sc(
                 ref("trace"),
                 #{}
+            )},
+        {"crl_cache",
+            sc(
+                ref("crl_cache"),
+                #{}
             )}
     ].
 
@@ -794,6 +799,40 @@ fields("listeners") ->
                 }
             )}
     ];
+fields("crl_cache") ->
+    %% Note: we make the refresh interval and HTTP timeout global (not
+    %% per-listener) because multiple SSL listeners might point to the
+    %% same URL.  If they had diverging timeout options, it would be
+    %% confusing.
+    [
+        {"refresh_interval",
+            sc(
+                duration(),
+                #{
+                    hidden => true,
+                    default => <<"15m">>,
+                    desc => ?DESC("crl_cache_refresh_interval")
+                }
+            )},
+        {"http_timeout",
+            sc(
+                duration(),
+                #{
+                    hidden => true,
+                    default => <<"15s">>,
+                    desc => ?DESC("crl_cache_refresh_http_timeout")
+                }
+            )},
+        {"capacity",
+            sc(
+                pos_integer(),
+                #{
+                    hidden => true,
+                    default => 100,
+                    desc => ?DESC("crl_cache_capacity")
+                }
+            )}
+    ];
 fields("mqtt_tcp_listener") ->
     mqtt_listener(1883) ++
         [
@@ -2063,6 +2102,8 @@ desc("shared_subscription_group") ->
     "Per group dispatch strategy for shared subscription";
 desc("ocsp") ->
     "Per listener OCSP Stapling configuration.";
+desc("crl_cache") ->
+    "Global CRL cache options.";
 desc(_) ->
     undefined.
 
@@ -2260,13 +2301,22 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
                             required => false,
                             validator => fun ocsp_inner_validator/1
                         }
+                    )},
+                {"enable_crl_check",
+                    sc(
+                        boolean(),
+                        #{
+                            default => false,
+                            desc => ?DESC("server_ssl_opts_schema_enable_crl_check")
+                        }
                     )}
             ]
         ].
 
 mqtt_ssl_listener_ssl_options_validator(Conf) ->
     Checks = [
-        fun ocsp_outer_validator/1
+        fun ocsp_outer_validator/1,
+        fun crl_outer_validator/1
     ],
     case emqx_misc:pipeline(Checks, Conf, not_used) of
         {ok, _, _} ->
@@ -2301,6 +2351,18 @@ ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := true} = Conf) ->
     ),
     ok.
 
+crl_outer_validator(
+    #{<<"enable_crl_check">> := true} = SSLOpts
+) ->
+    case maps:get(<<"verify">>, SSLOpts) of
+        verify_peer ->
+            ok;
+        _ ->
+            {error, "verify must be verify_peer when CRL check is enabled"}
+    end;
+crl_outer_validator(_SSLOpts) ->
+    ok.
+
 %% @doc Make schema for SSL client.
 -spec client_ssl_opts_schema(map()) -> hocon_schema:field_schema().
 client_ssl_opts_schema(Defaults) ->
diff --git a/apps/emqx/src/emqx_ssl_crl_cache.erl b/apps/emqx/src/emqx_ssl_crl_cache.erl
new file mode 100644
index 000000000..13eccbd83
--- /dev/null
+++ b/apps/emqx/src/emqx_ssl_crl_cache.erl
@@ -0,0 +1,237 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2015-2022. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%----------------------------------------------------------------------
+% Based on `otp/lib/ssl/src/ssl_crl_cache.erl'
+%----------------------------------------------------------------------
+
+%----------------------------------------------------------------------
+%% Purpose: Simple default CRL cache
+%%----------------------------------------------------------------------
+
+-module(emqx_ssl_crl_cache).
+
+-include_lib("ssl/src/ssl_internal.hrl").
+-include_lib("public_key/include/public_key.hrl").
+
+-behaviour(ssl_crl_cache_api).
+
+-export_type([crl_src/0, uri/0]).
+-type crl_src() :: {file, file:filename()} | {der, public_key:der_encoded()}.
+-type uri() :: uri_string:uri_string().
+
+-export([lookup/3, select/2, fresh_crl/2]).
+-export([insert/1, insert/2, delete/1]).
+
+%% Allow usage of OTP certificate record fields (camelCase).
+-elvis([
+    {elvis_style, atom_naming_convention, #{
+        regex => "^([a-z][a-z0-9]*_?)([a-zA-Z0-9]*_?)*$",
+        enclosed_atoms => ".*"
+    }}
+]).
+
+%%====================================================================
+%% Cache callback API
+%%====================================================================
+
+lookup(
+    #'DistributionPoint'{distributionPoint = {fullName, Names}},
+    _Issuer,
+    CRLDbInfo
+) ->
+    get_crls(Names, CRLDbInfo);
+lookup(_, _, _) ->
+    not_available.
+
+select(GenNames, CRLDbHandle) when is_list(GenNames) ->
+    lists:flatmap(
+        fun
+            ({directoryName, Issuer}) ->
+                select(Issuer, CRLDbHandle);
+            (_) ->
+                []
+        end,
+        GenNames
+    );
+select(Issuer, {{_Cache, Mapping}, _}) ->
+    case ssl_pkix_db:lookup(Issuer, Mapping) of
+        undefined ->
+            [];
+        CRLs ->
+            CRLs
+    end.
+
+fresh_crl(#'DistributionPoint'{distributionPoint = {fullName, Names}}, CRL) ->
+    case get_crls(Names, undefined) of
+        not_available ->
+            CRL;
+        NewCRL ->
+            NewCRL
+    end.
+
+%%====================================================================
+%% API
+%%====================================================================
+
+insert(CRLs) ->
+    insert(?NO_DIST_POINT, CRLs).
+
+insert(URI, {file, File}) when is_list(URI) ->
+    case file:read_file(File) of
+        {ok, PemBin} ->
+            PemEntries = public_key:pem_decode(PemBin),
+            CRLs = [
+                CRL
+             || {'CertificateList', CRL, not_encrypted} <-
+                    PemEntries
+            ],
+            do_insert(URI, CRLs);
+        Error ->
+            Error
+    end;
+insert(URI, {der, CRLs}) ->
+    do_insert(URI, CRLs).
+
+delete({file, File}) ->
+    case file:read_file(File) of
+        {ok, PemBin} ->
+            PemEntries = public_key:pem_decode(PemBin),
+            CRLs = [
+                CRL
+             || {'CertificateList', CRL, not_encrypted} <-
+                    PemEntries
+            ],
+            ssl_manager:delete_crls({?NO_DIST_POINT, CRLs});
+        Error ->
+            Error
+    end;
+delete({der, CRLs}) ->
+    ssl_manager:delete_crls({?NO_DIST_POINT, CRLs});
+delete(URI) ->
+    case uri_string:normalize(URI, [return_map]) of
+        #{scheme := "http", path := Path} ->
+            ssl_manager:delete_crls(string:trim(Path, leading, "/"));
+        _ ->
+            {error, {only_http_distribution_points_supported, URI}}
+    end.
+
+%%--------------------------------------------------------------------
+%%% Internal functions
+%%--------------------------------------------------------------------
+do_insert(URI, CRLs) ->
+    case uri_string:normalize(URI, [return_map]) of
+        #{scheme := "http", path := Path} ->
+            ssl_manager:insert_crls(string:trim(Path, leading, "/"), CRLs);
+        _ ->
+            {error, {only_http_distribution_points_supported, URI}}
+    end.
+
+get_crls([], _) ->
+    not_available;
+get_crls(
+    [{uniformResourceIdentifier, "http" ++ _ = URL} | Rest],
+    CRLDbInfo
+) ->
+    case cache_lookup(URL, CRLDbInfo) of
+        [] ->
+            handle_http(URL, Rest, CRLDbInfo);
+        CRLs ->
+            CRLs
+    end;
+get_crls([_ | Rest], CRLDbInfo) ->
+    %% unsupported CRL location
+    get_crls(Rest, CRLDbInfo).
+
+http_lookup(URL, Rest, CRLDbInfo, Timeout) ->
+    case application:ensure_started(inets) of
+        ok ->
+            http_get(URL, Rest, CRLDbInfo, Timeout);
+        _ ->
+            get_crls(Rest, CRLDbInfo)
+    end.
+
+http_get(URL, Rest, CRLDbInfo, Timeout) ->
+    case emqx_crl_cache:http_get(URL, Timeout) of
+        {ok, {_Status, _Headers, Body}} ->
+            case Body of
+                <<"-----BEGIN", _/binary>> ->
+                    Pem = public_key:pem_decode(Body),
+                    CRLs = lists:filtermap(
+                        fun
+                            ({'CertificateList', CRL, not_encrypted}) ->
+                                {true, CRL};
+                            (_) ->
+                                false
+                        end,
+                        Pem
+                    ),
+                    emqx_crl_cache:register_der_crls(URL, CRLs),
+                    CRLs;
+                _ ->
+                    try public_key:der_decode('CertificateList', Body) of
+                        _ ->
+                            CRLs = [Body],
+                            emqx_crl_cache:register_der_crls(URL, CRLs),
+                            CRLs
+                    catch
+                        _:_ ->
+                            get_crls(Rest, CRLDbInfo)
+                    end
+            end;
+        {error, _Reason} ->
+            get_crls(Rest, CRLDbInfo)
+    end.
+
+cache_lookup(_, undefined) ->
+    [];
+cache_lookup(URL, {{Cache, _}, _}) ->
+    #{path := Path} = uri_string:normalize(URL, [return_map]),
+    case ssl_pkix_db:lookup(string:trim(Path, leading, "/"), Cache) of
+        undefined ->
+            [];
+        [CRLs] ->
+            CRLs
+    end.
+
+handle_http(URI, Rest, {_, [{http, Timeout}]} = CRLDbInfo) ->
+    CRLs = http_lookup(URI, Rest, CRLDbInfo, Timeout),
+    %% Uncomment to improve performance, but need to
+    %% implement cache limit and or cleaning to prevent
+    %% DoS attack possibilities
+    %%insert(URI, {der, CRLs}),
+    CRLs;
+handle_http(_, Rest, CRLDbInfo) ->
+    get_crls(Rest, CRLDbInfo).
diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index 38f30b8c5..79dc4cd76 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -262,12 +262,13 @@ app_schema(App) ->
     end.
 
 mustache_vars(App, Opts) ->
-    ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, []),
-    [
-        {platform_data_dir, app_path(App, "data")},
-        {platform_etc_dir, app_path(App, "etc")},
-        {platform_log_dir, app_path(App, "log")}
-    ] ++ ExtraMustacheVars.
+    ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, #{}),
+    Defaults = #{
+        platform_data_dir => app_path(App, "data"),
+        platform_etc_dir => app_path(App, "etc"),
+        platform_log_dir => app_path(App, "log")
+    },
+    maps:merge(Defaults, ExtraMustacheVars).
 
 render_config_file(ConfigFile, Vars0) ->
     Temp =
@@ -275,7 +276,7 @@ render_config_file(ConfigFile, Vars0) ->
             {ok, T} -> T;
             {error, Reason} -> error({failed_to_read_config_template, ConfigFile, Reason})
         end,
-    Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- Vars0],
+    Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- maps:to_list(Vars0)],
     Targ = bbmustache:render(Temp, Vars),
     NewName = ConfigFile ++ ".rendered",
     ok = file:write_file(NewName, Targ),
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl
new file mode 100644
index 000000000..7a61f7835
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl
@@ -0,0 +1,1057 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_crl_cache_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+%% from ssl_manager.erl
+-record(state, {
+    session_cache_client,
+    session_cache_client_cb,
+    session_lifetime,
+    certificate_db,
+    session_validation_timer,
+    session_cache_client_max,
+    session_client_invalidator,
+    options,
+    client_session_order
+}).
+
+-define(DEFAULT_URL, "http://localhost:9878/intermediate.crl.pem").
+
+%%--------------------------------------------------------------------
+%% CT boilerplate
+%%--------------------------------------------------------------------
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    application:load(emqx),
+    emqx_config:save_schema_mod_and_names(emqx_schema),
+    emqx_common_test_helpers:boot_modules(all),
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(TestCase, Config) when
+    TestCase =:= t_cache;
+    TestCase =:= t_filled_cache;
+    TestCase =:= t_revoked
+->
+    ct:timetrap({seconds, 30}),
+    DataDir = ?config(data_dir, Config),
+    CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]),
+    {ok, CRLPem} = file:read_file(CRLFile),
+    [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem),
+    ok = snabbkaffe:start_trace(),
+    ServerPid = start_crl_server(CRLPem),
+    IsCached = lists:member(TestCase, [t_filled_cache, t_revoked]),
+    ok = setup_crl_options(Config, #{is_cached => IsCached}),
+    [
+        {crl_pem, CRLPem},
+        {crl_der, CRLDer},
+        {http_server, ServerPid}
+        | Config
+    ];
+init_per_testcase(t_revoke_then_refresh, Config) ->
+    ct:timetrap({seconds, 120}),
+    DataDir = ?config(data_dir, Config),
+    CRLFileNotRevoked = filename:join([DataDir, "intermediate-not-revoked.crl.pem"]),
+    {ok, CRLPemNotRevoked} = file:read_file(CRLFileNotRevoked),
+    [{'CertificateList', CRLDerNotRevoked, not_encrypted}] = public_key:pem_decode(
+        CRLPemNotRevoked
+    ),
+    CRLFileRevoked = filename:join([DataDir, "intermediate-revoked.crl.pem"]),
+    {ok, CRLPemRevoked} = file:read_file(CRLFileRevoked),
+    [{'CertificateList', CRLDerRevoked, not_encrypted}] = public_key:pem_decode(CRLPemRevoked),
+    ok = snabbkaffe:start_trace(),
+    ServerPid = start_crl_server(CRLPemNotRevoked),
+    ExtraVars = #{refresh_interval => <<"10s">>},
+    ok = setup_crl_options(Config, #{is_cached => true, extra_vars => ExtraVars}),
+    [
+        {crl_pem_not_revoked, CRLPemNotRevoked},
+        {crl_der_not_revoked, CRLDerNotRevoked},
+        {crl_pem_revoked, CRLPemRevoked},
+        {crl_der_revoked, CRLDerRevoked},
+        {http_server, ServerPid}
+        | Config
+    ];
+init_per_testcase(t_cache_overflow, Config) ->
+    ct:timetrap({seconds, 120}),
+    DataDir = ?config(data_dir, Config),
+    CRLFileRevoked = filename:join([DataDir, "intermediate-revoked.crl.pem"]),
+    {ok, CRLPemRevoked} = file:read_file(CRLFileRevoked),
+    ok = snabbkaffe:start_trace(),
+    ServerPid = start_crl_server(CRLPemRevoked),
+    ExtraVars = #{cache_capacity => <<"2">>},
+    ok = setup_crl_options(Config, #{is_cached => false, extra_vars => ExtraVars}),
+    [
+        {http_server, ServerPid}
+        | Config
+    ];
+init_per_testcase(t_not_cached_and_unreachable, Config) ->
+    ct:timetrap({seconds, 30}),
+    DataDir = ?config(data_dir, Config),
+    CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]),
+    {ok, CRLPem} = file:read_file(CRLFile),
+    [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem),
+    ok = snabbkaffe:start_trace(),
+    application:stop(cowboy),
+    ok = setup_crl_options(Config, #{is_cached => false}),
+    [
+        {crl_pem, CRLPem},
+        {crl_der, CRLDer}
+        | Config
+    ];
+init_per_testcase(t_refresh_config, Config) ->
+    ct:timetrap({seconds, 30}),
+    DataDir = ?config(data_dir, Config),
+    CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]),
+    {ok, CRLPem} = file:read_file(CRLFile),
+    [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem),
+    TestPid = self(),
+    ok = meck:new(emqx_crl_cache, [non_strict, passthrough, no_history, no_link]),
+    meck:expect(
+        emqx_crl_cache,
+        http_get,
+        fun(URL, _HTTPTimeout) ->
+            ct:pal("http get crl ~p", [URL]),
+            TestPid ! {http_get, URL},
+            {ok, {{"HTTP/1.0", 200, "OK"}, [], CRLPem}}
+        end
+    ),
+    ok = snabbkaffe:start_trace(),
+    ok = setup_crl_options(Config, #{is_cached => false}),
+    [
+        {crl_pem, CRLPem},
+        {crl_der, CRLDer}
+        | Config
+    ];
+init_per_testcase(TestCase, Config) when
+    TestCase =:= t_update_listener;
+    TestCase =:= t_validations
+->
+    %% when running emqx standalone tests, we can't use those
+    %% features.
+    case does_module_exist(emqx_mgmt_api_test_util) of
+        true ->
+            ct:timetrap({seconds, 30}),
+            DataDir = ?config(data_dir, Config),
+            PrivDir = ?config(priv_dir, Config),
+            CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]),
+            {ok, CRLPem} = file:read_file(CRLFile),
+            ok = snabbkaffe:start_trace(),
+            ServerPid = start_crl_server(CRLPem),
+            ConfFilePath = filename:join([DataDir, "emqx_just_verify.conf"]),
+            emqx_mgmt_api_test_util:init_suite(
+                [emqx_conf],
+                fun emqx_mgmt_api_test_util:set_special_configs/1,
+                #{
+                    extra_mustache_vars => #{
+                        test_data_dir => DataDir,
+                        test_priv_dir => PrivDir
+                    },
+                    conf_file_path => ConfFilePath
+                }
+            ),
+            [
+                {http_server, ServerPid}
+                | Config
+            ];
+        false ->
+            [{skip_does_not_apply, true} | Config]
+    end;
+init_per_testcase(_TestCase, Config) ->
+    ct:timetrap({seconds, 30}),
+    DataDir = ?config(data_dir, Config),
+    CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]),
+    {ok, CRLPem} = file:read_file(CRLFile),
+    [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem),
+    TestPid = self(),
+    ok = meck:new(emqx_crl_cache, [non_strict, passthrough, no_history, no_link]),
+    meck:expect(
+        emqx_crl_cache,
+        http_get,
+        fun(URL, _HTTPTimeout) ->
+            ct:pal("http get crl ~p", [URL]),
+            TestPid ! {http_get, URL},
+            {ok, {{"HTTP/1.0", 200, 'OK'}, [], CRLPem}}
+        end
+    ),
+    ok = snabbkaffe:start_trace(),
+    [
+        {crl_pem, CRLPem},
+        {crl_der, CRLDer}
+        | Config
+    ].
+
+end_per_testcase(TestCase, Config) when
+    TestCase =:= t_cache;
+    TestCase =:= t_filled_cache;
+    TestCase =:= t_revoked
+->
+    ServerPid = ?config(http_server, Config),
+    emqx_crl_cache_http_server:stop(ServerPid),
+    emqx_common_test_helpers:stop_apps([]),
+    clear_listeners(),
+    application:stop(cowboy),
+    clear_crl_cache(),
+    ok = snabbkaffe:stop(),
+    ok;
+end_per_testcase(TestCase, Config) when
+    TestCase =:= t_revoke_then_refresh;
+    TestCase =:= t_cache_overflow
+->
+    ServerPid = ?config(http_server, Config),
+    emqx_crl_cache_http_server:stop(ServerPid),
+    emqx_common_test_helpers:stop_apps([]),
+    clear_listeners(),
+    clear_crl_cache(),
+    application:stop(cowboy),
+    ok = snabbkaffe:stop(),
+    ok;
+end_per_testcase(t_not_cached_and_unreachable, _Config) ->
+    emqx_common_test_helpers:stop_apps([]),
+    clear_listeners(),
+    clear_crl_cache(),
+    ok = snabbkaffe:stop(),
+    ok;
+end_per_testcase(t_refresh_config, _Config) ->
+    meck:unload([emqx_crl_cache]),
+    clear_crl_cache(),
+    emqx_common_test_helpers:stop_apps([]),
+    clear_listeners(),
+    clear_crl_cache(),
+    application:stop(cowboy),
+    ok = snabbkaffe:stop(),
+    ok;
+end_per_testcase(TestCase, Config) when
+    TestCase =:= t_update_listener;
+    TestCase =:= t_validations
+->
+    Skip = proplists:get_bool(skip_does_not_apply, Config),
+    case Skip of
+        true ->
+            ok;
+        false ->
+            ServerPid = ?config(http_server, Config),
+            emqx_crl_cache_http_server:stop(ServerPid),
+            emqx_mgmt_api_test_util:end_suite([emqx_conf]),
+            clear_listeners(),
+            ok = snabbkaffe:stop(),
+            clear_crl_cache(),
+            ok
+    end;
+end_per_testcase(_TestCase, _Config) ->
+    meck:unload([emqx_crl_cache]),
+    clear_crl_cache(),
+    ok = snabbkaffe:stop(),
+    ok.
+
+%%--------------------------------------------------------------------
+%% Helper functions
+%%--------------------------------------------------------------------
+
+does_module_exist(Mod) ->
+    case erlang:module_loaded(Mod) of
+        true ->
+            true;
+        false ->
+            case code:ensure_loaded(Mod) of
+                ok ->
+                    true;
+                {module, Mod} ->
+                    true;
+                _ ->
+                    false
+            end
+    end.
+
+clear_listeners() ->
+    emqx_config:put([listeners], #{}),
+    emqx_config:put_raw([listeners], #{}),
+    ok.
+
+assert_http_get(URL) ->
+    receive
+        {http_get, URL} ->
+            ok
+    after 1000 ->
+        ct:pal("mailbox: ~p", [process_info(self(), messages)]),
+        error({should_have_requested, URL})
+    end.
+
+get_crl_cache_table() ->
+    #state{certificate_db = [_, _, _, {Ref, _}]} = sys:get_state(ssl_manager),
+    Ref.
+
+start_crl_server(Port, CRLPem) ->
+    {ok, LSock} = gen_tcp:listen(Port, [binary, {active, true}, reusedaddr]),
+    spawn_link(fun() -> accept_loop(LSock, CRLPem) end),
+    ok.
+
+accept_loop(LSock, CRLPem) ->
+    case gen_tcp:accept(LSock) of
+        {ok, Sock} ->
+            Worker = spawn_link(fun() -> crl_loop(Sock, CRLPem) end),
+            gen_tcp:controlling_process(Sock, Worker),
+            accept_loop(LSock, CRLPem);
+        {error, Reason} ->
+            error({accept_error, Reason})
+    end.
+
+crl_loop(Sock, CRLPem) ->
+    receive
+        {tcp, Sock, _Data} ->
+            gen_tcp:send(Sock, CRLPem),
+            crl_loop(Sock, CRLPem);
+        _Msg ->
+            ok
+    end.
+
+drain_msgs() ->
+    receive
+        _Msg ->
+            drain_msgs()
+    after 0 ->
+        ok
+    end.
+
+clear_crl_cache() ->
+    %% reset the CRL cache
+    exit(whereis(ssl_manager), kill),
+    ok.
+
+force_cacertfile(Cacertfile) ->
+    {SSLListeners0, OtherListeners} = lists:partition(
+        fun(#{proto := Proto}) -> Proto =:= ssl end,
+        emqx:get_env(listeners)
+    ),
+    SSLListeners =
+        lists:map(
+            fun(Listener = #{opts := Opts0}) ->
+                SSLOpts0 = proplists:get_value(ssl_options, Opts0),
+                %% it injects some garbage...
+                SSLOpts1 = lists:keydelete(cacertfile, 1, lists:keydelete(cacertfile, 1, SSLOpts0)),
+                SSLOpts2 = [{cacertfile, Cacertfile} | SSLOpts1],
+                Opts1 = lists:keyreplace(ssl_options, 1, Opts0, {ssl_options, SSLOpts2}),
+                Listener#{opts => Opts1}
+            end,
+            SSLListeners0
+        ),
+    application:set_env(emqx, listeners, SSLListeners ++ OtherListeners),
+    ok.
+
+setup_crl_options(Config, #{is_cached := IsCached} = Opts) ->
+    DataDir = ?config(data_dir, Config),
+    ConfFilePath = filename:join([DataDir, "emqx.conf"]),
+    Defaults = #{
+        refresh_interval => <<"11m">>,
+        cache_capacity => <<"100">>,
+        test_data_dir => DataDir
+    },
+    ExtraVars0 = maps:get(extra_vars, Opts, #{}),
+    ExtraVars = maps:merge(Defaults, ExtraVars0),
+    emqx_common_test_helpers:start_apps(
+        [],
+        fun(_) -> ok end,
+        #{
+            extra_mustache_vars => ExtraVars,
+            conf_file_path => ConfFilePath
+        }
+    ),
+    case IsCached of
+        true ->
+            %% wait the cache to be filled
+            emqx_crl_cache:refresh(?DEFAULT_URL),
+            receive
+                {http_get, <<?DEFAULT_URL>>} -> ok
+            after 1_000 ->
+                ct:pal("mailbox: ~p", [process_info(self(), messages)]),
+                error(crl_cache_not_filled)
+            end;
+        false ->
+            %% ensure cache is empty
+            clear_crl_cache(),
+            ct:sleep(200),
+            ok
+    end,
+    drain_msgs(),
+    ok.
+
+start_crl_server(CRLPem) ->
+    application:ensure_all_started(cowboy),
+    {ok, ServerPid} = emqx_crl_cache_http_server:start_link(self(), 9878, CRLPem, []),
+    receive
+        {ServerPid, ready} -> ok
+    after 1000 -> error(timeout_starting_http_server)
+    end,
+    ServerPid.
+
+request(Method, Url, QueryParams, Body) ->
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    Opts = #{return_all => true},
+    case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of
+        {ok, {Reason, Headers, BodyR}} ->
+            {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}};
+        Error ->
+            Error
+    end.
+
+get_listener_via_api(ListenerId) ->
+    Path = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]),
+    request(get, Path, [], []).
+
+update_listener_via_api(ListenerId, NewConfig) ->
+    Path = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]),
+    request(put, Path, [], NewConfig).
+
+assert_successful_connection(Config) ->
+    assert_successful_connection(Config, default).
+
+assert_successful_connection(Config, ClientNum) ->
+    DataDir = ?config(data_dir, Config),
+    Num =
+        case ClientNum of
+            default -> "";
+            _ -> integer_to_list(ClientNum)
+        end,
+    ClientCert = filename:join(DataDir, "client" ++ Num ++ ".cert.pem"),
+    ClientKey = filename:join(DataDir, "client" ++ Num ++ ".key.pem"),
+    %% 1) At first, the cache is empty, and the CRL is fetched and
+    %% cached on the fly.
+    {ok, C0} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    ?tp_span(
+        mqtt_client_connection,
+        #{client_num => ClientNum},
+        begin
+            {ok, _} = emqtt:connect(C0),
+            emqtt:stop(C0),
+            ok
+        end
+    ).
+
+trace_between(Trace0, Marker1, Marker2) ->
+    {Trace1, [_ | _]} = ?split_trace_at(#{?snk_kind := Marker2}, Trace0),
+    {[_ | _], [_ | Trace2]} = ?split_trace_at(#{?snk_kind := Marker1}, Trace1),
+    Trace2.
+
+of_kinds(Trace0, Kinds0) ->
+    Kinds = sets:from_list(Kinds0, [{version, 2}]),
+    lists:filter(
+        fun(#{?snk_kind := K}) -> sets:is_element(K, Kinds) end,
+        Trace0
+    ).
+
+%%--------------------------------------------------------------------
+%% Test cases
+%%--------------------------------------------------------------------
+
+t_init_empty_urls(_Config) ->
+    Ref = get_crl_cache_table(),
+    ?assertEqual([], ets:tab2list(Ref)),
+    ?assertMatch({ok, _}, emqx_crl_cache:start_link()),
+    receive
+        {http_get, _} ->
+            error(should_not_make_http_request)
+    after 1000 -> ok
+    end,
+    ?assertEqual([], ets:tab2list(Ref)),
+    ok.
+
+t_manual_refresh(Config) ->
+    CRLDer = ?config(crl_der, Config),
+    Ref = get_crl_cache_table(),
+    ?assertEqual([], ets:tab2list(Ref)),
+    {ok, _} = emqx_crl_cache:start_link(),
+    URL = "http://localhost/crl.pem",
+    ok = snabbkaffe:start_trace(),
+    ?wait_async_action(
+        ?assertEqual(ok, emqx_crl_cache:refresh(URL)),
+        #{?snk_kind := crl_cache_insert},
+        5_000
+    ),
+    ok = snabbkaffe:stop(),
+    ?assertEqual(
+        [{"crl.pem", [CRLDer]}],
+        ets:tab2list(Ref)
+    ),
+    ok.
+
+t_refresh_request_error(_Config) ->
+    meck:expect(
+        emqx_crl_cache,
+        http_get,
+        fun(_URL, _HTTPTimeout) ->
+            {ok, {{"HTTP/1.0", 404, 'Not Found'}, [], <<"not found">>}}
+        end
+    ),
+    {ok, _} = emqx_crl_cache:start_link(),
+    URL = "http://localhost/crl.pem",
+    ?check_trace(
+        ?wait_async_action(
+            ?assertEqual(ok, emqx_crl_cache:refresh(URL)),
+            #{?snk_kind := crl_cache_insert},
+            5_000
+        ),
+        fun(Trace) ->
+            ?assertMatch(
+                [#{error := {bad_response, #{code := 404}}}],
+                ?of_kind(crl_refresh_failure, Trace)
+            ),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
+
+t_refresh_invalid_response(_Config) ->
+    meck:expect(
+        emqx_crl_cache,
+        http_get,
+        fun(_URL, _HTTPTimeout) ->
+            {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"not a crl">>}}
+        end
+    ),
+    {ok, _} = emqx_crl_cache:start_link(),
+    URL = "http://localhost/crl.pem",
+    ?check_trace(
+        ?wait_async_action(
+            ?assertEqual(ok, emqx_crl_cache:refresh(URL)),
+            #{?snk_kind := crl_cache_insert},
+            5_000
+        ),
+        fun(Trace) ->
+            ?assertMatch(
+                [#{crls := []}],
+                ?of_kind(crl_cache_insert, Trace)
+            ),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
+
+t_refresh_http_error(_Config) ->
+    meck:expect(
+        emqx_crl_cache,
+        http_get,
+        fun(_URL, _HTTPTimeout) ->
+            {error, timeout}
+        end
+    ),
+    {ok, _} = emqx_crl_cache:start_link(),
+    URL = "http://localhost/crl.pem",
+    ?check_trace(
+        ?wait_async_action(
+            ?assertEqual(ok, emqx_crl_cache:refresh(URL)),
+            #{?snk_kind := crl_cache_insert},
+            5_000
+        ),
+        fun(Trace) ->
+            ?assertMatch(
+                [#{error := {http_error, timeout}}],
+                ?of_kind(crl_refresh_failure, Trace)
+            ),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+    ok.
+
+t_unknown_messages(_Config) ->
+    {ok, Server} = emqx_crl_cache:start_link(),
+    gen_server:call(Server, foo),
+    gen_server:cast(Server, foo),
+    Server ! foo,
+    ok.
+
+t_evict(_Config) ->
+    {ok, _} = emqx_crl_cache:start_link(),
+    URL = "http://localhost/crl.pem",
+    ?wait_async_action(
+        ?assertEqual(ok, emqx_crl_cache:refresh(URL)),
+        #{?snk_kind := crl_cache_insert},
+        5_000
+    ),
+    Ref = get_crl_cache_table(),
+    ?assertMatch([{"crl.pem", _}], ets:tab2list(Ref)),
+    {ok, {ok, _}} = ?wait_async_action(
+        emqx_crl_cache:evict(URL),
+        #{?snk_kind := crl_cache_evict}
+    ),
+    ?assertEqual([], ets:tab2list(Ref)),
+    ok.
+
+t_cache(Config) ->
+    DataDir = ?config(data_dir, Config),
+    ClientCert = filename:join(DataDir, "client.cert.pem"),
+    ClientKey = filename:join(DataDir, "client.key.pem"),
+    %% 1) At first, the cache is empty, and the CRL is fetched and
+    %% cached on the fly.
+    {ok, C0} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    {ok, _} = emqtt:connect(C0),
+    receive
+        {http_get, _} -> ok
+    after 500 ->
+        emqtt:stop(C0),
+        error(should_have_checked_server)
+    end,
+    emqtt:stop(C0),
+    %% 2) When another client using the cached CRL URL connects later,
+    %% it uses the cache.
+    {ok, C1} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    {ok, _} = emqtt:connect(C1),
+    receive
+        {http_get, _} ->
+            emqtt:stop(C1),
+            error(should_not_have_checked_server)
+    after 500 -> ok
+    end,
+    emqtt:stop(C1),
+
+    ok.
+
+t_cache_overflow(Config) ->
+    %% we have capacity = 2 here.
+    ?check_trace(
+        begin
+            %% First and second connections goes into the cache
+            ?tp(first_connections, #{}),
+            assert_successful_connection(Config, 1),
+            assert_successful_connection(Config, 2),
+            %% These should be cached
+            ?tp(first_reconnections, #{}),
+            assert_successful_connection(Config, 1),
+            assert_successful_connection(Config, 2),
+            %% A third client connects and evicts the oldest URL (1)
+            ?tp(first_eviction, #{}),
+            assert_successful_connection(Config, 3),
+            assert_successful_connection(Config, 3),
+            %% URL (1) connects again and needs to be re-cached; this
+            %% time, (2) gets evicted
+            ?tp(second_eviction, #{}),
+            assert_successful_connection(Config, 1),
+            %% TODO: force race condition where the same URL is fetched
+            %% at the same time and tries to be registered
+            ?tp(test_end, #{}),
+            ok
+        end,
+        fun(Trace) ->
+            URL1 = "http://localhost:9878/intermediate1.crl.pem",
+            URL2 = "http://localhost:9878/intermediate2.crl.pem",
+            URL3 = "http://localhost:9878/intermediate3.crl.pem",
+            Kinds = [
+                mqtt_client_connection,
+                new_crl_url_inserted,
+                crl_cache_ensure_timer,
+                crl_cache_overflow
+            ],
+            Trace1 = of_kinds(
+                trace_between(Trace, first_connections, first_reconnections),
+                Kinds
+            ),
+            ?assertMatch(
+                [
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := start,
+                        client_num := 1
+                    },
+                    #{
+                        ?snk_kind := new_crl_url_inserted,
+                        url := URL1
+                    },
+                    #{
+                        ?snk_kind := crl_cache_ensure_timer,
+                        url := URL1
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := {complete, ok},
+                        client_num := 1
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := start,
+                        client_num := 2
+                    },
+                    #{
+                        ?snk_kind := new_crl_url_inserted,
+                        url := URL2
+                    },
+                    #{
+                        ?snk_kind := crl_cache_ensure_timer,
+                        url := URL2
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := {complete, ok},
+                        client_num := 2
+                    }
+                ],
+                Trace1
+            ),
+            Trace2 = of_kinds(
+                trace_between(Trace, first_reconnections, first_eviction),
+                Kinds
+            ),
+            ?assertMatch(
+                [
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := start,
+                        client_num := 1
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := {complete, ok},
+                        client_num := 1
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := start,
+                        client_num := 2
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := {complete, ok},
+                        client_num := 2
+                    }
+                ],
+                Trace2
+            ),
+            Trace3 = of_kinds(
+                trace_between(Trace, first_eviction, second_eviction),
+                Kinds
+            ),
+            ?assertMatch(
+                [
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := start,
+                        client_num := 3
+                    },
+                    #{
+                        ?snk_kind := new_crl_url_inserted,
+                        url := URL3
+                    },
+                    #{
+                        ?snk_kind := crl_cache_overflow,
+                        oldest_url := URL1
+                    },
+                    #{
+                        ?snk_kind := crl_cache_ensure_timer,
+                        url := URL3
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := {complete, ok},
+                        client_num := 3
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := start,
+                        client_num := 3
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := {complete, ok},
+                        client_num := 3
+                    }
+                ],
+                Trace3
+            ),
+            Trace4 = of_kinds(
+                trace_between(Trace, second_eviction, test_end),
+                Kinds
+            ),
+            ?assertMatch(
+                [
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := start,
+                        client_num := 1
+                    },
+                    #{
+                        ?snk_kind := new_crl_url_inserted,
+                        url := URL1
+                    },
+                    #{
+                        ?snk_kind := crl_cache_overflow,
+                        oldest_url := URL2
+                    },
+                    #{
+                        ?snk_kind := crl_cache_ensure_timer,
+                        url := URL1
+                    },
+                    #{
+                        ?snk_kind := mqtt_client_connection,
+                        ?snk_span := {complete, ok},
+                        client_num := 1
+                    }
+                ],
+                Trace4
+            ),
+            ok
+        end
+    ).
+
+%% check that the URL in the certificate is *not* checked if the cache
+%% contains that URL.
+t_filled_cache(Config) ->
+    DataDir = ?config(data_dir, Config),
+    ClientCert = filename:join(DataDir, "client.cert.pem"),
+    ClientKey = filename:join(DataDir, "client.key.pem"),
+    {ok, C} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    {ok, _} = emqtt:connect(C),
+    receive
+        http_get ->
+            emqtt:stop(C),
+            error(should_have_used_cache)
+    after 500 -> ok
+    end,
+    emqtt:stop(C),
+    ok.
+
+%% If the CRL is not cached when the client tries to connect and the
+%% CRL server is unreachable, the client will be denied connection.
+t_not_cached_and_unreachable(Config) ->
+    DataDir = ?config(data_dir, Config),
+    ClientCert = filename:join(DataDir, "client.cert.pem"),
+    ClientKey = filename:join(DataDir, "client.key.pem"),
+    {ok, C} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    Ref = get_crl_cache_table(),
+    ?assertEqual([], ets:tab2list(Ref)),
+    process_flag(trap_exit, true),
+    ?assertMatch({error, {{shutdown, {tls_alert, {bad_certificate, _}}}, _}}, emqtt:connect(C)),
+    ok.
+
+t_revoked(Config) ->
+    DataDir = ?config(data_dir, Config),
+    ClientCert = filename:join(DataDir, "client-revoked.cert.pem"),
+    ClientKey = filename:join(DataDir, "client-revoked.key.pem"),
+    {ok, C} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    process_flag(trap_exit, true),
+    ?assertMatch({error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}}, emqtt:connect(C)),
+    ok.
+
+t_revoke_then_refresh(Config) ->
+    DataDir = ?config(data_dir, Config),
+    CRLPemRevoked = ?config(crl_pem_revoked, Config),
+    ClientCert = filename:join(DataDir, "client-revoked.cert.pem"),
+    ClientKey = filename:join(DataDir, "client-revoked.key.pem"),
+    {ok, C0} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    %% At first, the CRL contains no revoked entries, so the client
+    %% should be allowed connection.
+    ?assertMatch({ok, _}, emqtt:connect(C0)),
+    emqtt:stop(C0),
+
+    %% Now we update the CRL on the server and wait for the cache to
+    %% be refreshed.
+    {true, {ok, _}} =
+        ?wait_async_action(
+            emqx_crl_cache_http_server:set_crl(CRLPemRevoked),
+            #{?snk_kind := crl_refresh_timer_done},
+            70_000
+        ),
+
+    %% The *same client* should now be denied connection.
+    {ok, C1} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    process_flag(trap_exit, true),
+    ?assertMatch(
+        {error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}}, emqtt:connect(C1)
+    ),
+    ok.
+
+%% check that we can start with a non-crl listener and restart it with
+%% the new crl config.
+t_update_listener(Config) ->
+    case proplists:get_bool(skip_does_not_apply, Config) of
+        true ->
+            ok;
+        false ->
+            do_t_update_listener(Config)
+    end.
+
+do_t_update_listener(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Keyfile = filename:join([DataDir, "server.key.pem"]),
+    Certfile = filename:join([DataDir, "server.cert.pem"]),
+    Cacertfile = filename:join([DataDir, "ca-chain.cert.pem"]),
+    ClientCert = filename:join(DataDir, "client-revoked.cert.pem"),
+    ClientKey = filename:join(DataDir, "client-revoked.key.pem"),
+
+    %% no crl at first
+    ListenerId = "ssl:default",
+    {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId),
+    ?assertMatch(
+        #{
+            <<"ssl_options">> :=
+                #{
+                    <<"enable_crl_check">> := false,
+                    <<"verify">> := <<"verify_peer">>
+                }
+        },
+        ListenerData0
+    ),
+    {ok, C0} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    %% At first, the CRL contains no revoked entries, so the client
+    %% should be allowed connection.
+    ?assertMatch({ok, _}, emqtt:connect(C0)),
+    emqtt:stop(C0),
+
+    %% configure crl
+    CRLConfig =
+        #{
+            <<"ssl_options">> =>
+                #{
+                    <<"keyfile">> => Keyfile,
+                    <<"certfile">> => Certfile,
+                    <<"cacertfile">> => Cacertfile,
+                    <<"enable_crl_check">> => true
+                }
+        },
+    ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, CRLConfig),
+    {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1),
+    ?assertMatch(
+        #{
+            <<"ssl_options">> :=
+                #{
+                    <<"enable_crl_check">> := true,
+                    <<"verify">> := <<"verify_peer">>
+                }
+        },
+        ListenerData2
+    ),
+
+    %% Now should use CRL information to block connection
+    process_flag(trap_exit, true),
+    {ok, C1} = emqtt:start_link([
+        {ssl, true},
+        {ssl_opts, [
+            {certfile, ClientCert},
+            {keyfile, ClientKey}
+        ]},
+        {port, 8883}
+    ]),
+    ?assertMatch(
+        {error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}}, emqtt:connect(C1)
+    ),
+    assert_http_get(<<?DEFAULT_URL>>),
+
+    ok.
+
+t_validations(Config) ->
+    case proplists:get_bool(skip_does_not_apply, Config) of
+        true ->
+            ok;
+        false ->
+            do_t_validations(Config)
+    end.
+
+do_t_validations(_Config) ->
+    ListenerId = <<"ssl:default">>,
+    {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId),
+
+    ListenerData1 =
+        emqx_map_lib:deep_merge(
+            ListenerData0,
+            #{
+                <<"ssl_options">> =>
+                    #{
+                        <<"enable_crl_check">> => true,
+                        <<"verify">> => <<"verify_none">>
+                    }
+            }
+        ),
+    {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1),
+    #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} =
+        emqx_json:decode(ResRaw1, [return_maps]),
+    ?assertMatch(
+        #{
+            <<"mismatches">> :=
+                #{
+                    <<"listeners:ssl_not_required_bind">> :=
+                        #{
+                            <<"reason">> :=
+                                <<"verify must be verify_peer when CRL check is enabled">>
+                        }
+                }
+        },
+        emqx_json:decode(MsgRaw1, [return_maps])
+    ),
+
+    ok.
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/ca-chain.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/ca-chain.cert.pem
new file mode 100644
index 000000000..eaabd2445
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/ca-chain.cert.pem
@@ -0,0 +1,68 @@
+-----BEGIN CERTIFICATE-----
+MIIF+zCCA+OgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwbzELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQK
+DAlNeU9yZ05hbWUxETAPBgNVBAsMCE15Um9vdENBMREwDwYDVQQDDAhNeVJvb3RD
+QTAeFw0yMzAxMTIxMzA4MTZaFw0zMzAxMDkxMzA4MTZaMGsxCzAJBgNVBAYTAlNF
+MRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAoMCU15T3JnTmFtZTEZMBcGA1UE
+CwwQTXlJbnRlcm1lZGlhdGVDQTEZMBcGA1UEAwwQTXlJbnRlcm1lZGlhdGVDQTCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALQG7dMeU/y9HDNHzhydR0bm
+wN9UGplqJOJPwqJRaZZcrn9umgJ9SU2il2ceEVxMDwzBWCRKJO5/H9A9k13SqsXM
+2c2c9xXfIF1kb820lCm1Uow5hZ/auDjxliNk9kNJDigCRi3QoIs/dVeWzFsgEC2l
+gxRqauN2eNFb6/yXY788YALHBsCRV2NFOFXxtPsvLXpD9Q/8EqYsSMuLARRdHVNU
+ryaEF5lhShpcuz0TlIuTy2TiuXJUtJ+p7a4Z7friZ6JsrmQWsVQBj44F8TJRHWzW
+C7vm9c+dzEX9eqbr5iPL+L4ctMW9Lz6ePcYfIXne6CElusRUf8G+xM1uwovF9bpV
++9IqY7tAu9G1iY9iNtJgNNDKOCcOGKcZCx6Cg1XYOEKReNnUMazvYeqRrrjV5WQ0
+vOcD5zcBRNTXCddCLa7U0guXP9mQrfuk4NTH1Bt77JieTJ8cfDXHwtaKf6aGbmZP
+wl1Xi/GuXNUP/xeog78RKyFwBmjt2JKwvWzMpfmH4mEkG9moh2alva+aEz6LIJuP
+16g6s0Q6c793/OvUtpNcewHw4Vjn39LD9o6VLp854G4n8dVpUWSbWS+sXD1ZE69H
+g/sMNMyq+09ufkbewY8xoCm/rQ1pqDZAVMWsstJEaYu7b/eb7R+RGOj1YECCV/Yp
+EZPdDotbSNRkIi2d/a1NAgMBAAGjgaQwgaEwHQYDVR0OBBYEFExwhjsVUom6tQ+S
+qq6xMUETvnPzMB8GA1UdIwQYMBaAFD90kfU5pc5l48THu0Ayj9SNpHuhMBIGA1Ud
+EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMDsGA1UdHwQ0MDIwMKAuoCyG
+Kmh0dHA6Ly9sb2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUuY3JsLnBlbTANBgkq
+hkiG9w0BAQsFAAOCAgEAK6NgdWQYtPNKQNBGjsgtgqTRh+k30iqSO6Y3yE1KGABO
+EuQdVqkC2qUIbCB0M0qoV0ab50KNLfU6cbshggW4LDpcMpoQpI05fukNh1jm3ZuZ
+0xsB7vlmlsv00tpqmfIl/zykPDynHKOmFh/hJP/KetMy4+wDv4/+xP31UdEj5XvG
+HvMtuqOS23A+H6WPU7ol7KzKBnU2zz/xekvPbUD3JqV+ynP5bgbIZHAndd0o9T8e
+NFX23Us4cTenU2/ZlOq694bRzGaK+n3Ksz995Nbtzv5fbUgqmf7Mcq4iHGRVtV11
+MRyBrsXZp2vbF63c4hrf2Zd6SWRoaDKRhP2DMhajpH9zZASSTlfejg/ZRO2s+Clh
+YrSTkeMAdnRt6i/q4QRcOTCfsX75RFM5v67njvTXsSaSTnAwaPi78tRtf+WSh0EP
+VVPzy++BszBVlJ1VAf7soWZHCjZxZ8ZPqVTy5okoHwWQ09WmYe8GfulDh1oj0wbK
+3FjN7bODWHJN+bFf5aQfK+tumYKoPG8RXL6QxpEzjFWjxhIMJHHMKfDWnAV1o1+7
+/1/aDzq7MzEYBbrgQR7oE5ZHtyqhCf9LUgw0Kr7/8QWuNAdeDCJzjXRROU0hJczp
+dOyfRlLbHmLLmGOnROlx6LsGNQ17zuz6SPi7ei8/ylhykawDOAGkM1+xFakmQhM=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFzzCCA7egAwIBAgIUYjc7hD7/UJ0/VPADfNfp/WpOwRowDQYJKoZIhvcNAQEL
+BQAwbzELMAkGA1UEBhMCU0UxEjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJ
+U3RvY2tob2xtMRIwEAYDVQQKDAlNeU9yZ05hbWUxETAPBgNVBAsMCE15Um9vdENB
+MREwDwYDVQQDDAhNeVJvb3RDQTAeFw0yMzAxMTIxMzA4MTRaFw00MzAxMDcxMzA4
+MTRaMG8xCzAJBgNVBAYTAlNFMRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAcM
+CVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMREwDwYDVQQLDAhNeVJvb3RD
+QTERMA8GA1UEAwwITXlSb290Q0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK
+AoICAQCnBwSOYVJw47IoMHMXTVDtOYvUt3rqsurEhFcB4O8xmf2mmwr6m7s8A5Ft
+AvAehg1GvnXT3t/KiyU7BK+acTwcErGyZwS2wvdB0lpHWSpOn/u5y+4ZETvQefcj
+ZTdDOM9VN5nutpitgNb+1yL8sqSexfVbY7DnYYvFjOVBYoP/SGvM9jVjCad+0WL3
+FhuD+L8QAxzCieX3n9UMymlFwINQuEc+TDjuNcEqt+0J5EgS1fwzxb2RCVL0TNv4
+9a71hFGCNRj20AeZm99hbdufm7+0AFO7ocV5q43rLrWFUoBzqKPYIjga/cv/UdWZ
+c5RLRXw3JDSrCqkf/mOlaEhNPlmWRF9MSus5Da3wuwgGCaVzmrf30rWR5aHHcscG
+e+AOgJ4HayvBUQeb6ZlRXc0YlACiLToMKxuyxDyUcDfVEXpUIsDILF8dkiVQxEU3
+j9g6qjXiqPVdNiwpqXfBKObj8vNCzORnoHYs8cCgib3RgDVWeqkDmlSwlZE7CvQh
+U4Loj4l7813xxzYEKkVaT1JdXPWu42CG/b4Y/+f4V+3rkJkYzUwndX6kZNksIBai
+phmtvKt+CTdP1eAbT+C9AWWF3PT31+BIhuT0u9tR8BVSkXdQB8dG4M/AAJcTo640
+0mdYYOXT153gEKHJuUBm750ZTy+r6NjNvpw8VrMAakJwHqnIdQIDAQABo2MwYTAd
+BgNVHQ4EFgQUP3SR9TmlzmXjxMe7QDKP1I2ke6EwHwYDVR0jBBgwFoAUP3SR9Tml
+zmXjxMe7QDKP1I2ke6EwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYw
+DQYJKoZIhvcNAQELBQADggIBAFMFv4C+I0+xOAb9v6G/IOpfPBZ1ez31EXKJJBra
+lulP4nRHQMeb310JS8BIeQ3dl+7+PkSxPABZSwc3jkxdSMvhc+Z4MQtTgos+Qsjs
+gH7sTqwWeeQ0lHYxWmkXijrh5OPRZwTKzYQlkcn85BCUXl2KDuNEdiqPbDTao+lc
+lA0/UAvC6NCyFKq/jqf4CmW5Kx6yG1v1LaE+IXn7cbIXj+DaehocVXi0wsXqj03Q
+DDUHuLHZP+LBsg4e91/0Jy2ekNRTYJifSqr+9ufHl0ZX1pFDZyf396IgZ5CQZ0PJ
+nRxZHlCfsxWxmxxdy3FQSE6YwXhdTjjoAa1ApZcKkkt1beJa6/oRLze/ux5x+5q+
+4QczufHd6rjoKBi6BM3FgFQ8As5iNohHXlMHd/xITo1Go3CWw2j9TGH5vzksOElK
+B0mcwwt2zwNEjvfytc+tI5jcfGN3tiT5fVHS8hw9dWKevypLL+55Ua9G8ZgDHasT
+XFRJHgmnbyFcaAe26D2dSKmhC9u2mHBH+MaI8dj3e7wNBfpxNgp41aFIk+QTmiFW
+VXFED6DHQ/Mxq93ACalHdYg18PlIYClbT6Pf2xXBnn33YPhn5xzoTZ+cDH/RpaQp
+s0UUTSJT1UTXgtXPnZWQfvKlMjJEIiVFiLEC0sgZRlWuZDRAY0CdZJJxvQp59lqu
+cbTm
+-----END CERTIFICATE-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.cert.pem
new file mode 100644
index 000000000..038eec790
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFdTCCA12gAwIBAgICEAUwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDExODEyMzY1NloXDTMzMDQyNTEyMzY1NlowgYQxCzAJBgNVBAYTAlNFMRIw
+EAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAcMCVN0b2NraG9sbTESMBAGA1UECgwJ
+TXlPcmdOYW1lMRkwFwYDVQQLDBBNeUludGVybWVkaWF0ZUNBMR4wHAYDVQQDDBVj
+bGllbnQtbm8tZGlzdC1wb2ludHMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQCYQqNF7o20tEwyXphDgtwkZ628baYzQoCmmaufR+5SPQWdTN+GFeApv0dP
+4y/ncZV24rgButMo73e4+wPsILwSGhaVIU0mMaCmexyC4W6INBkQsVB5FAd/YM0O
+gdxS6A42h9HZTaAJ+4ftgFdOOHiP3lwicXeIYykAE7Y5ikxlnHgi8p1PTLowN4Q+
+AjuXChRzmU16cUEAevZKkTVf7VCcK66aJsxBsxfykkGHhc6qLqmlMt6Te6DPCi/R
+KP/kARTDWNEkp6qtpvzByYFYAKPSZxPuryajAC3RLuGNkVSB+PZ6NnZW6ASeTdra
+Lwuiwsi5XPBeFb0147naQOBzSGG/AgMBAAGjggEHMIIBAzAJBgNVHRMEAjAAMBEG
+CWCGSAGG+EIBAQQEAwIFoDBBBglghkgBhvhCAQ0ENBYyT3BlblNTTCBHZW5lcmF0
+ZWQgQ2xpZW50IENlcnRpZmljYXRlIChubyBDUkwgaW5mbykwHQYDVR0OBBYEFBiV
+sjDe46MixvftT/wej1mxGuN7MB8GA1UdIwQYMBaAFExwhjsVUom6tQ+Sqq6xMUET
+vnPzMA4GA1UdDwEB/wQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUH
+AwQwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0
+Ojk4NzcwDQYJKoZIhvcNAQELBQADggIBAKBEnKYVLFtZb3MI0oMJkrWBssVCq5ja
+OYomZ61I13QLEeyPevTSWAcWFQ4zQDF/SWBsXjsrC+JIEjx2xac6XCpxcx3jDUgo
+46u/hx2rT8tMKa60hW0V1Dk6w8ZHiCe94BlFLsWFKnn6dVzoJd2u3vgUaleh3uxF
+hug8XY+wmHd36rO0kVe3DrsqdIdOfhMiJLDxU0cBA79vI5kCvqB8DIwCWtOzkA82
+EPl3Iws5NPhuFAR9u0xOQu0akzmSJFcEGLZ4qfatHD/tZGRduyFvMKy5iIeMzuEs
+2etm01tfLHqgKGOKp5LjPm7Aoac/GeVoTvctGF+wayvOuYE7inlGZToz3kQMMzHZ
+ZGBBgOhXbR2y74QoFv6DUqmmTRbGfiLYyErA5r881ntgciQi02xrGjoAFntvKb+H
+HNB22Qprz16OmdC9dJKF2RhO6Cketdhv65wFWw6xlhRMCWYPY3CI8tWkxS4A4yit
+RZQZg3yaeHXMaCAu5HxuqAQXKGjz+7w7N6diwbT7o7CfKk8iHUrGfkQ5nCS0GZ1r
+lU1vgKtdzVvJ6HmBrCRcdNqh/L/wdIltwI/52j+TKRtELM1qHuLAYmhcRBW+2wuH
+ewaNA9KEgEk6JC+iR8uOBi0ZLkMIm47j+ZLJRJVUfgkVEEFjyiYSFfpwwcgT+/Aw
+EczVZOdUEbDM
+-----END CERTIFICATE-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.key.pem
new file mode 100644
index 000000000..02b865f5e
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCYQqNF7o20tEwy
+XphDgtwkZ628baYzQoCmmaufR+5SPQWdTN+GFeApv0dP4y/ncZV24rgButMo73e4
++wPsILwSGhaVIU0mMaCmexyC4W6INBkQsVB5FAd/YM0OgdxS6A42h9HZTaAJ+4ft
+gFdOOHiP3lwicXeIYykAE7Y5ikxlnHgi8p1PTLowN4Q+AjuXChRzmU16cUEAevZK
+kTVf7VCcK66aJsxBsxfykkGHhc6qLqmlMt6Te6DPCi/RKP/kARTDWNEkp6qtpvzB
+yYFYAKPSZxPuryajAC3RLuGNkVSB+PZ6NnZW6ASeTdraLwuiwsi5XPBeFb0147na
+QOBzSGG/AgMBAAECggEACSMuozq+vFJ5pCgzIRIQXgruzTkTWU4rZFQijYuGjN7m
+oFsFqwlTC45UHEI5FL2nR5wxiMEKfRFp8Or3gEsyni98nXSDKcCesH8A5gXbWUcv
+HeZWOv3tuUI47B709vDAMZuTB2R2L0MuFB24n5QaACBLDTIcB05UHpIQRIG9NffH
+MhxqFB2kuakp67VekYGZkBCNkqfL3VQZIGRpQC8SvpnRXELqZgI4MyJgvkK6myWj
+Vtpwm8YiOQoJHJx4raoVfS2NWTsCwL0M0aXMMtmM2QfMP/xB9OifxnmDDBs7Tie8
+0Wri845xLTCYthaU8B06rhoQdKXoqKmQMoF2doPm8QKBgQDN+0E0PtPkyxIho8pV
+CsQnmif91EQQqWxOdkHbE96lT0UKu6ziBSbB4ClRHYil5c8p7INxRpj7pruOY3Kw
+MAcacIMMBNhLBJL4R0hr/pwr18WOZxCIMcLHTaCfbVqL71TKp4/6C+GexZfaYJ46
+IZEpLU5RPmD4f9MPIDDm6KcPxwKBgQC9O9TOor93g+A4sU54CGOqvVDrdi5TnGF8
+YdimvUsT20gl2WGX5vq3OohzZi7U8FuxKHWpbgh2efqGLcFsRNFZ/T0ZXX4DDafN
+Gzyu/DMVuFO4ccgFJNnl45w3/yFG40kL6yS8kss/iEYu550/uOZ1FjH+kJ0vjV6G
+JD8q0PgOSQKBgG2i9cLcSia2nBEBwFlhoKS/ndeyWwRPWZGtykHUoqZ0ufgLiurG
++SkqqnM9eBVta8YR2Ki7fgQ8bApPDqWO+sjs6CPGlGXhqmSydG7fF7sSX1n7q8YC
+Tn2M6RjSuOZQ3l37sFvUZSQAYmJfGPkyErTLI6uEu1KpnuqnJMBTR1DTAoGAIGQn
+bx9oirqmHM4s0lsNRGKXgVZ/Y4x3G2VcQl5QhZuZY/ErxWaiL87zIF2zUnu6Fj8I
+tPHCvRTwDxux6ih1dWPlm3vnX/psaK1q28ELtYIRwpanWEoQiktFqEghmBK7pDCh
+3y15YOygptK6lfe+avhboml6nnMiZO+7aEbQzxECgYALuUM4fo1dQYmYuZIqZoFJ
+TXGyzMkNGs61SMiD6mW6XgXj5h5T8Q0MdpmHkwsm+z9A/1of5cxkE6d8HCCz+dt5
+tnY7OC0gYB1+gDld8MZgFgP6k0qklreLVhzEz11TbMldifa1EE4VjUDG/NeAEtbq
+GbLaw0NhGJtRCgL9Bc7i7g==
+-----END PRIVATE KEY-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.cert.pem
new file mode 100644
index 000000000..d0a23bf2f
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFnDCCA4SgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNlowfTELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExFzAVBgNVBAMMDmNs
+aWVudC1yZXZva2VkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs+R6
+PDtIxVlUoLYbDBbaVcxgoLjnWcvqL8wSqyWuqi/Y3cjuNYCziR9nR5dWajtkBjzJ
+HyhgAr6gBVSRt4RRmDXoOcprK3GcpowAr65UAmC4hdH0af6FdKjKCnFw67byUg52
+f7ueXZ6t/XuuKxlU/f2rjXVwmmnlhBi5EHDkXxvfgWXJekDfsPbW9j0kaCUWCpfj
+rzGbfkXqrPkslO41PYlCbPxoiRItJjindFjcQySYvRq7A2uYMGsrxv4n3rzo5NGt
+goBmnGj61ii9WOdopcFxKirhIB9zrxC4x0opRfIaF/n1ZXk6NOnaDxu1LTZ18wfC
+ZB979ge6pleeKoPf7QIDAQABo4IBNjCCATIwCQYDVR0TBAIwADARBglghkgBhvhC
+AQEEBAMCBaAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIENsaWVu
+dCBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUQeItXr3nc6CZ++G9UCoq1YlQ9oowHwYD
+VR0jBBgwFoAUTHCGOxVSibq1D5KqrrExQRO+c/MwDgYDVR0PAQH/BAQDAgXgMB0G
+A1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDBDA7BgNVHR8ENDAyMDCgLqAshipo
+dHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW0wMQYIKwYB
+BQUHAQEEJTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJ
+KoZIhvcNAQELBQADggIBAIFuhokODd54/1B2JiNyG6FMq/2z8B+UquC2iw3p2pyM
+g/Jz4Ouvg6gGwUwmykEua06FRCxx5vJ5ahdhXvKst/zH/0qmYTFNMhNsDy76J/Ot
+Ss+VwQ8ddpEG3EIUI9BQxB3xL7z7kRQzploQjakNcDWtDt1BmN05Iy2vz4lnYJky
+Kss6ya9jEkNibHekhxJuchJ0fVGlVe74MO7RNDFG7+O3tMlxu0zH/LpW093V7BI2
+snXNAwQBizvWTrDKWLDu5JsX8KKkrmDtFTs9gegnxDCOYdtG5GbbMq+H1SjWUJPV
+wiXTF8/eE02s4Jzm7ZAxre4bRt/hAg7xTGmDQ1Hn+LzLn18I9LaW5ZWqSwwpgv+g
+Z/jiLO9DJ/y525Cl7DLCpSFoDTWlQXouKhcgALcVay/cXCsZ3oFZCustburLiJi/
+zgBeEk1gVpwljriJLeZifyfWtJx6yfgB/h6fid8XLsGRD+Yc8Tzs8J1LIgi+j4ZT
+UzKX3B85Kht/dr43UDMtWOF3edkOMaJu7rcg5tTsK+LIyHtXvebKPVvvA9f27Dz/
+4gmhAwwqS87Xv3FMVhZ03DNOJ6XAF+T6OTEqwYs+iK56IMSl1Jy+bCzo0j5jZVbl
+XFwGxUHzM7pfM6PDx657oUxG1QwM/fIWA18F+kY/yigXxq6pYMeAiQsPanOThgHp
+-----END CERTIFICATE-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.key.pem
new file mode 100644
index 000000000..0b7698da9
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz5Ho8O0jFWVSg
+thsMFtpVzGCguOdZy+ovzBKrJa6qL9jdyO41gLOJH2dHl1ZqO2QGPMkfKGACvqAF
+VJG3hFGYNeg5ymsrcZymjACvrlQCYLiF0fRp/oV0qMoKcXDrtvJSDnZ/u55dnq39
+e64rGVT9/auNdXCaaeWEGLkQcORfG9+BZcl6QN+w9tb2PSRoJRYKl+OvMZt+Reqs
++SyU7jU9iUJs/GiJEi0mOKd0WNxDJJi9GrsDa5gwayvG/ifevOjk0a2CgGacaPrW
+KL1Y52ilwXEqKuEgH3OvELjHSilF8hoX+fVleTo06doPG7UtNnXzB8JkH3v2B7qm
+V54qg9/tAgMBAAECggEAAml+HRgjZ+gEezot3yngSBW7NvR7v6e9DmKDXpGdB7Go
+DANBdGyzG5PU9/AGy9pbgzzl6nnJXcgOD7w8TvRifrK8WCgHa1f05IPMj458GGMR
+HlQ8HX647eFEgkLWo4Z6tdB1VM2geDtkNFmn8nJ+wgAYgIdSWPOyDOUi+B43ZbIN
+eaLWkP2fiX9tcJp41cytW+ng2YIm4s90Nt4FJPNBNzOrhVm35jciId02MmEjCEnr
+0YbK9uoMDC2YLg8vhRcjtsUHV2rREkwEAQj8nCWvWWheIwk943d6OicGAD/yebpV
+PTjtlZlpIbrovfvuMcoTxJg3WS8LTg/+cNWAX5a3eQKBgQDcRY7nVSJusYyN0Bij
+YWc9H47wU+YucaGT25xKe26w1pl6s4fmr1Sc3NcaN2iyUv4BuAvaQzymHe4g9deU
+D9Ws/NCQ9EjHJJsklNyn2KCgkSp7oPKhPwyl64XfPdV2gr5AD6MILf7Rkyib5sSf
+1WK8i25KatT7M4mCtrBVJYHNpQKBgQDREjwPIaQBPXouVpnHhSwRHfKD0B1a2koq
+4VE6Fnf3ogkiGfV9kqXwIfPHL0tfotFraM3FFmld8RcxhKUPr4oj+K9KTxmMD9lm
+9Hal0ANXYmHs5a1iHyoNmTpBGHALWLT9fCoeg+EIYabi2+P1c7cDIdUPkEzo4GmI
+nCIpv7hGqQKBgEFUC+8GK+EinWoN1tDV+ZWCP5V9fJ43q1E7592bQBgIfZqLlnnP
+dEvVn6Ix3sZMoPMHj9Ra7qjh5Zc28ooCLEBS9tSW7uLJM44k7FCHihQ1GaFy+aLj
+HTA0aw7rutycKCq9uH+bjKDBgWDDj3tMAS2kOMCvcJ1UCquO3TtTlWzVAoGBAIDN
+8yJ/X0NEVNnnkKZTbWq+QILk3LD0e20fk6Nt5Es0ENxpkczjZEglIsM8Z/trnAnI
+b71UqWWu+tMPHYIka77tn1DwmpSnzxCW2+Ib3XMgsaP5fHBPMuFd3X3tSFo1NIxW
+yrwyE5nOT7rELhUyTTYoydLk2/09BMedKY7/BtDBAoGAXeX1pX74K1i/uWyYKwYZ
+sskRueSo9whDJuZWgNiUovArr57eA+oA+bKdFpiE419348bkFF8jNoGFQ6MXMedD
+LqHAYIj+ZPIC4+rObHqO5EaIyblgutwx3citkQp7HXDBxojnOKA9mKQXj1vxCaL1
+/1fFNJQCzEqwnKwnhI2MJ28=
+-----END PRIVATE KEY-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client.cert.pem
new file mode 100644
index 000000000..b37d1b0ba
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client.cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFljCCA36gAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNlowdzELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExETAPBgNVBAMMCE15
+Q2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvGuAShewEo8V
+/+aWVO/MuUt92m8K0Ut4nC2gOvpjMjf8mhSSf6KfnxPklsFwP4fdyPOjOiXwCsf3
+1QO5fjVr8to3iGTHhEyZpzRcRqmw1eYJC7iDh3BqtYLAT30R+Kq6Mk+f4tXB5Lp/
+2jXgdi0wshWagCPgJO3CtiwGyE8XSa+Q6EBYwzgh3NFbgYdJma4x+S86Y/5WfmXP
+zF//UipsFp4gFUqwGuj6kJrN9NnA1xCiuOxCyN4JuFNMfM/tkeh26jAp0OHhJGsT
+s3YiUm9Dpt7Rs7o0so9ov9K+hgDFuQw9HZW3WIJI99M5a9QZ4ZEQqKpABtYBl/Nb
+VPXcr+T3fQIDAQABo4IBNjCCATIwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMC
+BaAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIENsaWVudCBDZXJ0
+aWZpY2F0ZTAdBgNVHQ4EFgQUOIChBA5aZB0dPWEtALfMIfSopIIwHwYDVR0jBBgw
+FoAUTHCGOxVSibq1D5KqrrExQRO+c/MwDgYDVR0PAQH/BAQDAgXgMB0GA1UdJQQW
+MBQGCCsGAQUFBwMCBggrBgEFBQcDBDA7BgNVHR8ENDAyMDCgLqAshipodHRwOi8v
+bG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW0wMQYIKwYBBQUHAQEE
+JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN
+AQELBQADggIBAE0qTL5WIWcxRPU9oTrzJ+oxMTp1JZ7oQdS+ZekLkQ8mP7T6C/Ew
+6YftjvkopnHUvn842+PTRXSoEtlFiTccmA60eMAai2tn5asxWBsLIRC9FH3LzOgV
+/jgyY7HXuh8XyDBCDD+Sj9QityO+accTHijYAbHPAVBwmZU8nO5D/HsxLjRrCfQf
+qf4OQpX3l1ryOi19lqoRXRGwcoZ95dqq3YgTMlLiEqmerQZSR6iSPELw3bcwnAV1
+hoYYzeKps3xhwszCTz2+WaSsUO2sQlcFEsZ9oHex/02UiM4a8W6hGFJl5eojErxH
+7MqaSyhwwyX6yt8c75RlNcUThv+4+TLkUTbTnWgC9sFjYfd5KSfAdIMp3jYzw3zw
+XEMTX5FaLaOCAfUDttPzn+oNezWZ2UyFTQXQE2CazpRdJoDd04qVg9WLpQxLYRP7
+xSFEHulOPccdAYF2C45yNtJAZyWKfGaAZIxrgEXbMkcdDMlYphpRwpjS8SIBNZ31
+KFE8BczKrg2qO0ywIjanPaRgrFVmeSvBKeU/YLQVx6fZMgOk6vtidLGZLyDXy0Ff
+yaZSoj+on++RDz1IXb96Y8scuNlfcYI8QeoNjwiLtf80BV8SRJiG4e/jTvMf/z9L
+kWrnDWvx4xkUmxFg4TK42dkNp7sEYBTlVVq9fjKE92ha7FGZRqsxOLNQ
+-----END CERTIFICATE-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client.key.pem
new file mode 100644
index 000000000..2e767d81f
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client.key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC8a4BKF7ASjxX/
+5pZU78y5S33abwrRS3icLaA6+mMyN/yaFJJ/op+fE+SWwXA/h93I86M6JfAKx/fV
+A7l+NWvy2jeIZMeETJmnNFxGqbDV5gkLuIOHcGq1gsBPfRH4qroyT5/i1cHkun/a
+NeB2LTCyFZqAI+Ak7cK2LAbITxdJr5DoQFjDOCHc0VuBh0mZrjH5Lzpj/lZ+Zc/M
+X/9SKmwWniAVSrAa6PqQms302cDXEKK47ELI3gm4U0x8z+2R6HbqMCnQ4eEkaxOz
+diJSb0Om3tGzujSyj2i/0r6GAMW5DD0dlbdYgkj30zlr1BnhkRCoqkAG1gGX81tU
+9dyv5Pd9AgMBAAECggEAAifx6dZKIeNkQ8OaNp5V2IKIPSqBOV4/h/xKMkUZXisV
+eDmTCf8du0PR7hfLqrt9xYsGDv+6FQ1/8K231l8qR0tP/6CTl/0ynM4qqEAGeFXN
+3h2LvM4liFbdjImechrcwcnVaNKg/DogT5zHUYSMtB/rokaG0VBO3IX/+SGz0aXi
+LOLAx6SPaLOVX9GYUCiigTSEDwaQA+F3F6J2fR4u8PrXo+OQUqxjQ/fGXWp+4IfA
+6djlpvzO2849/WPB1tL20iLXJlL2OL0UgQNtbKWTjexMe+wgCR5BzCwTyPsQvMwX
+YOQrTOwgF3b6O+gLks5wSRT0ivq1sKgzA534+X4M+wKBgQDirPTLlrYobOO8KUpV
+LOJU8x9leiRNU9CZWrW/mOw/BXGXikqNWvgL595vvADsjYciuRxSqEE7lClB8Pp9
+20TMlES9orx7gdoQJCodpNV1BuBJhE9YtUiXzWAj+7m3D9LsXM1ewW/2A7Vvopj3
+4zKY7uHAFlo3nXwLOfChG5/i9wKBgQDUy5fPFa58xmn7Elb6x4vmUDHg6P4pf75E
+XHRQvNA8I7DTrpqfcsF1N4WuJ3Lm//RSpw7bnyqP20GoEfGHu/iCUPf29B7CuXhO
+vvD+I8uPdn8EcKUBWV+V0xNQN/gCe0TzrEjAkZcO2Lq0j93R8HVl3BbowxgRvQV9
+GmxQG/boKwKBgFeV8uSzsGEAaiKrZbBxrmaappgEUQCcES8gULfes/JJ/TFL2zCx
+ZMTc7CMKZuUAbqXpFtuNbd9CiYqUPYXh8ryF0eXgeqnSa9ruzmMz7NLSPFnLyQkC
+yzD0x2BABOuKLrrrxOMHJWbO2g1vq2GlJUjYjNw3BtcUf/iqg6MM1IPTAoGAWYWJ
+SSqS7JVAcsrFYt1eIrdsNHVwr565OeM3X9v/Mr3FH1jeXeQWNSz1hU29Ticx7y+u
+1YBBlKGmHoHl/bd7lb9ggjkzU7JZRa+YjSIb+i/cwc5t7IJf7xUMk/vnz4tyd5zs
+Qm89gJZ2/Y1kwXSKvx53WNbyokvGKlpaZN1O418CgYACliGux77pe4bWeXSFFd9N
+50ipxDLVghw1c5AiZn25GR5YHJZaV4R0wmFcHdZvogLKi0jDMPvU69PaiT8eX/A1
+COkxv7jY1vtKlEtb+gugMjMN8wvb2va4kyFamjqnleiZlBSqIF/Y17wBoMvaWgZ0
+bEPCN//ts5hBwgb1TwGrrg==
+-----END PRIVATE KEY-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.cert.pem
new file mode 100644
index 000000000..4e41c15bb
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFljCCA36gAwIBAgICEAowDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDMxNjIwMjAzMloXDTMzMDYyMTIwMjAzMlowdjELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEDAOBgNVBAMMB0Ns
+aWVudDEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDcDhlEvUIYc9uA
+ocOBXt5thKrovs+8V0Eus/WrHMTKBk0Kw4X+7HBaRBoZj2sZpYfN63lVaO75kW4I
+uJuorGj5PAXYWJj+4uAsCc95xAN/liCuHJnxE5togWVt8W+z0Zll98RIpiCohqiE
+FLDL4X6FREL07GLgQZ/BFORvAwU+Gog05AFh43iZDnJl8MmrG2HBSRXtSZ6vQj9A
+NrOSqz5eK4YIHEEsgwTWQmhtNwu3Y+GzrAPWCA4TeYrSRwIrnGh20fOWXkAMldS4
+eRXmBztEsXMGqbe6oYO1QPYOlmoGO8EaaDPJ2sFIuM0zn98Alq3kCnRhM5Bi9RpJ
+7IpudIopAgMBAAGjggE3MIIBMzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
+oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
+ZmljYXRlMB0GA1UdDgQWBBQoIuXq3wG6JEzAEj9wPe7am0OVgjAfBgNVHSMEGDAW
+gBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwIGCCsGAQUFBwMEMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9s
+b2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUxLmNybC5wZW0wMQYIKwYBBQUHAQEE
+JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN
+AQELBQADggIBAHqKYcwkm3ODPD7Mqxq3bsswSXregWfc8tqfIBc5FZg2F+IzhxcJ
+kINB0lmcNdLALK6ka0sDs1Nrj1KB96NcHUqE+WY/qPS1Yksr34yFatb1ddlKQ9HK
+VRrIsi0ZfjBpHpvoQ0GsLeyRKm7iN/Fm5H9u8rw6RBu0Oe/l20FVSQIDzldYw51L
+uV/E9No8ZhdQ2Dffujs8madI7b7I1NMXS+Z1pZ+gYrz6O60tDEprE+rYuYWypURr
+fK+DnLLl+KQ+eekTPynw7LRpFzI/1cOMmd4BRnsBHCbCObfNp7WPasemZOEXGIlZ
+CQwZS62DYOJE4u4Nz5pSF+JgXfr6X/Im6Y1SV900xVHfoL0GpFDI9k+0Y5ncHfSH
++V9HlRWB3zqQF+yla32XOpBbER0vFDH52gp8/o1ZGg7rr6KrP4QKxnqywNLiAPDX
+txaAykZhON7uG8j+Lbjx5Ik91NRn9Fd5NH/vtT33a4uig2TP9EWd7EPcD2z8ONuD
+yiK3S37XAnmSKKX4HcCpEb+LedtqQo/+sqWyWXkpKdpkUSozvcYS4J/ob3z9N2IE
+qIH5I+Mty1I4EB4W89Pem8DHNq86Lt0Ea6TBtPTV8NwR5aG2vvLzb5lNdpANXYcp
+nGr57mTWaHnQh+yqgy66J++k+WokWkAkwE989AvUfNoQ+Jr6cTH8nKo2
+-----END CERTIFICATE-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.key.pem
new file mode 100644
index 000000000..b355a3814
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDcDhlEvUIYc9uA
+ocOBXt5thKrovs+8V0Eus/WrHMTKBk0Kw4X+7HBaRBoZj2sZpYfN63lVaO75kW4I
+uJuorGj5PAXYWJj+4uAsCc95xAN/liCuHJnxE5togWVt8W+z0Zll98RIpiCohqiE
+FLDL4X6FREL07GLgQZ/BFORvAwU+Gog05AFh43iZDnJl8MmrG2HBSRXtSZ6vQj9A
+NrOSqz5eK4YIHEEsgwTWQmhtNwu3Y+GzrAPWCA4TeYrSRwIrnGh20fOWXkAMldS4
+eRXmBztEsXMGqbe6oYO1QPYOlmoGO8EaaDPJ2sFIuM0zn98Alq3kCnRhM5Bi9RpJ
+7IpudIopAgMBAAECggEARcly2gnrXDXh9vlWN0EO6UyZpxZcay6AzX7k+k81WZyF
+8lPvutjhCL9wR4rkPE3ys6tp31xX7W3hp4JkWynSYLhYYjQ20R7CWTUDR2qScXP7
+CTyo1XuSXaIruKJI+o4OR/g7l46X7NpHtxuYtg/dQAZV9bbB5LzrHSCzEUGz9+17
+jV//cBgLBiMdlbdLuQoGt4NQpBkNrauBVFq7Nq648uKkICmUo3Bzn/dfn3ehB+Zc
++580S+tawYd224j19tFQmd5oK8tfjqKuHenNGjp/gsRoY86N7qAtc3VIQ0yjE6ez
+tgREo/ftCb8kGfwRJOAQIeeDamBv+FWNT6QzcOtbwQKBgQDzWhY9BUgI8JVzjYg0
+oWfU90On81BtckKsEo//8MTlgwOD2PnUF0hTZF3RcSPYT+HybouTqHT8EOLBAzqy
+1+koH06MnAc/Y2ipaAe2fGaVH3SuXAsV/b8VcWWl4Qx7tYJDhE7sKmdl3/+jHZ7A
+hZQzgOQnxxCANBo3pwF9KboDbwKBgQDnfglSpgYdGzFpWp1hZnPl2RKIfX/4M2z2
+s+hVN1tp+1VySPrBRUC3J6hKPQUzzeUzPICclHOnO+kP7jAos/rlJ9VcNdAQTbTL
+7Ds9Em1KJTBphE038YbW3e93rydQpukXh50wRD9RI/3F3A/1rKhab92DXZZr6Wqu
+JouhNV8f5wKBgQCLQ3XQi/Iyc4QDse5NuETUgoCsX7kaOTZghOr1nFMByV08mfI2
+5vAUES8DigzqYKS8eXjVEqWIDx3FOVThPmCG/ouUOkKHixs9P3SSgVSvaGX81l3d
+wu4UlmWGbWkYbsJSYyhLTOUJTwxby7qrEIbEhrGK9gfCZo7OZHucpkF2bwKBgFhl
+1qWK5JbExY+XnLWO6/7/b4ZTdkSPTrK+bJ/t7aiA41Yq7CZVjarjJ+6BcrUfkMCK
+AArK3Yck55C/wgApCkvrdBwsKHGxWrLsWIqvuLAxl1UTwnD0eCsgwMsRRZAUzLnB
+fZLq3MrdVZDywd1suzUdtpbta/11OtmZuoQq31JNAoGAIzmevuPaUZRqnjDpLXAm
+Bo11q6CunhG5qZX4wifeZ9Fp5AaQu97F36igZ5/eDxFkDCrFRq6teMiNjRQZSLA3
+5tMBkq6BtN2Ozzm/6D135c4BF14ODHqAMPUy4sXbw5PS/kRFs4fKAH/+LcAOPgyI
+N/jJIY1LfM7PzrG2NdyscMU=
+-----END PRIVATE KEY-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.cert.pem
new file mode 100644
index 000000000..0cba3fb26
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFljCCA36gAwIBAgICEAswDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDMxNjIwMjAzMloXDTMzMDYyMTIwMjAzMlowdjELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEDAOBgNVBAMMB0Ns
+aWVudDIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFLcCjzNhfY6Sk
+2nSdrB/6UPPeTCCH5NBBVLvu1hdlqLS4qEdq8+EjyMDZTtspRtYPkBfjpOrlBWUO
+lKyxw2mZOjZ8iWvd4sJaAI/6KZl5X0Rdsg1RjzW03kUdLx9XJCyrYY0YFrT1dgJo
+Ih56jk2SJX7wrz0NCJ05VPIdpaOF6CcziA+YhdVHcE6xyHagsYI0JdDWxFZrl9zT
+LyhaDgBUN/yUQBnxKzxs8TMT4YVSi73ouh5IP9Xvs52hd6HO8ZGVr+YthQZKo95p
+OlwFF+AQWxdDIKoPYUPFo8XMOXvOeQ9iUJarxrYSrelLXtGkaGLBolAvqo/YKE7j
+rcJWjRGHAgMBAAGjggE3MIIBMzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
+oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
+ZmljYXRlMB0GA1UdDgQWBBTOo9YSgx1h5k/imP7nOfRfzQrRxjAfBgNVHSMEGDAW
+gBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwIGCCsGAQUFBwMEMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9s
+b2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUyLmNybC5wZW0wMQYIKwYBBQUHAQEE
+JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN
+AQELBQADggIBAFo91lLqjPY67Wmj2yWxZuTTuUwXdXXUQxL6sEUUnfkECvRhNyBA
+eCHkfVopNbXZ5tdLfsUvXF0ulaC76GCK/P7gHOG9D/RJX/85VzhuJcqa4dsEEifg
+IiKIG7viYxSA6HFXuyzHvwNco3FqTBHbY46lKf1lWRVLhiAtcwcyPP34/RWcPfQi
+6NZfLyitu5U7Z9XVN5wCp8sg0ayaO5Ib2ejIYuBCUddV1gV//tSDf+rKCgtAbm/X
+K64Bf3GdaX3h6EhoqMZ+Z2f4XpKSXTabsWAU44xdVxisI82eo+NwT8KleE65GpOv
+nPvr/dLq5fQ6VtHbRL3wWqhzB1VKVCtd8a6RE2k8HVWflU3qgwJ+woF19ed921eq
+OZxc+KzjsGFyW1D2fPdgoZFmePadSstIME7qtCNEi7D3im01/1KKzE2m/nosrHeW
+ePjY2YrXu0w47re/N2kBJL2xRbj+fAjBsfNn9RhvQsWheXG6mgg8w1ac6y72ZA2W
+72pWoDkgXQMX5XBBj/zMnmwtrX9zTILFjNGFuWMPYgBRI0xOf2FoqqZ67cQ2yTW/
+1T/6Mp0FSh4cIo/ENiNSdvlt3BIo84EyOm3iHHy28Iv5SiFjF0pkwtXlYYvjM3+R
+BeWqlPsVCZXcVC1rPVDzfWZE219yghldY4I3QPJ7dlmszi8eI0HtzhTK
+-----END CERTIFICATE-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.key.pem
new file mode 100644
index 000000000..29196b1e2
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDFLcCjzNhfY6Sk
+2nSdrB/6UPPeTCCH5NBBVLvu1hdlqLS4qEdq8+EjyMDZTtspRtYPkBfjpOrlBWUO
+lKyxw2mZOjZ8iWvd4sJaAI/6KZl5X0Rdsg1RjzW03kUdLx9XJCyrYY0YFrT1dgJo
+Ih56jk2SJX7wrz0NCJ05VPIdpaOF6CcziA+YhdVHcE6xyHagsYI0JdDWxFZrl9zT
+LyhaDgBUN/yUQBnxKzxs8TMT4YVSi73ouh5IP9Xvs52hd6HO8ZGVr+YthQZKo95p
+OlwFF+AQWxdDIKoPYUPFo8XMOXvOeQ9iUJarxrYSrelLXtGkaGLBolAvqo/YKE7j
+rcJWjRGHAgMBAAECggEABJYUCcyJcnbagytBxfnaNQUuAp8AIypFG3kipq0l5Stk
+gGaTJq5F4OTGS4ofRsqeu07IgBSAfqJcJH8toPkDQqfvs6ftO1Mso2UzakMOcP51
+Ywxd91Kjm+LKOyHkHGDirPGnutUg/YpLLrrMvTk/bJHDZCM4i/WP1WTREVFjUgl7
+4L6Y53x2Lk5shJJhv0MzTGaoZzQcW0EbhNH1AI6MBv5/CN5m/7/+HCPlHSNKnozl
+o3PXD6l0XNfOY2Hi6MgS/Vd70s3VmDT9UCJNsDjdFpKNHmI7vr9FScOLN8EwbqWe
+maFa0TPknmPDmVjEGMtgGlJWL7Sm0MpNW+WsEXcDPQKBgQDv3sp0nVML9pxdzX/w
+rGebFaZaMYDWmV9w0V1uXYh4ZkpFmqrWkq/QSTGpwIP/x8WH9FBDUZqspLpPBNgG
+ft1XhuY34y3hoCxOyRhQcR/1dY+lgCzuN4G4MG3seq/cAXhrmkPljma/iO8KzoRK
+Pa+uaKFGHy1vWY2AmOhT20zr4wKBgQDScA3478TFHg9THlSFzqpzvVn5eAvmmrCQ
+RMYIZKFWPortqzeJHdA5ShVF1XBBar1yNMid7E7FXqi/P8Oh+E6Nuc7JxyVIJWlV
+mcBE1ceTKdZn7A0nuQIaU6hcn7xz/UHmxGur1ZcNQm3diFJ2CPn11lzZlkSZLSCN
+V86nndA9DQKBgQCWsUxXPo7xsRhDBdseg/ECyPMdLoRWTTxcT+t2bmRR31FBsQ0q
+iDTTkWgV0NAcXJCH/MB/ykB1vXceNVjRm9nKJwFyktI8MLglNsiDoM4HErgPrRqM
+/WoNIL+uFNVuTa4tS1jkWjXKlmg2Tc9mJKK92xWWS/frQENZSraKF/eXKQKBgGR9
+ni6CUTTQZgELOtGrHzql8ZFwAj7dH/PE48yeQW0t8KoOWTbhRc4V0pLGmhSjJFSl
+YCgJ8JPP4EVz7bgrG1gSou04bFVHiEWYZnh4nhVopTp7Psz5TEfGK2AP5658Ajxx
+D/m+xaNPVae0sawsHTGIbE57s8ZyBll41Pa2JfsBAoGBANtS7SOehkflSdry0eAZ
+50Ec3CmY+fArC044hQCmXxol5SiTUnXf/OIQH8y+RZUjF4F3PbbrFNLm/J6wuUPw
+XUIb4gAL75srakL8DXqyTYfO02aNrFEHhXzMs+GoAnRkFy16IAAFwpjbYSPanfed
+PfHCTWz6Y0pGdh1hUJAFV/3v
+-----END PRIVATE KEY-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.cert.pem
new file mode 100644
index 000000000..94092fad9
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFljCCA36gAwIBAgICEAwwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDMxNjIwMjAzMloXDTMzMDYyMTIwMjAzMlowdjELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEDAOBgNVBAMMB0Ns
+aWVudDMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEOZ6fYNjZDNXX
+eOyapHMOMeNeYM3b7vsWXAbiJIt4utVrTS0A+/G640t/U0g8F9jbKgbjEEPtgPJ7
+GltjLWObfqDWKSO2D9/ei2+NauqgiN/HX+dQnSKHob0McXBXvLfrA4tn4braKrbg
+p1fZB8bAECuT/bUhVBqWlzrUwDMpqjMJWDab48ixezb2gnc/ePE6wq/d3ecDb0/k
+cYWQ0LX4JiQBgaTGhwczyoGfL1z2vx5kJqptK+r0Hc2jNCn6kFvoZUCYjCWgWNxZ
+sQk7fObQQkUb/XQyqRaKJBWDyqsNcuK2gOg3LGeolAlgtMiEqGhHv77XdJnJug/w
+3OiHpP/7AgMBAAGjggE3MIIBMzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
+oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
+ZmljYXRlMB0GA1UdDgQWBBRxZFdIkSg6zDZCakXmIest5a6dBzAfBgNVHSMEGDAW
+gBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwIGCCsGAQUFBwMEMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9s
+b2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUzLmNybC5wZW0wMQYIKwYBBQUHAQEE
+JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN
+AQELBQADggIBAEntkhiPpQtModUF/ffnxruq+cqopPhIdMXhMD8gtU5e4e7o3EHX
+lfZKIbxyw56v6dFPrl4TuHBiBudqIvBCsPtllWKixWvg6FV3CrEeTcg4shUIaJcD
+pqv1qHLwS4pue6oau/lb8jv1GuzuBXoMFQwlmiOXO7xXqXjV2GdmkFJCDdB/0BW1
+VHvh0DXgotaxITWKhCpSNB7F7LSvegRwZIAN6JXrLDpue7tgqLqBB1EzpmS6ALbn
+uZDdikOs/tGAFB3un/3Gl7jEPL8UGOoSj/H9PUT5AFHrHJDH72+QSXu09agz8RWJ
+V939njYFCAxQ8Jt2mOK8BJQDJgPtLfIIb1iYicQV13Eypt8uIUYvp0i0Wq8WxPbq
+rOEvQYpcGUsreS5XqZ7y68hgq6ePiR18Fnc3GyTV5o6qT3W7IOvPArTzNV5fFCwM
+lx8xSEm+ebJrJPphp6Uc/h8evohvAN8R/Z7FSo9OL6V+F3ywPqWTXaqiIiRc9PS0
+0vxsYZ96EeZY5HzjN6LzHxmkv4KYM5I1qmXlviQlaU+sotp3tzegADlM4K78nUFh
+HuXamecEcS73eAgjk+FGqJ9E25B0TLlQMcP6tCKdaUIGn6ZsF5wT87GzqT99wL/5
+foHCYIkyG7ZmAQmoaKBd4q6xqVOUHovmsPza69FuSrsBxoRR39PtAnrY
+-----END CERTIFICATE-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.key.pem
new file mode 100644
index 000000000..6ede63fd2
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDEOZ6fYNjZDNXX
+eOyapHMOMeNeYM3b7vsWXAbiJIt4utVrTS0A+/G640t/U0g8F9jbKgbjEEPtgPJ7
+GltjLWObfqDWKSO2D9/ei2+NauqgiN/HX+dQnSKHob0McXBXvLfrA4tn4braKrbg
+p1fZB8bAECuT/bUhVBqWlzrUwDMpqjMJWDab48ixezb2gnc/ePE6wq/d3ecDb0/k
+cYWQ0LX4JiQBgaTGhwczyoGfL1z2vx5kJqptK+r0Hc2jNCn6kFvoZUCYjCWgWNxZ
+sQk7fObQQkUb/XQyqRaKJBWDyqsNcuK2gOg3LGeolAlgtMiEqGhHv77XdJnJug/w
+3OiHpP/7AgMBAAECggEADSe89sig5E63SKAlFXcGw0H2XgqIzDP/TGMnqPvNoYhX
+eSXUgxDhBptpB9e9a4RaKwaFxxPjlSXEdYFX9O22YSN1RMMl6Q8Zl9g3edhcDR6W
+b7Qbx2x8qj6Rjibnlh8JiFPiaDjN2wUeSDBss/9D98NkKiJ9Ue2YCYmJAOA3B3w9
+2t4Co5+3YrxkdzkvibTQCUSEwHFeB1Nim21126fknMPxyrf+AezRBRc8JNAHqzWb
+4QEeMnmIJDOzc3Oh7+P85tNyejOeRm9T7X3EQ0jKXgLYe+HUzXclBQ66b9x9Nc9b
+tNn6XkMlLlsQ3f149Th6PtHksH3hM+GF8bMuCp9yxQKBgQDGk0PYPkLqTD8jHjJW
+s8wBNhozigZPGaynxdTsD7L6UtDdOl1sSW/jFOj9UIs1duBce9dP1IjFc0jY+Kin
+lMLv3qCtk5ZjxRglOoLipR9hdClcM69rDoRZdoQK8KYa+QHcOTSazIp3fnw4gWSX
+nscelMfd1rtVP0dOGTuqE/73/QKBgQD8+F5WAi2IOVPHnBxAAOP+6XTs9Ntn1sDi
+L5wNgm+QA28aJJ4KRAwdXIc3IFPlHxZI77c2K1L9dKDu9X4UcyZIZYDvGVLuOOt5
+twaRaGuJW03cjbgOWC7rGyfzfZ49YlCZi2YuxERclBkbqgWD9hfa8twUfKNguF2Y
+AyiOhohtVwKBgQCJB8zUp7pzhqQ3LrpcHHzWBSi1kjTiVvxPVnSlZfwDRCz/zSv0
+8wRz9tUFIZS/E0ama4tcenTblL+bgpSX+E9BSiclQOiR9su/vQ3fK0Vpccis6LnP
+rdflCKT8C68Eg/slppBHloijBzTfpWLuQlJ0JwV5b5ocrKsfGMiUiHH1XQKBgQDg
+RnakfEPP7TtY0g+9ssxwOJxAZImM0zmojpsk4wpzvIeovuQap9+xvFHoztFyZhBE
+07oz3U8zhE4V7TI9gSVktBEOaf47U914yIqbKd+FJJywODkBBq96I1ZVKn67X0mk
+B5GtTrZo+agU/bTsHKdjp0L1KtdSLcJUviAb1Cxp+wKBgDrGqS01CCgxSUwMaZe4
+8HFWp/oMSyVDG9lTSC3uP/VL76zNFI55T3X06Q87hDN3gCJGUOmHzDZ/oCOgM4/S
+SU55M4lXeSEdFe84tMXJKOv5JXTkulzBYzATJ5J8DeS/4YZxMKyPDLXX8wgwmU+l
+i6Imd3qCPhh5eI3z9eSNDX+6
+-----END PRIVATE KEY-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/crl.pem
new file mode 100644
index 000000000..a119cede2
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/crl.pem
@@ -0,0 +1,20 @@
+-----BEGIN X509 CRL-----
+MIIDPDCCASQCAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV
+BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu
+dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMjA3MjAy
+MDIzNTNaFw0zMjEwMjUyMDIzNTNaMBUwEwICEAIXDTIyMDYxMzEyNDIwNVqgbjBs
+MB8GA1UdIwQYMBaAFCuv1TkzC1fSgTfzE1m1u5pRCJsVMDwGA1UdHAQ1MDOgLqAs
+hipodHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW2EAf8w
+CwYDVR0UBAQCAhADMA0GCSqGSIb3DQEBCwUAA4ICAQBbWdqRFsIrG6coL6ln1RL+
+uhgW9l3XMmjNlyiYHHNzOgnlBok9xu9UdaVCOKC6GEthWSzSlBY1AZugje57DQQd
+RkIJol9am94lKMTjF/qhzFLiSjho8fwZGDGyES5YeZXkLqNMOf6m/drKaI3iofWf
+l63qU9jY8dnSrVDkwgCguUL2FTx60v5H9NPxSctQ3VDxDvDj0sTAcHFknQcZbfvY
+ZWpOYNS0FAJlQPVK9wUoDxI0LhrWDq5h/T1jcGO34fPT8RUA5HRtFVUevqSuOLWx
+WTfTx5oDeMZPJTvHWUcg4yMElHty4tEvtkFxLSYbZqj7qTU+mi/LAN3UKBH/gBEN
+y2OsJvFhVRgHf+zPYegf3WzBSoeaXNAJZ4UnRo34P9AL3Mrh+4OOUP++oYRKjWno
+pYtAmTrIwEYoLyisEhhZ6aD92f/Op3dIYsxwhHt0n0lKrbTmUfiJUAe7kUZ4PMn4
+Gg/OHlbEDaDxW1dCymjyRGl+3/8kjy7bkYUXCf7w6JBeL2Hw2dFp1Gh13NRjre93
+PYlSOvI6QNisYGscfuYPwefXogVrNjf/ttCksMa51tUk+ylw7ZMZqQjcPPSzmwKc
+5CqpnQHfolvRuN0xIVZiAn5V6/MdHm7ocrXxOkzWQyaoNODTq4js8h8eYXgAkt1w
+p1PTEFBucGud7uBDE6Ub6A==
+-----END X509 CRL-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf
new file mode 100644
index 000000000..f34ab1456
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf
@@ -0,0 +1,12 @@
+crl_cache.refresh_interval = {{ refresh_interval }}
+crl_cache.http_timeout = 17s
+crl_cache.capacity = {{ cache_capacity }}
+listeners.ssl.default {
+  ssl_options {
+    keyfile = "{{ test_data_dir }}/server.key.pem"
+    certfile = "{{ test_data_dir }}/server.cert.pem"
+    cacertfile = "{{ test_data_dir }}/ca-chain.cert.pem"
+    verify = verify_peer
+    enable_crl_check = true
+  }
+}
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_crl_cache_http_server.erl b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_crl_cache_http_server.erl
new file mode 100644
index 000000000..4e8b989fa
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_crl_cache_http_server.erl
@@ -0,0 +1,67 @@
+-module(emqx_crl_cache_http_server).
+
+-behaviour(gen_server).
+-compile([nowarn_export_all, export_all]).
+
+set_crl(CRLPem) ->
+    ets:insert(?MODULE, {crl, CRLPem}).
+
+%%--------------------------------------------------------------------
+%% `gen_server' APIs
+%%--------------------------------------------------------------------
+
+start_link(Parent, BasePort, CRLPem, Opts) ->
+    process_flag(trap_exit, true),
+    stop_http(),
+    timer:sleep(100),
+    gen_server:start_link(?MODULE, {Parent, BasePort, CRLPem, Opts}, []).
+
+init({Parent, BasePort, CRLPem, Opts}) ->
+    Tab = ets:new(?MODULE, [named_table, ordered_set, public]),
+    ets:insert(Tab, {crl, CRLPem}),
+    ok = start_http(Parent, [{port, BasePort} | Opts]),
+    Parent ! {self(), ready},
+    {ok, #{parent => Parent}}.
+
+handle_call(_Request, _From, State) ->
+    {reply, ignored, State}.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    stop_http().
+
+stop(Pid) ->
+    ok = gen_server:stop(Pid).
+
+%%--------------------------------------------------------------------
+%% Callbacks
+%%--------------------------------------------------------------------
+
+start_http(Parent, Opts) ->
+    {ok, _Pid1} = cowboy:start_clear(http, Opts, #{
+        env => #{dispatch => compile_router(Parent)}
+    }),
+    ok.
+
+stop_http() ->
+    cowboy:stop_listener(http),
+    ok.
+
+compile_router(Parent) ->
+    {ok, _} = application:ensure_all_started(cowboy),
+    cowboy_router:compile([
+        {'_', [{'_', ?MODULE, #{parent => Parent}}]}
+    ]).
+
+init(Req, #{parent := Parent} = State) ->
+    %% assert
+    <<"GET">> = cowboy_req:method(Req),
+    [{crl, CRLPem}] = ets:lookup(?MODULE, crl),
+    Parent ! {http_get, iolist_to_binary(cowboy_req:uri(Req))},
+    Reply = reply(Req, CRLPem),
+    {ok, Reply, State}.
+
+reply(Req, CRLPem) ->
+    cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, CRLPem, Req).
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf
new file mode 100644
index 000000000..8b9549823
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf
@@ -0,0 +1,12 @@
+node.name = test@127.0.0.1
+node.cookie = emqxsecretcookie
+node.data_dir = "{{ test_priv_dir }}"
+listeners.ssl.default {
+  ssl_options {
+    keyfile = "{{ test_data_dir }}/server.key.pem"
+    certfile = "{{ test_data_dir }}/server.cert.pem"
+    cacertfile = "{{ test_data_dir }}/ca-chain.cert.pem"
+    verify = verify_peer
+    enable_crl_check = false
+  }
+}
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-not-revoked.crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-not-revoked.crl.pem
new file mode 100644
index 000000000..e484b44c0
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-not-revoked.crl.pem
@@ -0,0 +1,19 @@
+-----BEGIN X509 CRL-----
+MIIDJTCCAQ0CAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV
+BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu
+dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMzAxMTIx
+MzA4MTZaFw0zMzAxMDkxMzA4MTZaoG4wbDAfBgNVHSMEGDAWgBRMcIY7FVKJurUP
+kqqusTFBE75z8zA8BgNVHRwENTAzoC6gLIYqaHR0cDovL2xvY2FsaG9zdDo5ODc4
+L2ludGVybWVkaWF0ZS5jcmwucGVthAH/MAsGA1UdFAQEAgIQADANBgkqhkiG9w0B
+AQsFAAOCAgEAJGOZuqZL4m7zUaRyBrxeT6Tqo+XKz7HeD5zvO4BTNX+0E0CRyki4
+HhIGbxjv2NKWoaUv0HYbGAiZdO4TaPu3w3tm4+pGEDBclBj2KTdbB+4Hlzv956gD
+KXZ//ziNwx1SCoxxkxB+TALxReN0shE7Mof9GlB5HPskhLorZgg/pmgJtIykEpsq
+QAjJo4aq+f2/L+9dzRM205fVFegtsHvgEVNKz6iK6skt+kDhj/ks9BKsnfCDIGr+
+XnPYwS9yDnnhFdoJ40AQQDtomxggAjfgcSnqtHCxZwKJohuztbSWUgD/4yxzlrwP
+Dk1cT/Ajjjqb2dXVOfTLK1VB2168uuouArxZ7KYbXwBjHduYWGGkA6FfkNJO/jpF
+SL9qhX3oxcRF3hDhWigN1ZRD7NpDKwVal3Y9tmvO5bWhb5VF+3qv0HGeSGp6V0dp
+sjwhIj+78bkUrcXxrivACLAXgSTGonx1uXD+T4P4NCt148dgRAbgd8sUXK5FcgU2
+cdBl8Kv2ZUjEaod5gUzDtf22VGSoO9lHvfHdpG9o2H3wC7s4tyLTidNrduIguJff
+IIgc44Y252iV0sOmZ5S0jjTRiF1YUUPy9qA/6bOnr2LohbwbNZv9tDlNj8cdhxUz
+cKiS+c7Qsz+YCcrp19QRiJoQae/gUqz7kmUZQgyPmDd+ArE0V+kDZEE=
+-----END X509 CRL-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked-no-dp.crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked-no-dp.crl.pem
new file mode 100644
index 000000000..4d3611d49
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked-no-dp.crl.pem
@@ -0,0 +1,19 @@
+-----BEGIN X509 CRL-----
+MIIC/TCB5gIBATANBgkqhkiG9w0BAQsFADBrMQswCQYDVQQGEwJTRTESMBAGA1UE
+CAwJU3RvY2tob2xtMRIwEAYDVQQKDAlNeU9yZ05hbWUxGTAXBgNVBAsMEE15SW50
+ZXJtZWRpYXRlQ0ExGTAXBgNVBAMMEE15SW50ZXJtZWRpYXRlQ0EXDTIzMDExODEz
+Mjc1M1oXDTMzMDExNTEzMjc1M1owFTATAgIQAhcNMjMwMTEyMTMwODE2WqAwMC4w
+HwYDVR0jBBgwFoAUTHCGOxVSibq1D5KqrrExQRO+c/MwCwYDVR0UBAQCAhACMA0G
+CSqGSIb3DQEBCwUAA4ICAQCxoRYDc5MaBpDI+HQUX60+obFeZJdBkPO2wMb6HBQq
+e0lZM2ukS+4n5oGhRelsvmEz0qKvnYS6ISpuFzv4Qy6Vaun/KwIYAdXsEQVwDHsu
+Br4m1V01igjFnujowwR/7F9oPnZOmBaBdiyYbjgGV0YMF7sOfl4UO2MqI2GSGqVk
+63wELT1AXjx31JVoyATQOQkq1A5HKFYLEbFmdF/8lNfbxSCBY2tuJ+uWVQtzjM0y
+i+/owz5ez1BZ/Swx8akYhuvs8DVVTbjXydidVSrxt/QEf3+oJCzTA9qFqt4MH7gL
+6BAglCGtRiYTHqeMHrwddaHF2hzR61lHJlkMCL61yhVuL8WsEJ/AxVX0W3MfQ4Cw
+x/A6xIkgqtu+HtQnPyDcJxyaFHtFC+U67nSbEQySFvHfMw42DGdIGojKQCeUer9W
+ECFC8OATQwN2h//f8QkY7D0H3k/brrNYDfdFIcCti9iZiFrrPFxO7NbOTfkeKCt3
+7IwYduRc8DWKmS8c7j2re1KkdYnfE1sfwbn3trImkcET5tvDlVCZ1glnBQzk82PS
+HvKmSjD2pZI7upfLkoMgMhYyYJhYk7Mw2o4JXuddYGKmmw3bJyHkG/Ot5NAKjb7g
+k1QCeWzxO1xXm8PNDDFWMn351twUGDQ/cwrUw0ODeUZpfL0BtTn4YnfCLLTvZDxo
+Vg==
+-----END X509 CRL-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked.crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked.crl.pem
new file mode 100644
index 000000000..4c5cdd441
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked.crl.pem
@@ -0,0 +1,20 @@
+-----BEGIN X509 CRL-----
+MIIDPDCCASQCAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV
+BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu
+dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMzAxMTIx
+MzA4MTZaFw0zMzAxMDkxMzA4MTZaMBUwEwICEAIXDTIzMDExMjEzMDgxNlqgbjBs
+MB8GA1UdIwQYMBaAFExwhjsVUom6tQ+Sqq6xMUETvnPzMDwGA1UdHAQ1MDOgLqAs
+hipodHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW2EAf8w
+CwYDVR0UBAQCAhABMA0GCSqGSIb3DQEBCwUAA4ICAQCPadbaehEqLv4pwqF8em8T
+CW8TOQ4Vjz02uiVk9Bo0za1dQqQmwCBA6UE1BcOh+aWzQxBRz56NeUcfhgDxTntG
+xLs896N9MHIG6UxpqJH8cH+DXKHsQjvvCjXtiObmBQR1RiG5C1vEMkfzTt/WSrq5
+7blowLDs4NP6YbtqXEyyUkF7DQSUEUuIDWPQdx1f++nSpVaHWW4xpoO4umesaJco
+FuxaXQnZpTHHQfqUJVIL2Mmzvez9thgfKTV3vgkYrGiSLW2m2+Tfga30pUc0qaVI
+RrBVORVbcu9m1sV0aJyk96b2T/+i2FRR/np4TOcLgckBpHKeK2FH69lHFr0W/71w
+CErNTxahoh82Yi8POenu+S1m2sDnrF1FMf+ZG/i2wr0nW6/+zVGQsEOw77Spbmei
+dbEchu3iWF1XEO/n4zVBzl6a1o2RyVg+1pItYd5C5bPwcrfZnBrm4WECPxO+6rbW
+2/wz9Iku4XznTLqLEpXLAtenAdo73mLGC7riviX7mhcxfN2UjNfLuVGHmG8XwIsM
+Lgpr6DKaxHwpHgW3wA3SGJrY5dj0TvGWaoInrNt1cOMnIpoxRNy5+ko71Ubx3yrV
+RhbUMggd1GG1ct9uZn82v74RYF6J8Xcxn9vDFJu5LLT5kvfy414kdJeTXKqfKXA/
+atdUgFa0otoccn5FzyUuzg==
+-----END X509 CRL-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate.crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate.crl.pem
new file mode 100644
index 000000000..a119cede2
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate.crl.pem
@@ -0,0 +1,20 @@
+-----BEGIN X509 CRL-----
+MIIDPDCCASQCAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV
+BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu
+dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMjA3MjAy
+MDIzNTNaFw0zMjEwMjUyMDIzNTNaMBUwEwICEAIXDTIyMDYxMzEyNDIwNVqgbjBs
+MB8GA1UdIwQYMBaAFCuv1TkzC1fSgTfzE1m1u5pRCJsVMDwGA1UdHAQ1MDOgLqAs
+hipodHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW2EAf8w
+CwYDVR0UBAQCAhADMA0GCSqGSIb3DQEBCwUAA4ICAQBbWdqRFsIrG6coL6ln1RL+
+uhgW9l3XMmjNlyiYHHNzOgnlBok9xu9UdaVCOKC6GEthWSzSlBY1AZugje57DQQd
+RkIJol9am94lKMTjF/qhzFLiSjho8fwZGDGyES5YeZXkLqNMOf6m/drKaI3iofWf
+l63qU9jY8dnSrVDkwgCguUL2FTx60v5H9NPxSctQ3VDxDvDj0sTAcHFknQcZbfvY
+ZWpOYNS0FAJlQPVK9wUoDxI0LhrWDq5h/T1jcGO34fPT8RUA5HRtFVUevqSuOLWx
+WTfTx5oDeMZPJTvHWUcg4yMElHty4tEvtkFxLSYbZqj7qTU+mi/LAN3UKBH/gBEN
+y2OsJvFhVRgHf+zPYegf3WzBSoeaXNAJZ4UnRo34P9AL3Mrh+4OOUP++oYRKjWno
+pYtAmTrIwEYoLyisEhhZ6aD92f/Op3dIYsxwhHt0n0lKrbTmUfiJUAe7kUZ4PMn4
+Gg/OHlbEDaDxW1dCymjyRGl+3/8kjy7bkYUXCf7w6JBeL2Hw2dFp1Gh13NRjre93
+PYlSOvI6QNisYGscfuYPwefXogVrNjf/ttCksMa51tUk+ylw7ZMZqQjcPPSzmwKc
+5CqpnQHfolvRuN0xIVZiAn5V6/MdHm7ocrXxOkzWQyaoNODTq4js8h8eYXgAkt1w
+p1PTEFBucGud7uBDE6Ub6A==
+-----END X509 CRL-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/server.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/server.cert.pem
new file mode 100644
index 000000000..38cc63534
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/server.cert.pem
@@ -0,0 +1,35 @@
+-----BEGIN CERTIFICATE-----
+MIIGCTCCA/GgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNloweDELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEjAQBgNVBAMMCWxv
+Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKdU9FaA/n0Z
+TXkd10XA9l+UV9xKR65ZTy2ApCFlw2gGWLiUh96a6hX+GQZFUV7ECIDDf+7nC85o
+xo1Xyf0rHGABQ0uHlhqSemc12F9APIzRLlQkhtV4vMBBbGQFekje4F9bhY9JQtGd
+XJGmwsR+XWo6SUY7K5l9FuSSSRXC0kSYYQfSTPR/LrF6efdHf+ZN4huP7lM2qIFd
+afX+qBOI1/Y2LtITo2TaU/hXyKh9wEiuynoq0RZ2KkYQll5cKD9fSD+pW3Xm0XWX
+TQy4RZEe3WoYEQsklNw3NC92ocA/PQB9BGNO1fKhzDn6kW2HxDxruDKOuO/meGek
+ApCayu3e/I0CAwEAAaOCAagwggGkMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQD
+AgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2Vy
+dGlmaWNhdGUwHQYDVR0OBBYEFGy5LQPzIelruJl7mL0mtUXM57XhMIGaBgNVHSME
+gZIwgY+AFExwhjsVUom6tQ+Sqq6xMUETvnPzoXOkcTBvMQswCQYDVQQGEwJTRTES
+MBAGA1UECAwJU3RvY2tob2xtMRIwEAYDVQQHDAlTdG9ja2hvbG0xEjAQBgNVBAoM
+CU15T3JnTmFtZTERMA8GA1UECwwITXlSb290Q0ExETAPBgNVBAMMCE15Um9vdENB
+ggIQADAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwOwYDVR0f
+BDQwMjAwoC6gLIYqaHR0cDovL2xvY2FsaG9zdDo5ODc4L2ludGVybWVkaWF0ZS5j
+cmwucGVtMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDovL2xvY2Fs
+aG9zdDo5ODc3MA0GCSqGSIb3DQEBCwUAA4ICAQCX3EQgiCVqLhnCNd0pmptxXPxo
+l1KyZkpdrFa/NgSqRhkuZSAkszwBDDS/gzkHFKEUhmqs6/UZwN4+Rr3LzrHonBiN
+aQ6GeNNXZ/3xAQfUCwjjGmz9Sgw6kaX19Gnk2CjI6xP7T+O5UmsMI9hHUepC9nWa
+XX2a0hsO/KOVu5ZZckI16Ek/jxs2/HEN0epYdvjKFAaVmzZZ5PATNjrPQXvPmq2r
+x++La+3bXZsrH8P2FhPpM5t/IxKKW/Tlpgz92c2jVSIHF5khSA/MFDC+dk80OFmm
+v4ZTPIMuZ//Q+wo0f9P48rsL9D27qS7CA+8pn9wu+cfnBDSt7JD5Yipa1gHz71fy
+YTa9qRxIAPpzW2v7TFZE8eSKFUY9ipCeM2BbdmCQGmq4+v36b5TZoyjH4k0UVWGo
+Gclos2cic5Vxi8E6hb7b7yZpjEfn/5lbCiGMfAnI6aoOyrWg6keaRA33kaLUEZiK
+OgFNbPkjiTV0ZQyLXf7uK9YFhpVzJ0dv0CFNse8rZb7A7PLn8VrV/ZFnJ9rPoawn
+t7ZGxC0d5BRSEyEeEgsQdxuY4m8OkE18zwhCkt2Qs3uosOWlIrYmqSEa0i/sPSQP
+jiwB4nEdBrf8ZygzuYjT5T9YRSwhVox4spS/Av8Ells5JnkuKAhCVv9gHxYwbj0c
+CzyLJgE1z9Tq63m+gQ==
+-----END CERTIFICATE-----
diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/server.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/server.key.pem
new file mode 100644
index 000000000..d456ece72
--- /dev/null
+++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/server.key.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCnVPRWgP59GU15
+HddFwPZflFfcSkeuWU8tgKQhZcNoBli4lIfemuoV/hkGRVFexAiAw3/u5wvOaMaN
+V8n9KxxgAUNLh5YaknpnNdhfQDyM0S5UJIbVeLzAQWxkBXpI3uBfW4WPSULRnVyR
+psLEfl1qOklGOyuZfRbkkkkVwtJEmGEH0kz0fy6xenn3R3/mTeIbj+5TNqiBXWn1
+/qgTiNf2Ni7SE6Nk2lP4V8iofcBIrsp6KtEWdipGEJZeXCg/X0g/qVt15tF1l00M
+uEWRHt1qGBELJJTcNzQvdqHAPz0AfQRjTtXyocw5+pFth8Q8a7gyjrjv5nhnpAKQ
+msrt3vyNAgMBAAECggEABnWvIQ/Fw0qQxRYz00uJt1LguW5cqgxklBsdOvTUwFVO
+Y4HIZP2R/9tZV/ahF4l10pK5g52DxSoiUB6Ne6qIY+RolqfbUZdKBmX7vmGadM02
+fqUSV3dbwghEiO/1Mo74FnZQB6IKZFEw26aWakN+k7VAUufB3SEJGzXSgHaO63ru
+dFGSiYI8U+q+YnhUJjCnmI12fycNfy451TdUQtGZb6pNmm5HRUF6hpAV8Le9LojP
+Ql9eacPpsrzU15X5ElCQZ/f9iNh1bplcISuhrULgKUKOvAVrBlEK67uRVy6g98xA
+c/rgNLkbL/jZEsAc3/vHAyFgd3lABfwpBGLHej3QgQKBgQDFNYmfBNQr89HC5Zc+
+M6jXcAT/R+0GNczBTfC4iyNemwqsumSSRelNZ748UefKuS3F6Mvb2CBqE2LbB61G
+hrnCffG2pARjZ491SefRwghhWWVGLP1p8KliLgOGBehA1REgJb+XULncjuHZuh4O
+LVn3HVnWGxeBGg+yKa6Z4YQi3QKBgQDZN0O8ZcZY74lRJ0UjscD9mJ1yHlsssZag
+njkX/f0GR/iVpfaIxQNC3gvWUy2LsU0He9sidcB0cfej0j/qZObQyFsCB0+utOgy
++hX7gokV2pes27WICbNWE2lJL4QZRJgvf82OaEy57kfDrm+eK1XaSZTZ10P82C9u
+gAmMnontcQKBgGu29lhY9tqa7jOZ26Yp6Uri8JfO3XPK5u+edqEVvlfqL0Zw+IW8
+kdWpmIqx4f0kcA/tO4v03J+TvycLZmVjKQtGZ0PvCkaRRhY2K9yyMomZnmtaH4BB
+5wKtR1do2pauyg/ZDnDDswD5OfsGYWw08TK8YVlEqu3lIjWZ9rguKVIxAoGAZYUk
+zVqr10ks3pcCA2rCjkPT4lA5wKvHgI4ylPoKVfMxRY/pp4acvZXV5ne9o7pcDBFh
+G7v5FPNnEFPlt4EtN4tMragJH9hBZgHoYEJkG6islweg0lHmVWaBIMlqbfzXO+v5
+gINSyNuLAvP2CvCqEXmubhnkFrpbgMOqsuQuBqECgYB3ss2PDhBF+5qoWgqymFof
+1ovRPuQ9sPjWBn5IrCdoYITDnbBzBZERx7GLs6A/PUlWgST7jkb1PY/TxYSUfXzJ
+SNd47q0mCQ+IUdqUbHgpK9b1ncwLMsnexpYZdHJWRLgnUhOx7OMjJc/4iLCAFCoN
+3KJ7/V1keo7GBHOwnsFcCA==
+-----END PRIVATE KEY-----
diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl
index c45bc15ef..90cb5fd4d 100644
--- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl
+++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl
@@ -76,7 +76,7 @@ init_per_testcase(t_openssl_client, Config) ->
         [],
         Handler,
         #{
-            extra_mustache_vars => [{test_data_dir, DataDir}],
+            extra_mustache_vars => #{test_data_dir => DataDir},
             conf_file_path => ConfFilePath
         }
     ),
diff --git a/changes/ce/feat-10164.en.md b/changes/ce/feat-10164.en.md
new file mode 100644
index 000000000..9acea755f
--- /dev/null
+++ b/changes/ce/feat-10164.en.md
@@ -0,0 +1 @@
+Add CRL check support for TLS MQTT listeners.
diff --git a/rebar.config b/rebar.config
index 5ce9138ce..343c6be69 100644
--- a/rebar.config
+++ b/rebar.config
@@ -37,7 +37,13 @@
 
 {cover_opts, [verbose]}.
 {cover_export_enabled, true}.
-{cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb]}.
+{cover_excl_mods,
+ [ %% generated protobuf modules
+   emqx_exproto_pb,
+   emqx_exhook_pb,
+   %% taken almost as-is from OTP
+   emqx_ssl_crl_cache
+ ]}.
 
 {provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}.
 

From f2b54588e79d57a87e04d822d75964daddbeef7f Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Mon, 20 Mar 2023 11:16:32 -0300
Subject: [PATCH 12/52] docs: improve descriptions

Co-authored-by: Zaiming (Stone) Shi <zmstone@gmail.com>
---
 apps/emqx/i18n/emqx_schema_i18n.conf | 8 ++++----
 apps/emqx/src/emqx_schema.erl        | 5 +----
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf
index e05ee4c35..6f926ec39 100644
--- a/apps/emqx/i18n/emqx_schema_i18n.conf
+++ b/apps/emqx/i18n/emqx_schema_i18n.conf
@@ -1824,8 +1824,8 @@ server_ssl_opts_schema_enable_crl_check {
 crl_cache_refresh_http_timeout {
     desc {
         en: "The timeout for the HTTP request when fetching CRLs.  This is"
-            " global for all listeners."
-        zh: "获取 CRLs 时 HTTP 请求的超时。 这对所有监听器来说是全局的。"
+            " a global setting for all listeners."
+        zh: "获取 CRLs 时 HTTP 请求的超时。 该配置对所有启用 CRL 检查的监听器监听器有效。"
     }
     label: {
         en: "CRL Cache Refresh HTTP Timeout"
@@ -1835,9 +1835,9 @@ crl_cache_refresh_http_timeout {
 
 crl_cache_refresh_interval {
     desc {
-        en: "The period to refresh the CRLs from the servers.  This is global"
+        en: "The period to refresh the CRLs from the servers.  This is a global setting"
             " for all URLs and listeners."
-        zh: "从服务器刷新CRL的周期。 这对所有URL和监听器来说是全局性的。"
+        zh: "从服务器刷新CRL的周期。 该配置对所有 URL 和监听器有效。"
     }
     label: {
         en: "CRL Cache Refresh Interval"
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 25b1dd1d6..433fb20e5 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -230,7 +230,7 @@ roots(low) ->
         {"crl_cache",
             sc(
                 ref("crl_cache"),
-                #{}
+                #{hidden => true}
             )}
     ].
 
@@ -809,7 +809,6 @@ fields("crl_cache") ->
             sc(
                 duration(),
                 #{
-                    hidden => true,
                     default => <<"15m">>,
                     desc => ?DESC("crl_cache_refresh_interval")
                 }
@@ -818,7 +817,6 @@ fields("crl_cache") ->
             sc(
                 duration(),
                 #{
-                    hidden => true,
                     default => <<"15s">>,
                     desc => ?DESC("crl_cache_refresh_http_timeout")
                 }
@@ -827,7 +825,6 @@ fields("crl_cache") ->
             sc(
                 pos_integer(),
                 #{
-                    hidden => true,
                     default => 100,
                     desc => ?DESC("crl_cache_capacity")
                 }

From 89782e6d2113e794b6931e5e4d75c9d3d15a46e2 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Tue, 21 Mar 2023 06:26:04 +0100
Subject: [PATCH 13/52] docs: add info about that rule engine can handle
 different types of structured data

Co-authored-by: Zaiming (Stone) Shi <zmstone@gmail.com>
---
 apps/emqx_rule_engine/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_rule_engine/README.md b/apps/emqx_rule_engine/README.md
index f1533a0e3..2c2e43db3 100644
--- a/apps/emqx_rule_engine/README.md
+++ b/apps/emqx_rule_engine/README.md
@@ -14,8 +14,8 @@ specifying a topic filter and a set of conditions that need to be met. If a
 message matches the topic filter and all the conditions are met, the rule is
 triggered. The rule can then transform the message and route it to a different
 topic, or send it to another service (defined by an EMQX bridge). The rule
-engine's message data transformation is designed to work well with JSON
-formatted data.
+engine's message data transformation is designed to work well with structured data
+such as JSON, avro, and protobuf.
 
 
 A rule consists of the three parts **MATCH**, **TRANSFORM** and **ACTIONS** that are

From b6e583143fae9e5cbb5483e731226ef573819e88 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 21 Mar 2023 09:01:25 +0100
Subject: [PATCH 14/52] chore: bump quicer 0.0.114

for fetching quicer binary for macOS M1 builds.
---
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 2 +-
 rebar.config.erl              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index 0827570ff..12e6124ac 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.113"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.114"}}}.
 
 Dialyzer = fun(Config) ->
                    {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
diff --git a/mix.exs b/mix.exs
index 42354f8dc..74be81bcd 100644
--- a/mix.exs
+++ b/mix.exs
@@ -651,7 +651,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.113", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.114", override: true}],
       else: []
   end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index e976d7729..98cd30570 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.113"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.114"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From 092daf409a8956c5edee7ec13bc8be4871f56ce2 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 21 Mar 2023 09:03:31 +0100
Subject: [PATCH 15/52] chore(testlib): update include for CI of emqtt lib

---
 apps/emqx/test/emqx_common_test_helpers.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index 38f30b8c5..01a01b1fc 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -16,7 +16,7 @@
 
 -module(emqx_common_test_helpers).
 
--include("emqx_authentication.hrl").
+-include_lib("emqx/include/emqx_authentication.hrl").
 
 -type special_config_handler() :: fun().
 

From b62e9bba807d29eb0b65c707476a6438bbc84cb9 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 21 Mar 2023 10:04:02 +0100
Subject: [PATCH 16/52] ci: rerun failed checks on master branch

---
 .github/workflows/geen_master.yaml |  26 ++++++
 scripts/rerun-failed-checks.py     | 128 +++++++++++++++++++++++++++++
 2 files changed, 154 insertions(+)
 create mode 100644 .github/workflows/geen_master.yaml
 create mode 100644 scripts/rerun-failed-checks.py

diff --git a/.github/workflows/geen_master.yaml b/.github/workflows/geen_master.yaml
new file mode 100644
index 000000000..1161ca7d4
--- /dev/null
+++ b/.github/workflows/geen_master.yaml
@@ -0,0 +1,26 @@
+---
+
+name: Keep master green
+
+on:
+  schedule:
+    # run hourly
+    - cron: "0 * * * *"
+  workflow_dispatch:
+
+jobs:
+  rerun-failed-jobs:
+    runs-on: ubuntu-22.04
+    if: github.repository_owner == 'emqx'
+    permissions:
+      checks: read
+      actions: write
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: run script
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 scripts/rerun-failed-checks.py
diff --git a/scripts/rerun-failed-checks.py b/scripts/rerun-failed-checks.py
new file mode 100644
index 000000000..167d8e81c
--- /dev/null
+++ b/scripts/rerun-failed-checks.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# Usage: python3 rerun-failed-checks.py -t <github token> -r <repo> -b <branch>
+#
+# Description: This script will fetch the latest commit from a branch, and check the status of all check runs of the commit.
+# If any check run is not successful, it will trigger a rerun of the failed jobs.
+#
+# Default branch is master, default repo is emqx/emqx
+#
+# Limitation: only works for upstream repo, not for forked.
+import requests
+import http.client
+import json
+import os
+import sys
+import time
+from optparse import OptionParser
+
+job_black_list = [
+    'windows',
+    'publish_artifacts',
+    'stale'
+]
+
+def fetch_latest_commit(token: str, repo: str, branch: str):
+    url = f'https://api.github.com/repos/{repo}/commits/{branch}'
+    headers = {'Accept': 'application/vnd.github+json',
+               'Authorization': f'Bearer {token}',
+               'X-GitHub-Api-Version': '2022-11-28',
+               'User-Agent': 'python3'
+               }
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        res = r.json()
+        return res
+    else:
+        print(
+            f'Failed to fetch latest commit from {branch} branch, code: {r.status_code}')
+        sys.exit(1)
+
+
+'''
+fetch check runs of a commit.
+@note, only works for public repos
+'''
+def fetch_check_runs(token: str, repo: str, ref: str):
+    failed_checks = []
+    url = f'https://api.github.com/repos/{repo}/commits/{ref}/check-runs?per_page=100'
+    headers = {'Accept': 'application/vnd.github.v3+json',
+               'Authorization': f'Bearer {token}'
+               }
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        for crun in r.json()['check_runs']:
+            if crun['status'] == 'completed' and crun['conclusion'] != 'success':
+                print('Failed check: ', crun['name'])
+                failed_checks.append(
+                    {'id': crun['id'], 'name': crun['name'], 'url': crun['url']})
+            else:
+                # pretty print crun
+                # print(json.dumps(crun, indent=4))
+                print('successed:', crun['id'], crun['name'],
+                      crun['status'], crun['conclusion'])
+    else:
+        print(f'Failed to fetch check runs {r.status_code}')
+        sys.exit(1)
+    return failed_checks
+
+
+'''
+rerquest a check-run
+'''
+def trigger_build(failed_checks: list, repo: str, token: str):
+    reruns = []
+    for crun in failed_checks:
+        if crun['name'] in job_black_list:
+            print(f'Skip black listed job {crun["name"]}')
+            continue
+
+        r = requests.get(crun['url'], headers={'Accept': 'application/vnd.github.v3+json',
+                                               'User-Agent': 'python3',
+                                               'Authorization': f'Bearer {token}'}
+                         )
+        if r.status_code == 200:
+            # url example: https://github.com/qzhuyan/emqx/actions/runs/4469557961/jobs/7852858687
+            run_id = r.json()['details_url'].split('/')[-3]
+            reruns.append(run_id)
+        else:
+            print(f'failed to fetch check run {crun["name"]}')
+
+    # remove duplicates
+    for run_id in set(reruns):
+        url = f'https://api.github.com/repos/{repo}/actions/runs/{run_id}/rerun-failed-jobs'
+
+        r = requests.post(url, headers={'Accept': 'application/vnd.github.v3+json',
+                                        'User-Agent': 'python3',
+                                        'Authorization': f'Bearer {token}'}
+                          )
+        if r.status_code == 201:
+            print(f'Successfully triggered build for {crun["name"]}')
+
+        else:
+            # Only complain but not exit.
+            print(
+                f'Failed to trigger build for {crun["name"]} : {r.status_code} : {r.text}')
+
+
+def main():
+    parser = OptionParser()
+    parser.add_option("-r", "--repo", dest="repo",
+                      help="github repo", default="emqx/emqx")
+    parser.add_option("-t", "--token", dest="gh_token",
+                      help="github API token")
+    parser.add_option("-b", "--branch", dest="branch", default='master',
+                      help="Branch that workflow runs on")
+    (options, args) = parser.parse_args()
+
+    # Get gh token from env var GITHUB_TOKEN if provided, else use the one from command line
+    token = os.environ['GITHUB_TOKEN'] if 'GITHUB_TOKEN' in os.environ else options.gh_token
+
+    target_commit = fetch_latest_commit(token, options.repo, options.branch)
+
+    failed_checks = fetch_check_runs(token, options.repo, target_commit['sha'])
+
+    trigger_build(failed_checks, options.repo, token)
+
+
+if __name__ == '__main__':
+    main()

From 97e7c439f515239b9edc1d2735d328af65d0e51b Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 21 Mar 2023 10:49:18 +0100
Subject: [PATCH 17/52] ci(gree-master): handle pagination

---
 scripts/rerun-failed-checks.py | 59 +++++++++++++++++++++-------------
 1 file changed, 37 insertions(+), 22 deletions(-)

diff --git a/scripts/rerun-failed-checks.py b/scripts/rerun-failed-checks.py
index 167d8e81c..ff9b9f33e 100644
--- a/scripts/rerun-failed-checks.py
+++ b/scripts/rerun-failed-checks.py
@@ -13,6 +13,7 @@ import json
 import os
 import sys
 import time
+import math
 from optparse import OptionParser
 
 job_black_list = [
@@ -43,28 +44,42 @@ fetch check runs of a commit.
 @note, only works for public repos
 '''
 def fetch_check_runs(token: str, repo: str, ref: str):
+    all_checks = []
+    page = 1
+    total_pages = 1
+    per_page = 100
     failed_checks = []
-    url = f'https://api.github.com/repos/{repo}/commits/{ref}/check-runs?per_page=100'
-    headers = {'Accept': 'application/vnd.github.v3+json',
-               'Authorization': f'Bearer {token}'
-               }
-    r = requests.get(url, headers=headers)
-    if r.status_code == 200:
-        for crun in r.json()['check_runs']:
-            if crun['status'] == 'completed' and crun['conclusion'] != 'success':
-                print('Failed check: ', crun['name'])
-                failed_checks.append(
-                    {'id': crun['id'], 'name': crun['name'], 'url': crun['url']})
-            else:
-                # pretty print crun
-                # print(json.dumps(crun, indent=4))
-                print('successed:', crun['id'], crun['name'],
-                      crun['status'], crun['conclusion'])
-    else:
-        print(f'Failed to fetch check runs {r.status_code}')
-        sys.exit(1)
-    return failed_checks
+    while page <= total_pages:
+        print(f'Fetching check runs for page {page} of {total_pages} pages')
+        url = f'https://api.github.com/repos/{repo}/commits/{ref}/check-runs?per_page={per_page}&page={page}'
+        headers = {'Accept': 'application/vnd.github.v3+json',
+                    'Authorization': f'Bearer {token}'
+                  }
+        r = requests.get(url, headers=headers)
+        if r.status_code == 200:
+            resp = r.json()
+            all_checks.extend(resp['check_runs'])
 
+            page += 1
+            if 'total_count' in resp and resp['total_count'] > per_page:
+                total_pages = math.ceil(resp['total_count'] / per_page)
+        else:
+            print(f'Failed to fetch check runs {r.status_code}')
+            sys.exit(1)
+
+
+    for crun in all_checks:
+        if crun['status'] == 'completed' and crun['conclusion'] != 'success':
+            print('Failed check: ', crun['name'])
+            failed_checks.append(
+                {'id': crun['id'], 'name': crun['name'], 'url': crun['url']})
+        else:
+            # pretty print crun
+            # print(json.dumps(crun, indent=4))
+            print('successed:', crun['id'], crun['name'],
+                    crun['status'], crun['conclusion'])
+
+    return failed_checks
 
 '''
 rerquest a check-run
@@ -72,7 +87,7 @@ rerquest a check-run
 def trigger_build(failed_checks: list, repo: str, token: str):
     reruns = []
     for crun in failed_checks:
-        if crun['name'] in job_black_list:
+        if crun['name'].strip() in job_black_list:
             print(f'Skip black listed job {crun["name"]}')
             continue
 
@@ -101,7 +116,7 @@ def trigger_build(failed_checks: list, repo: str, token: str):
         else:
             # Only complain but not exit.
             print(
-                f'Failed to trigger build for {crun["name"]} : {r.status_code} : {r.text}')
+                f'Failed to trigger rerun for {run_id}, {crun["name"]}: {r.status_code} : {r.text}')
 
 
 def main():

From 53825b9abace7225d52458c9e0c04aaa4b934ff9 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 15 Mar 2023 14:50:40 +0100
Subject: [PATCH 18/52] fix(emqx_bridge): propagate connection error to
 resource status

---
 apps/emqx_bridge/i18n/emqx_bridge_schema.conf |  11 ++
 apps/emqx_bridge/src/emqx_bridge_api.erl      | 102 +++++++++--------
 .../src/schema/emqx_bridge_schema.erl         |  14 ++-
 .../test/emqx_bridge_api_SUITE.erl            | 107 +++++++++++-------
 apps/emqx_resource/include/emqx_resource.hrl  |   1 +
 .../src/emqx_resource_manager.erl             |   1 +
 changes/ce/fix-10145.en.md                    |   1 +
 7 files changed, 152 insertions(+), 85 deletions(-)
 create mode 100644 changes/ce/fix-10145.en.md

diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
index 901f25455..de4ceb0d5 100644
--- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
+++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf
@@ -54,6 +54,17 @@ emqx_bridge_schema {
                           }
                   }
 
+    desc_status_reason {
+                   desc {
+                        en: "This is the reason given in case a bridge is failing to connect."
+                        zh: "桥接连接失败的原因。"
+                   }
+                   label: {
+                          en: "Failure reason"
+                          zh: "失败原因"
+                   }
+    }
+
     desc_node_status {
                    desc {
                          en: """The status of the bridge for each node.
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 8ac3e476a..c0e69ce83 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -748,7 +748,7 @@ pick_bridges_by_id(Type, Name, BridgesAllNodes) ->
 
 format_bridge_info_with_metrics([FirstBridge | _] = Bridges) ->
     Res = maps:remove(node, FirstBridge),
-    NodeStatus = collect_status(Bridges),
+    NodeStatus = node_status(Bridges),
     NodeMetrics = collect_metrics(Bridges),
     redact(Res#{
         status => aggregate_status(NodeStatus),
@@ -765,8 +765,8 @@ format_bridge_metrics(Bridges) ->
     Res = format_bridge_info_with_metrics(Bridges),
     maps:with([metrics, node_metrics], Res).
 
-collect_status(Bridges) ->
-    [maps:with([node, status], B) || B <- Bridges].
+node_status(Bridges) ->
+    [maps:with([node, status, status_reason], B) || B <- Bridges].
 
 aggregate_status(AllStatus) ->
     Head = fun([A | _]) -> A end,
@@ -837,52 +837,63 @@ format_resource(
         )
     ).
 
-format_resource_data(#{status := Status, metrics := Metrics}) ->
-    #{status => Status, metrics => format_metrics(Metrics)};
-format_resource_data(#{status := Status}) ->
-    #{status => Status}.
+format_resource_data(ResData) ->
+    maps:fold(fun format_resource_data/3, #{}, maps:with([status, metrics, error], ResData)).
 
-format_metrics(#{
-    counters := #{
-        'dropped' := Dropped,
-        'dropped.other' := DroppedOther,
-        'dropped.expired' := DroppedExpired,
-        'dropped.queue_full' := DroppedQueueFull,
-        'dropped.resource_not_found' := DroppedResourceNotFound,
-        'dropped.resource_stopped' := DroppedResourceStopped,
-        'matched' := Matched,
-        'retried' := Retried,
-        'late_reply' := LateReply,
-        'failed' := SentFailed,
-        'success' := SentSucc,
-        'received' := Rcvd
+format_resource_data(error, undefined, Result) ->
+    Result;
+format_resource_data(error, Error, Result) ->
+    Result#{status_reason => emqx_misc:readable_error_msg(Error)};
+format_resource_data(
+    metrics,
+    #{
+        counters := #{
+            'dropped' := Dropped,
+            'dropped.other' := DroppedOther,
+            'dropped.expired' := DroppedExpired,
+            'dropped.queue_full' := DroppedQueueFull,
+            'dropped.resource_not_found' := DroppedResourceNotFound,
+            'dropped.resource_stopped' := DroppedResourceStopped,
+            'matched' := Matched,
+            'retried' := Retried,
+            'late_reply' := LateReply,
+            'failed' := SentFailed,
+            'success' := SentSucc,
+            'received' := Rcvd
+        },
+        gauges := Gauges,
+        rate := #{
+            matched := #{current := Rate, last5m := Rate5m, max := RateMax}
+        }
     },
-    gauges := Gauges,
-    rate := #{
-        matched := #{current := Rate, last5m := Rate5m, max := RateMax}
-    }
-}) ->
+    Result
+) ->
     Queued = maps:get('queuing', Gauges, 0),
     SentInflight = maps:get('inflight', Gauges, 0),
-    ?METRICS(
-        Dropped,
-        DroppedOther,
-        DroppedExpired,
-        DroppedQueueFull,
-        DroppedResourceNotFound,
-        DroppedResourceStopped,
-        Matched,
-        Queued,
-        Retried,
-        LateReply,
-        SentFailed,
-        SentInflight,
-        SentSucc,
-        Rate,
-        Rate5m,
-        RateMax,
-        Rcvd
-    ).
+    Result#{
+        metrics =>
+            ?METRICS(
+                Dropped,
+                DroppedOther,
+                DroppedExpired,
+                DroppedQueueFull,
+                DroppedResourceNotFound,
+                DroppedResourceStopped,
+                Matched,
+                Queued,
+                Retried,
+                LateReply,
+                SentFailed,
+                SentInflight,
+                SentSucc,
+                Rate,
+                Rate5m,
+                RateMax,
+                Rcvd
+            )
+    };
+format_resource_data(K, V, Result) ->
+    Result#{K => V}.
 
 fill_defaults(Type, RawConf) ->
     PackedConf = pack_bridge_conf(Type, RawConf),
@@ -924,6 +935,7 @@ filter_out_request_body(Conf) ->
         <<"type">>,
         <<"name">>,
         <<"status">>,
+        <<"error">>,
         <<"node_status">>,
         <<"node_metrics">>,
         <<"metrics">>,
diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
index 74d2a5ca1..6c278a5ec 100644
--- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
+++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl
@@ -106,6 +106,12 @@ common_bridge_fields() ->
 status_fields() ->
     [
         {"status", mk(status(), #{desc => ?DESC("desc_status")})},
+        {"status_reason",
+            mk(binary(), #{
+                required => false,
+                desc => ?DESC("desc_status_reason"),
+                example => <<"Connection refused">>
+            })},
         {"node_status",
             mk(
                 hoconsc:array(ref(?MODULE, "node_status")),
@@ -190,7 +196,13 @@ fields("node_metrics") ->
 fields("node_status") ->
     [
         node_name(),
-        {"status", mk(status(), #{})}
+        {"status", mk(status(), #{})},
+        {"status_reason",
+            mk(binary(), #{
+                required => false,
+                desc => ?DESC("desc_status_reason"),
+                example => <<"Connection refused">>
+            })}
     ].
 
 desc(bridges) ->
diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
index 8b388a771..b919dba6b 100644
--- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
@@ -23,7 +23,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -define(CONF_DEFAULT, <<"bridges: {}">>).
--define(BRIDGE_TYPE, <<"webhook">>).
+-define(BRIDGE_TYPE_HTTP, <<"webhook">>).
 -define(BRIDGE_NAME, (atom_to_binary(?FUNCTION_NAME))).
 -define(URL(PORT, PATH),
     list_to_binary(
@@ -48,7 +48,7 @@
 }).
 -define(MQTT_BRIDGE(SERVER), ?MQTT_BRIDGE(SERVER, <<"mqtt_egress_test_bridge">>)).
 
--define(HTTP_BRIDGE(URL, TYPE, NAME), ?BRIDGE(NAME, TYPE)#{
+-define(HTTP_BRIDGE(URL, NAME), ?BRIDGE(NAME, ?BRIDGE_TYPE_HTTP)#{
     <<"url">> => URL,
     <<"local_topic">> => <<"emqx_webhook/#">>,
     <<"method">> => <<"post">>,
@@ -57,6 +57,7 @@
         <<"content-type">> => <<"application/json">>
     }
 }).
+-define(HTTP_BRIDGE(URL), ?HTTP_BRIDGE(URL, ?BRIDGE_NAME)).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -206,12 +207,12 @@ t_http_crud_apis(Config) ->
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
 
     %ct:pal("---bridge: ~p", [Bridge]),
     #{
-        <<"type">> := ?BRIDGE_TYPE,
+        <<"type">> := ?BRIDGE_TYPE_HTTP,
         <<"name">> := Name,
         <<"enable">> := true,
         <<"status">> := _,
@@ -219,7 +220,7 @@ t_http_crud_apis(Config) ->
         <<"url">> := URL1
     } = emqx_json:decode(Bridge, [return_maps]),
 
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
     %% send an message to emqx and the message should be forwarded to the HTTP server
     Body = <<"my msg">>,
     emqx:publish(emqx_message:make(<<"emqx_webhook/1">>, Body)),
@@ -243,11 +244,11 @@ t_http_crud_apis(Config) ->
     {ok, 200, Bridge2} = request(
         put,
         uri(["bridges", BridgeID]),
-        ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL2, Name)
     ),
     ?assertMatch(
         #{
-            <<"type">> := ?BRIDGE_TYPE,
+            <<"type">> := ?BRIDGE_TYPE_HTTP,
             <<"name">> := Name,
             <<"enable">> := true,
             <<"status">> := _,
@@ -262,7 +263,7 @@ t_http_crud_apis(Config) ->
     ?assertMatch(
         [
             #{
-                <<"type">> := ?BRIDGE_TYPE,
+                <<"type">> := ?BRIDGE_TYPE_HTTP,
                 <<"name">> := Name,
                 <<"enable">> := true,
                 <<"status">> := _,
@@ -279,7 +280,7 @@ t_http_crud_apis(Config) ->
     {ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID]), []),
     ?assertMatch(
         #{
-            <<"type">> := ?BRIDGE_TYPE,
+            <<"type">> := ?BRIDGE_TYPE_HTTP,
             <<"name">> := Name,
             <<"enable">> := true,
             <<"status">> := _,
@@ -311,7 +312,7 @@ t_http_crud_apis(Config) ->
     {ok, 404, ErrMsg2} = request(
         put,
         uri(["bridges", BridgeID]),
-        ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL2, Name)
     ),
     ?assertMatch(
         #{
@@ -340,6 +341,34 @@ t_http_crud_apis(Config) ->
         },
         emqx_json:decode(ErrMsg3, [return_maps])
     ),
+
+    %% Create non working bridge
+    BrokenURL = ?URL(Port + 1, "/foo"),
+    {ok, 201, BrokenBridge} = request(
+        post,
+        uri(["bridges"]),
+        ?HTTP_BRIDGE(BrokenURL, Name)
+    ),
+    #{
+        <<"type">> := ?BRIDGE_TYPE_HTTP,
+        <<"name">> := Name,
+        <<"enable">> := true,
+        <<"status">> := <<"disconnected">>,
+        <<"status_reason">> := <<"Connection refused">>,
+        <<"node_status">> := [
+            #{<<"status">> := <<"disconnected">>, <<"status_reason">> := <<"Connection refused">>}
+            | _
+        ],
+        <<"url">> := BrokenURL
+    } = emqx_json:decode(BrokenBridge, [return_maps]),
+    {ok, 200, FixedBridgeResponse} = request(put, uri(["bridges", BridgeID]), ?HTTP_BRIDGE(URL1)),
+    #{
+        <<"status">> := <<"connected">>,
+        <<"node_status">> := [FixedNodeStatus = #{<<"status">> := <<"connected">>} | _]
+    } = FixedBridge = emqx_json:decode(FixedBridgeResponse, [return_maps]),
+    ?assert(not maps:is_key(<<"status_reason">>, FixedBridge)),
+    ?assert(not maps:is_key(<<"status_reason">>, FixedNodeStatus)),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
     ok.
 
 t_http_bridges_local_topic(Config) ->
@@ -356,16 +385,16 @@ t_http_bridges_local_topic(Config) ->
     {ok, 201, _} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name1)
+        ?HTTP_BRIDGE(URL1, Name1)
     ),
     %% and we create another one without local_topic
     {ok, 201, _} = request(
         post,
         uri(["bridges"]),
-        maps:remove(<<"local_topic">>, ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name2))
+        maps:remove(<<"local_topic">>, ?HTTP_BRIDGE(URL1, Name2))
     ),
-    BridgeID1 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name1),
-    BridgeID2 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name2),
+    BridgeID1 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name1),
+    BridgeID2 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name2),
     %% Send an message to emqx and the message should be forwarded to the HTTP server.
     %% This is to verify we can have 2 bridges with and without local_topic fields
     %% at the same time.
@@ -400,11 +429,11 @@ t_check_dependent_actions_on_delete(Config) ->
     %% POST /bridges/ will create a bridge
     URL1 = ?URL(Port, "path1"),
     Name = <<"t_http_crud_apis">>,
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
     {ok, 201, _} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
     {ok, 201, Rule} = request(
         post,
@@ -438,11 +467,11 @@ t_cascade_delete_actions(Config) ->
     %% POST /bridges/ will create a bridge
     URL1 = ?URL(Port, "path1"),
     Name = <<"t_http_crud_apis">>,
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
     {ok, 201, _} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
     {ok, 201, Rule} = request(
         post,
@@ -472,7 +501,7 @@ t_cascade_delete_actions(Config) ->
     {ok, 201, _} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
     {ok, 201, _} = request(
         post,
@@ -496,9 +525,9 @@ t_broken_bpapi_vsn(Config) ->
     {ok, 201, _Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
     %% still works since we redirect to 'restart'
     {ok, 501, <<>>} = request(post, operation_path(cluster, start, BridgeID), <<"">>),
     {ok, 501, <<>>} = request(post, operation_path(node, start, BridgeID), <<"">>),
@@ -511,9 +540,9 @@ t_old_bpapi_vsn(Config) ->
     {ok, 201, _Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
     {ok, 204, <<>>} = request(post, operation_path(cluster, stop, BridgeID), <<"">>),
     {ok, 204, <<>>} = request(post, operation_path(node, stop, BridgeID), <<"">>),
     %% still works since we redirect to 'restart'
@@ -551,18 +580,18 @@ do_start_stop_bridges(Type, Config) ->
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
     %ct:pal("the bridge ==== ~p", [Bridge]),
     #{
-        <<"type">> := ?BRIDGE_TYPE,
+        <<"type">> := ?BRIDGE_TYPE_HTTP,
         <<"name">> := Name,
         <<"enable">> := true,
         <<"status">> := <<"connected">>,
         <<"node_status">> := [_ | _],
         <<"url">> := URL1
     } = emqx_json:decode(Bridge, [return_maps]),
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
     %% stop it
     {ok, 204, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>),
     {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []),
@@ -633,18 +662,18 @@ t_enable_disable_bridges(Config) ->
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
     %ct:pal("the bridge ==== ~p", [Bridge]),
     #{
-        <<"type">> := ?BRIDGE_TYPE,
+        <<"type">> := ?BRIDGE_TYPE_HTTP,
         <<"name">> := Name,
         <<"enable">> := true,
         <<"status">> := <<"connected">>,
         <<"node_status">> := [_ | _],
         <<"url">> := URL1
     } = emqx_json:decode(Bridge, [return_maps]),
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
     %% disable it
     {ok, 204, <<>>} = request(put, enable_path(false, BridgeID), <<"">>),
     {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []),
@@ -690,18 +719,18 @@ t_reset_bridges(Config) ->
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
     %ct:pal("the bridge ==== ~p", [Bridge]),
     #{
-        <<"type">> := ?BRIDGE_TYPE,
+        <<"type">> := ?BRIDGE_TYPE_HTTP,
         <<"name">> := Name,
         <<"enable">> := true,
         <<"status">> := <<"connected">>,
         <<"node_status">> := [_ | _],
         <<"url">> := URL1
     } = emqx_json:decode(Bridge, [return_maps]),
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
     {ok, 204, <<>>} = request(put, uri(["bridges", BridgeID, "metrics/reset"]), []),
 
     %% delete the bridge
@@ -748,20 +777,20 @@ t_bridges_probe(Config) ->
     {ok, 204, <<>>} = request(
         post,
         uri(["bridges_probe"]),
-        ?HTTP_BRIDGE(URL, ?BRIDGE_TYPE, ?BRIDGE_NAME)
+        ?HTTP_BRIDGE(URL)
     ),
 
     %% second time with same name is ok since no real bridge created
     {ok, 204, <<>>} = request(
         post,
         uri(["bridges_probe"]),
-        ?HTTP_BRIDGE(URL, ?BRIDGE_TYPE, ?BRIDGE_NAME)
+        ?HTTP_BRIDGE(URL)
     ),
 
     {ok, 400, NxDomain} = request(
         post,
         uri(["bridges_probe"]),
-        ?HTTP_BRIDGE(<<"http://203.0.113.3:1234/foo">>, ?BRIDGE_TYPE, ?BRIDGE_NAME)
+        ?HTTP_BRIDGE(<<"http://203.0.113.3:1234/foo">>)
     ),
     ?assertMatch(
         #{
@@ -882,12 +911,12 @@ t_metrics(Config) ->
     {ok, 201, Bridge} = request(
         post,
         uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name)
+        ?HTTP_BRIDGE(URL1, Name)
     ),
 
     %ct:pal("---bridge: ~p", [Bridge]),
     #{
-        <<"type">> := ?BRIDGE_TYPE,
+        <<"type">> := ?BRIDGE_TYPE_HTTP,
         <<"name">> := Name,
         <<"enable">> := true,
         <<"status">> := _,
@@ -895,7 +924,7 @@ t_metrics(Config) ->
         <<"url">> := URL1
     } = emqx_json:decode(Bridge, [return_maps]),
 
-    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
+    BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name),
 
     %% check for empty bridge metrics
     {ok, 200, Bridge1Str} = request(get, uri(["bridges", BridgeID, "metrics"]), []),
@@ -963,7 +992,7 @@ t_inconsistent_webhook_request_timeouts(Config) ->
     Name = ?BRIDGE_NAME,
     BadBridgeParams =
         emqx_map_lib:deep_merge(
-            ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name),
+            ?HTTP_BRIDGE(URL1, Name),
             #{
                 <<"request_timeout">> => <<"1s">>,
                 <<"resource_opts">> => #{<<"request_timeout">> => <<"2s">>}
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 41be9e8a0..ae22e27e0 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -41,6 +41,7 @@
     callback_mode := callback_mode(),
     query_mode := query_mode(),
     config := resource_config(),
+    error := term(),
     state := resource_state(),
     status := resource_status(),
     metrics => emqx_metrics_worker:metrics()
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 40f9fe1ab..1bc6a3b6a 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -666,6 +666,7 @@ maybe_reply(Actions, From, Reply) ->
 data_record_to_external_map(Data) ->
     #{
         id => Data#data.id,
+        error => Data#data.error,
         mod => Data#data.mod,
         callback_mode => Data#data.callback_mode,
         query_mode => Data#data.query_mode,
diff --git a/changes/ce/fix-10145.en.md b/changes/ce/fix-10145.en.md
new file mode 100644
index 000000000..bddbd9085
--- /dev/null
+++ b/changes/ce/fix-10145.en.md
@@ -0,0 +1 @@
+Fix `bridges` API to report error conditions for a failing bridge as `status_reason`.

From c1384b6e6ed805985e93b13d5083a974dbc55fa8 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 15 Mar 2023 14:52:03 +0100
Subject: [PATCH 19/52] feat(emqx_resource): include error with alarm for
 resource_down

---
 .../emqx_resource/src/emqx_resource_manager.erl | 17 +++++++++++------
 changes/ce/fix-10145.en.md                      |  4 +++-
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 1bc6a3b6a..b21ffcae3 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -522,7 +522,7 @@ start_resource(Data, From) ->
                 id => Data#data.id,
                 reason => Reason
             }),
-            _ = maybe_alarm(disconnected, Data#data.id),
+            _ = maybe_alarm(disconnected, Data#data.id, Data#data.error),
             %% Keep track of the error reason why the connection did not work
             %% so that the Reason can be returned when the verification call is made.
             UpdatedData = Data#data{status = disconnected, error = Reason},
@@ -597,7 +597,7 @@ with_health_check(Data, Func) ->
     ResId = Data#data.id,
     HCRes = emqx_resource:call_health_check(Data#data.manager_id, Data#data.mod, Data#data.state),
     {Status, NewState, Err} = parse_health_check_result(HCRes, Data),
-    _ = maybe_alarm(Status, ResId),
+    _ = maybe_alarm(Status, ResId, Err),
     ok = maybe_resume_resource_workers(ResId, Status),
     UpdatedData = Data#data{
         state = NewState, status = Status, error = Err
@@ -616,15 +616,20 @@ update_state(Data, _DataWas) ->
 health_check_interval(Opts) ->
     maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL).
 
-maybe_alarm(connected, _ResId) ->
+maybe_alarm(connected, _ResId, _Error) ->
     ok;
-maybe_alarm(_Status, <<?TEST_ID_PREFIX, _/binary>>) ->
+maybe_alarm(_Status, <<?TEST_ID_PREFIX, _/binary>>, _Error) ->
     ok;
-maybe_alarm(_Status, ResId) ->
+maybe_alarm(_Status, ResId, Error) ->
+    HrError =
+        case Error of
+            undefined -> <<"Unknown reason">>;
+            _Else -> emqx_misc:readable_error_msg(Error)
+        end,
     emqx_alarm:activate(
         ResId,
         #{resource_id => ResId, reason => resource_down},
-        <<"resource down: ", ResId/binary>>
+        <<"resource down: ", HrError/binary>>
     ).
 
 maybe_resume_resource_workers(ResId, connected) ->
diff --git a/changes/ce/fix-10145.en.md b/changes/ce/fix-10145.en.md
index bddbd9085..eaa896793 100644
--- a/changes/ce/fix-10145.en.md
+++ b/changes/ce/fix-10145.en.md
@@ -1 +1,3 @@
-Fix `bridges` API to report error conditions for a failing bridge as `status_reason`.
+Fix `bridges` API to report error conditions for a failing bridge as
+`status_reason`. Also when creating an alarm for a failing resource we include
+this error condition with the alarm's message.

From 8af3fb4ee7536170159001eb8282db69e96d15b4 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 15 Mar 2023 14:53:18 +0100
Subject: [PATCH 20/52] feat: move human readable error translations to
 emqx_misc

---
 apps/emqx/src/emqx_misc.erl              | 15 ++++++++++++++-
 apps/emqx_bridge/src/emqx_bridge_api.erl | 17 ++---------------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl
index 18ecc644a..59b5b1be9 100644
--- a/apps/emqx/src/emqx_misc.erl
+++ b/apps/emqx/src/emqx_misc.erl
@@ -545,10 +545,23 @@ readable_error_msg(Error) ->
                 {ok, Msg} ->
                     Msg;
                 false ->
-                    iolist_to_binary(io_lib:format("~0p", [Error]))
+                    to_hr_error(Error)
             end
     end.
 
+to_hr_error(nxdomain) ->
+    <<"Host not found">>;
+to_hr_error(econnrefused) ->
+    <<"Connection refused">>;
+to_hr_error({unauthorized_client, _}) ->
+    <<"Unauthorized client">>;
+to_hr_error({not_authorized, _}) ->
+    <<"Not authorized">>;
+to_hr_error({malformed_username_or_password, _}) ->
+    <<"Malformed username or password">>;
+to_hr_error(Error) ->
+    iolist_to_binary(io_lib:format("~0p", [Error])).
+
 try_to_existing_atom(Convert, Data, Encoding) ->
     try Convert(Data, Encoding) of
         Atom ->
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index c0e69ce83..e466b418d 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -568,7 +568,7 @@ schema("/bridges_probe") ->
                 ok ->
                     204;
                 {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' ->
-                    {400, error_msg('TEST_FAILED', to_hr_reason(Reason))}
+                    {400, error_msg('TEST_FAILED', emqx_misc:readable_error_msg(Reason))}
             end;
         BadRequest ->
             BadRequest
@@ -979,7 +979,7 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) ->
         {error, {node_not_found, Node}} ->
             ?NOT_FOUND(<<"Node not found: ", (atom_to_binary(Node))/binary>>);
         {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' ->
-            ?BAD_REQUEST(to_hr_reason(Reason))
+            ?BAD_REQUEST(emqx_misc:readable_error_msg(Reason))
     end.
 
 maybe_try_restart(all, start_bridges_to_all_nodes, Args) ->
@@ -1018,19 +1018,6 @@ supported_versions(start_bridge_to_node) -> [2, 3];
 supported_versions(start_bridges_to_all_nodes) -> [2, 3];
 supported_versions(_Call) -> [1, 2, 3].
 
-to_hr_reason(nxdomain) ->
-    <<"Host not found">>;
-to_hr_reason(econnrefused) ->
-    <<"Connection refused">>;
-to_hr_reason({unauthorized_client, _}) ->
-    <<"Unauthorized client">>;
-to_hr_reason({not_authorized, _}) ->
-    <<"Not authorized">>;
-to_hr_reason({malformed_username_or_password, _}) ->
-    <<"Malformed username or password">>;
-to_hr_reason(Reason) ->
-    Reason.
-
 redact(Term) ->
     emqx_misc:redact(Term).
 

From 84fc64822ef532ac51c53eeac0cff08195e883e4 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan@strigler.de>
Date: Tue, 21 Mar 2023 10:29:24 +0100
Subject: [PATCH 21/52] style: fix wording for nxdomain and
 malformed_username_or_password

Co-authored-by: Zaiming (Stone) Shi <zmstone@gmail.com>
---
 apps/emqx/src/emqx_misc.erl                     | 4 ++--
 apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl
index 59b5b1be9..cdd62df11 100644
--- a/apps/emqx/src/emqx_misc.erl
+++ b/apps/emqx/src/emqx_misc.erl
@@ -550,7 +550,7 @@ readable_error_msg(Error) ->
     end.
 
 to_hr_error(nxdomain) ->
-    <<"Host not found">>;
+    <<"Could not resolve host">>;
 to_hr_error(econnrefused) ->
     <<"Connection refused">>;
 to_hr_error({unauthorized_client, _}) ->
@@ -558,7 +558,7 @@ to_hr_error({unauthorized_client, _}) ->
 to_hr_error({not_authorized, _}) ->
     <<"Not authorized">>;
 to_hr_error({malformed_username_or_password, _}) ->
-    <<"Malformed username or password">>;
+    <<"Bad username or password">>;
 to_hr_error(Error) ->
     iolist_to_binary(io_lib:format("~0p", [Error])).
 
diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
index b919dba6b..986c0d29d 100644
--- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
@@ -819,7 +819,7 @@ t_bridges_probe(Config) ->
         emqx_json:decode(ConnRefused, [return_maps])
     ),
 
-    {ok, 400, HostNotFound} = request(
+    {ok, 400, CouldNotResolveHost} = request(
         post,
         uri(["bridges_probe"]),
         ?MQTT_BRIDGE(<<"nohost:2883">>)
@@ -827,9 +827,9 @@ t_bridges_probe(Config) ->
     ?assertMatch(
         #{
             <<"code">> := <<"TEST_FAILED">>,
-            <<"message">> := <<"Host not found">>
+            <<"message">> := <<"Could not resolve host">>
         },
-        emqx_json:decode(HostNotFound, [return_maps])
+        emqx_json:decode(CouldNotResolveHost, [return_maps])
     ),
 
     AuthnConfig = #{
@@ -873,7 +873,7 @@ t_bridges_probe(Config) ->
     ?assertMatch(
         #{
             <<"code">> := <<"TEST_FAILED">>,
-            <<"message">> := <<"Malformed username or password">>
+            <<"message">> := <<"Bad username or password">>
         },
         emqx_json:decode(Malformed, [return_maps])
     ),

From 4b0ea562a28e0ac8ae874998fd8b12c40855b04c Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Mar 2023 14:21:30 +0100
Subject: [PATCH 22/52] refactor(emqx_bridge): consistently use macros for http
 response

---
 apps/emqx_bridge/src/emqx_bridge_api.erl | 79 +++++++++++++-----------
 1 file changed, 42 insertions(+), 37 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index e466b418d..917b44096 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -46,18 +46,33 @@
 
 -export([lookup_from_local_node/2]).
 
--define(BAD_REQUEST(Reason), {400, error_msg('BAD_REQUEST', Reason)}).
+%% [TODO] Move those to a commonly shared header file
+-define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_misc:readable_error_msg(REASON)}).
+
+-define(OK(CONTENT), {200, CONTENT}).
+
+-define(NO_CONTENT, 204).
+
+-define(BAD_REQUEST(CODE, REASON), {400, ?ERROR_MSG(CODE, REASON)}).
+-define(BAD_REQUEST(REASON), ?BAD_REQUEST('BAD_REQUEST', REASON)).
+
+-define(NOT_FOUND(REASON), {404, ?ERROR_MSG('NOT_FOUND', REASON)}).
+
+-define(INTERNAL_ERROR(REASON), {500, ?ERROR_MSG('INTERNAL_ERROR', REASON)}).
+
+-define(NOT_IMPLEMENTED, 501).
+
+-define(SERVICE_UNAVAILABLE(REASON), {503, ?ERROR_MSG('SERVICE_UNAVAILABLE', REASON)}).
+%% End TODO
 
 -define(BRIDGE_NOT_ENABLED,
     ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>)
 ).
 
--define(NOT_FOUND(Reason), {404, error_msg('NOT_FOUND', Reason)}).
-
--define(BRIDGE_NOT_FOUND(BridgeType, BridgeName),
+-define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME),
     ?NOT_FOUND(
-        <<"Bridge lookup failed: bridge named '", BridgeName/binary, "' of type ",
-            (atom_to_binary(BridgeType))/binary, " does not exist.">>
+        <<"Bridge lookup failed: bridge named '", BRIDGE_NAME/binary, "' of type ",
+            (atom_to_binary(BRIDGE_TYPE))/binary, " does not exist.">>
     )
 ).
 
@@ -480,7 +495,7 @@ schema("/bridges_probe") ->
 '/bridges'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) ->
     case emqx_bridge:lookup(BridgeType, BridgeName) of
         {ok, _} ->
-            {400, error_msg('ALREADY_EXISTS', <<"bridge already exists">>)};
+            ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>);
         {error, not_found} ->
             Conf = filter_out_request_body(Conf0),
             {ok, _} = emqx_bridge:create(BridgeType, BridgeName, Conf),
@@ -495,9 +510,9 @@ schema("/bridges_probe") ->
                 format_resource(Data, Node)
              || {Node, Bridges} <- lists:zip(Nodes, NodeBridges), Data <- Bridges
             ],
-            {200, zip_bridges([AllBridges])};
+            ?OK(zip_bridges([AllBridges]));
         {error, Reason} ->
-            {500, error_msg('INTERNAL_ERROR', Reason)}
+            ?INTERNAL_ERROR(Reason)
     end.
 
 '/bridges/:id'(get, #{bindings := #{id := Id}}) ->
@@ -529,16 +544,16 @@ schema("/bridges_probe") ->
                     end,
                 case emqx_bridge:check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActs) of
                     {ok, _} ->
-                        204;
+                        ?NO_CONTENT;
                     {error, {rules_deps_on_this_bridge, RuleIds}} ->
                         ?BAD_REQUEST(
                             {<<"Cannot delete bridge while active rules are defined for this bridge">>,
                                 RuleIds}
                         );
                     {error, timeout} ->
-                        {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+                        ?SERVICE_UNAVAILABLE(<<"request timeout">>);
                     {error, Reason} ->
-                        {500, error_msg('INTERNAL_ERROR', Reason)}
+                        ?INTERNAL_ERROR(Reason)
                 end;
             {error, not_found} ->
                 ?BRIDGE_NOT_FOUND(BridgeType, BridgeName)
@@ -555,7 +570,7 @@ schema("/bridges_probe") ->
             ok = emqx_bridge_resource:reset_metrics(
                 emqx_bridge_resource:resource_id(BridgeType, BridgeName)
             ),
-            {204}
+            ?NO_CONTENT
         end
     ).
 
@@ -566,9 +581,9 @@ schema("/bridges_probe") ->
             Params1 = maybe_deobfuscate_bridge_probe(Params),
             case emqx_bridge_resource:create_dry_run(ConnType, maps:remove(<<"type">>, Params1)) of
                 ok ->
-                    204;
+                    ?NO_CONTENT;
                 {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' ->
-                    {400, error_msg('TEST_FAILED', emqx_misc:readable_error_msg(Reason))}
+                    ?BAD_REQUEST('TEST_FAILED', Reason)
             end;
         BadRequest ->
             BadRequest
@@ -602,7 +617,7 @@ do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun) ->
         {ok, [{error, not_found} | _]} ->
             ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
         {error, Reason} ->
-            {500, error_msg('INTERNAL_ERROR', Reason)}
+            ?INTERNAL_ERROR(Reason)
     end.
 
 lookup_from_local_node(BridgeType, BridgeName) ->
@@ -620,15 +635,15 @@ lookup_from_local_node(BridgeType, BridgeName) ->
             OperFunc ->
                 case emqx_bridge:disable_enable(OperFunc, BridgeType, BridgeName) of
                     {ok, _} ->
-                        204;
+                        ?NO_CONTENT;
                     {error, {pre_config_update, _, bridge_not_found}} ->
                         ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
                     {error, {_, _, timeout}} ->
-                        {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+                        ?SERVICE_UNAVAILABLE(<<"request timeout">>);
                     {error, timeout} ->
-                        {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+                        ?SERVICE_UNAVAILABLE(<<"request timeout">>);
                     {error, Reason} ->
-                        {500, error_msg('INTERNAL_ERROR', Reason)}
+                        ?INTERNAL_ERROR(Reason)
                 end
         end
     ).
@@ -943,9 +958,6 @@ filter_out_request_body(Conf) ->
     ],
     maps:without(ExtraConfs, Conf).
 
-error_msg(Code, Msg) ->
-    #{code => Code, message => emqx_misc:readable_error_msg(Msg)}.
-
 bin(S) when is_list(S) ->
     list_to_binary(S);
 bin(S) when is_atom(S) ->
@@ -956,30 +968,23 @@ bin(S) when is_binary(S) ->
 call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) ->
     case is_ok(do_bpapi_call(NodeOrAll, OperFunc, Args)) of
         Ok when Ok =:= ok; is_tuple(Ok), element(1, Ok) =:= ok ->
-            204;
+            ?NO_CONTENT;
         {error, not_implemented} ->
             %% Should only happen if we call `start` on a node that is
             %% still on an older bpapi version that doesn't support it.
             maybe_try_restart(NodeOrAll, OperFunc, Args);
         {error, timeout} ->
-            {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+            ?SERVICE_UNAVAILABLE(<<"Request timeout">>);
         {error, {start_pool_failed, Name, Reason}} ->
-            {503,
-                error_msg(
-                    'SERVICE_UNAVAILABLE',
-                    bin(
-                        io_lib:format(
-                            "failed to start ~p pool for reason ~p",
-                            [Name, Reason]
-                        )
-                    )
-                )};
+            ?SERVICE_UNAVAILABLE(
+                bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason]))
+            );
         {error, not_found} ->
             ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
         {error, {node_not_found, Node}} ->
             ?NOT_FOUND(<<"Node not found: ", (atom_to_binary(Node))/binary>>);
         {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' ->
-            ?BAD_REQUEST(emqx_misc:readable_error_msg(Reason))
+            ?BAD_REQUEST(Reason)
     end.
 
 maybe_try_restart(all, start_bridges_to_all_nodes, Args) ->
@@ -987,7 +992,7 @@ maybe_try_restart(all, start_bridges_to_all_nodes, Args) ->
 maybe_try_restart(Node, start_bridge_to_node, Args) ->
     call_operation(Node, restart_bridge_to_node, Args);
 maybe_try_restart(_, _, _) ->
-    501.
+    ?NOT_IMPLEMENTED.
 
 do_bpapi_call(all, Call, Args) ->
     maybe_unwrap(

From 3880862c810b386b35ebcd83fe0c596cb10b3809 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Mar 2023 14:22:04 +0100
Subject: [PATCH 23/52] fix(emqx_bridge): return 503 for inconsistency in
 bridge setup

---
 apps/emqx_bridge/src/emqx_bridge_api.erl      | 10 ++++++-
 .../test/emqx_bridge_api_SUITE.erl            | 30 +++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 917b44096..f7a1bb345 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -980,7 +980,15 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) ->
                 bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason]))
             );
         {error, not_found} ->
-            ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
+            BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
+            ?SLOG(warning, #{
+                msg => "bridge_inconsistent_in_cluster_for_call_operation",
+                reason => not_found,
+                type => BridgeType,
+                name => BridgeName,
+                bridge => BridgeId
+            }),
+            ?SERVICE_UNAVAILABLE(<<"Bridge not found on remote node: ", BridgeId/binary>>);
         {error, {node_not_found, Node}} ->
             ?NOT_FOUND(<<"Node not found: ", (atom_to_binary(Node))/binary>>);
         {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' ->
diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
index 986c0d29d..45ab2b623 100644
--- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
@@ -98,6 +98,20 @@ init_per_testcase(t_old_bpapi_vsn, Config) ->
     meck:expect(emqx_bpapi, supported_version, 1, 1),
     meck:expect(emqx_bpapi, supported_version, 2, 1),
     init_per_testcase(common, Config);
+init_per_testcase(StartStop, Config) when
+    StartStop == t_start_stop_bridges_cluster;
+    StartStop == t_start_stop_bridges_node
+->
+    meck:new(emqx_bridge_resource, [passthrough]),
+    meck:expect(
+        emqx_bridge_resource,
+        stop,
+        fun
+            (_, <<"bridge_not_found">>) -> {error, not_found};
+            (Type, Name) -> meck:passthrough([Type, Name])
+        end
+    ),
+    init_per_testcase(common, Config);
 init_per_testcase(_, Config) ->
     {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
     {Port, Sock, Acceptor} = start_http_server(fun handle_fun_200_ok/2),
@@ -109,6 +123,12 @@ end_per_testcase(t_broken_bpapi_vsn, Config) ->
 end_per_testcase(t_old_bpapi_vsn, Config) ->
     meck:unload([emqx_bpapi]),
     end_per_testcase(common, Config);
+end_per_testcase(StartStop, Config) when
+    StartStop == t_start_stop_bridges_cluster;
+    StartStop == t_start_stop_bridges_node
+->
+    meck:unload([emqx_bridge_resource]),
+    end_per_testcase(common, Config);
 end_per_testcase(_, Config) ->
     Sock = ?config(sock, Config),
     Acceptor = ?config(acceptor, Config),
@@ -626,6 +646,16 @@ do_start_stop_bridges(Type, Config) ->
     %% Looks ok but doesn't exist
     {ok, 404, _} = request(post, operation_path(Type, start, <<"webhook:cptn_hook">>), <<"">>),
 
+    %%
+    {ok, 201, _Bridge} = request(
+        post,
+        uri(["bridges"]),
+        ?HTTP_BRIDGE(URL1, <<"bridge_not_found">>)
+    ),
+    {ok, 503, _} = request(
+        post, operation_path(Type, stop, <<"webhook:bridge_not_found">>), <<"">>
+    ),
+
     %% Create broken bridge
     {ListenPort, Sock} = listen_on_random_port(),
     %% Connecting to this endpoint should always timeout

From d2c5cbbcaaa3063358ca46bfd8d6eaf4f01e2ab3 Mon Sep 17 00:00:00 2001
From: Ingmar Delsink <ingmardelsink@hotmail.com>
Date: Tue, 21 Mar 2023 15:21:08 +0100
Subject: [PATCH 24/52] fix(helm): helm chart extraVolumeMounts and
 extraVolumes indentation

---
 .../emqx-enterprise/templates/StatefulSet.yaml       | 12 ++++++------
 deploy/charts/emqx/templates/StatefulSet.yaml        | 12 ++++++------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml
index 00751aceb..3e9e39f2c 100644
--- a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml
@@ -74,9 +74,9 @@ spec:
         secret:
           secretName: {{ .Values.emqxLicenseSecretName }}
       {{- end }}
-      {{- if .Values.extraVolumes }}
-        {{- toYaml .Values.extraVolumes | nindent 8 }}
-      {{- end }}
+    {{- if .Values.extraVolumes }}
+      {{- toYaml .Values.extraVolumes | nindent 6 }}
+    {{- end }}
       {{- if .Values.podSecurityContext.enabled }}
       securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
@@ -141,9 +141,9 @@ spec:
             subPath: "emqx.lic"
             readOnly: true
           {{- end }}
-          {{- if .Values.extraVolumeMounts }}
-            {{- toYaml .Values.extraVolumeMounts | nindent 12 }}
-          {{- end }}
+        {{- if .Values.extraVolumeMounts }}
+          {{- toYaml .Values.extraVolumeMounts | nindent 10 }}
+        {{- end }}
           readinessProbe:
             httpGet:
               path: /status
diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml
index 00751aceb..3e9e39f2c 100644
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@@ -74,9 +74,9 @@ spec:
         secret:
           secretName: {{ .Values.emqxLicenseSecretName }}
       {{- end }}
-      {{- if .Values.extraVolumes }}
-        {{- toYaml .Values.extraVolumes | nindent 8 }}
-      {{- end }}
+    {{- if .Values.extraVolumes }}
+      {{- toYaml .Values.extraVolumes | nindent 6 }}
+    {{- end }}
       {{- if .Values.podSecurityContext.enabled }}
       securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
@@ -141,9 +141,9 @@ spec:
             subPath: "emqx.lic"
             readOnly: true
           {{- end }}
-          {{- if .Values.extraVolumeMounts }}
-            {{- toYaml .Values.extraVolumeMounts | nindent 12 }}
-          {{- end }}
+        {{- if .Values.extraVolumeMounts }}
+          {{- toYaml .Values.extraVolumeMounts | nindent 10 }}
+        {{- end }}
           readinessProbe:
             httpGet:
               path: /status

From 981268911a3cca621cc83506588a4377cce5d467 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Mar 2023 16:56:37 +0100
Subject: [PATCH 25/52] style: make summary lower-case

---
 apps/emqx_bridge/src/emqx_bridge_api.erl      | 20 +++++-----
 .../emqx_dashboard/src/emqx_dashboard_api.erl |  2 +-
 apps/emqx_gateway/src/emqx_gateway_api.erl    | 24 +++++------
 .../src/emqx_gateway_api_authn.erl            | 20 +++++-----
 .../emqx_gateway_api_authn_user_import.erl    |  4 +-
 .../src/emqx_gateway_api_clients.erl          | 28 ++++++-------
 .../src/emqx_gateway_api_listeners.erl        | 40 +++++++++----------
 .../src/emqx_rule_engine_api.erl              | 18 ++++-----
 8 files changed, 78 insertions(+), 78 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index 8ac3e476a..532daf448 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -301,7 +301,7 @@ schema("/bridges") ->
         'operationId' => '/bridges',
         get => #{
             tags => [<<"bridges">>],
-            summary => <<"List Bridges">>,
+            summary => <<"List bridges">>,
             description => ?DESC("desc_api1"),
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_example(
@@ -312,7 +312,7 @@ schema("/bridges") ->
         },
         post => #{
             tags => [<<"bridges">>],
-            summary => <<"Create Bridge">>,
+            summary => <<"Create bridge">>,
             description => ?DESC("desc_api2"),
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
                 emqx_bridge_schema:post_request(),
@@ -329,7 +329,7 @@ schema("/bridges/:id") ->
         'operationId' => '/bridges/:id',
         get => #{
             tags => [<<"bridges">>],
-            summary => <<"Get Bridge">>,
+            summary => <<"Get bridge">>,
             description => ?DESC("desc_api3"),
             parameters => [param_path_id()],
             responses => #{
@@ -339,7 +339,7 @@ schema("/bridges/:id") ->
         },
         put => #{
             tags => [<<"bridges">>],
-            summary => <<"Update Bridge">>,
+            summary => <<"Update bridge">>,
             description => ?DESC("desc_api4"),
             parameters => [param_path_id()],
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -354,7 +354,7 @@ schema("/bridges/:id") ->
         },
         delete => #{
             tags => [<<"bridges">>],
-            summary => <<"Delete Bridge">>,
+            summary => <<"Delete bridge">>,
             description => ?DESC("desc_api5"),
             parameters => [param_path_id()],
             responses => #{
@@ -373,7 +373,7 @@ schema("/bridges/:id/metrics") ->
         'operationId' => '/bridges/:id/metrics',
         get => #{
             tags => [<<"bridges">>],
-            summary => <<"Get Bridge Metrics">>,
+            summary => <<"Get bridge metrics">>,
             description => ?DESC("desc_bridge_metrics"),
             parameters => [param_path_id()],
             responses => #{
@@ -387,7 +387,7 @@ schema("/bridges/:id/metrics/reset") ->
         'operationId' => '/bridges/:id/metrics/reset',
         put => #{
             tags => [<<"bridges">>],
-            summary => <<"Reset Bridge Metrics">>,
+            summary => <<"Reset bridge metrics">>,
             description => ?DESC("desc_api6"),
             parameters => [param_path_id()],
             responses => #{
@@ -402,7 +402,7 @@ schema("/bridges/:id/enable/:enable") ->
         put =>
             #{
                 tags => [<<"bridges">>],
-                summary => <<"Enable or Disable Bridge">>,
+                summary => <<"Enable or disable bridge">>,
                 desc => ?DESC("desc_enable_bridge"),
                 parameters => [param_path_id(), param_path_enable()],
                 responses =>
@@ -418,7 +418,7 @@ schema("/bridges/:id/:operation") ->
         'operationId' => '/bridges/:id/:operation',
         post => #{
             tags => [<<"bridges">>],
-            summary => <<"Stop or Restart Bridge">>,
+            summary => <<"Stop or restart bridge">>,
             description => ?DESC("desc_api7"),
             parameters => [
                 param_path_id(),
@@ -440,7 +440,7 @@ schema("/nodes/:node/bridges/:id/:operation") ->
         'operationId' => '/nodes/:node/bridges/:id/:operation',
         post => #{
             tags => [<<"bridges">>],
-            summary => <<"Stop/Restart Bridge">>,
+            summary => <<"Stop/Restart bridge">>,
             description => ?DESC("desc_api8"),
             parameters => [
                 param_path_node(),
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl
index cc2a1337d..d5655d99d 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl
@@ -74,7 +74,7 @@ schema("/login") ->
         post => #{
             tags => [<<"dashboard">>],
             desc => ?DESC(login_api),
-            summary => <<"Dashboard Auth">>,
+            summary => <<"Dashboard authentication">>,
             'requestBody' => fields([username, password]),
             responses => #{
                 200 => fields([token, version, license]),
diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl
index 1c43340e2..62f723d59 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api.erl
@@ -180,7 +180,7 @@ schema("/gateways") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(list_gateway),
-                summary => <<"List All Gateways">>,
+                summary => <<"List all gateways">>,
                 parameters => params_gateway_status_in_qs(),
                 responses =>
                     #{
@@ -201,7 +201,7 @@ schema("/gateways/:name") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(get_gateway),
-                summary => <<"Get the Gateway">>,
+                summary => <<"Get gateway">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
                     #{
@@ -608,7 +608,7 @@ examples_gateway_confs() ->
     #{
         stomp_gateway =>
             #{
-                summary => <<"A simple STOMP gateway configs">>,
+                summary => <<"A simple STOMP gateway config">>,
                 value =>
                     #{
                         enable => true,
@@ -636,7 +636,7 @@ examples_gateway_confs() ->
             },
         mqttsn_gateway =>
             #{
-                summary => <<"A simple MQTT-SN gateway configs">>,
+                summary => <<"A simple MQTT-SN gateway config">>,
                 value =>
                     #{
                         enable => true,
@@ -672,7 +672,7 @@ examples_gateway_confs() ->
             },
         coap_gateway =>
             #{
-                summary => <<"A simple CoAP gateway configs">>,
+                summary => <<"A simple CoAP gateway config">>,
                 value =>
                     #{
                         enable => true,
@@ -699,7 +699,7 @@ examples_gateway_confs() ->
             },
         lwm2m_gateway =>
             #{
-                summary => <<"A simple LwM2M gateway configs">>,
+                summary => <<"A simple LwM2M gateway config">>,
                 value =>
                     #{
                         enable => true,
@@ -735,7 +735,7 @@ examples_gateway_confs() ->
             },
         exproto_gateway =>
             #{
-                summary => <<"A simple ExProto gateway configs">>,
+                summary => <<"A simple ExProto gateway config">>,
                 value =>
                     #{
                         enable => true,
@@ -765,7 +765,7 @@ examples_update_gateway_confs() ->
     #{
         stomp_gateway =>
             #{
-                summary => <<"A simple STOMP gateway configs">>,
+                summary => <<"A simple STOMP gateway config">>,
                 value =>
                     #{
                         enable => true,
@@ -782,7 +782,7 @@ examples_update_gateway_confs() ->
             },
         mqttsn_gateway =>
             #{
-                summary => <<"A simple MQTT-SN gateway configs">>,
+                summary => <<"A simple MQTT-SN gateway config">>,
                 value =>
                     #{
                         enable => true,
@@ -803,7 +803,7 @@ examples_update_gateway_confs() ->
             },
         coap_gateway =>
             #{
-                summary => <<"A simple CoAP gateway configs">>,
+                summary => <<"A simple CoAP gateway config">>,
                 value =>
                     #{
                         enable => true,
@@ -819,7 +819,7 @@ examples_update_gateway_confs() ->
             },
         lwm2m_gateway =>
             #{
-                summary => <<"A simple LwM2M gateway configs">>,
+                summary => <<"A simple LwM2M gateway config">>,
                 value =>
                     #{
                         enable => true,
@@ -844,7 +844,7 @@ examples_update_gateway_confs() ->
             },
         exproto_gateway =>
             #{
-                summary => <<"A simple ExProto gateway configs">>,
+                summary => <<"A simple ExProto gateway config">>,
                 value =>
                     #{
                         enable => true,
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
index f52b26cd2..41b1b11d5 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
@@ -185,13 +185,13 @@ schema("/gateways/:name/authentication") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(get_authn),
-                summary => <<"Get Authenticator Configuration">>,
+                summary => <<"Get authenticator configuration">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
                     ?STANDARD_RESP(
                         #{
                             200 => schema_authn(),
-                            204 => <<"Authenticator doesn't initiated">>
+                            204 => <<"Authenticator not initialized">>
                         }
                     )
             },
@@ -199,7 +199,7 @@ schema("/gateways/:name/authentication") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(update_authn),
-                summary => <<"Update Authenticator Configuration">>,
+                summary => <<"Update authenticator configuration">>,
                 parameters => params_gateway_name_in_path(),
                 'requestBody' => schema_authn(),
                 responses =>
@@ -209,7 +209,7 @@ schema("/gateways/:name/authentication") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(add_authn),
-                summary => <<"Create an Authenticator for a Gateway">>,
+                summary => <<"Create authenticator for gateway">>,
                 parameters => params_gateway_name_in_path(),
                 'requestBody' => schema_authn(),
                 responses =>
@@ -219,7 +219,7 @@ schema("/gateways/:name/authentication") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(delete_authn),
-                summary => <<"Delete the Gateway Authenticator">>,
+                summary => <<"Delete gateway authenticator">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
                     ?STANDARD_RESP(#{204 => <<"Deleted">>})
@@ -232,7 +232,7 @@ schema("/gateways/:name/authentication/users") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(list_users),
-                summary => <<"List users for a Gateway Authenticator">>,
+                summary => <<"List users for gateway authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_paging_in_qs() ++
                     params_fuzzy_in_qs(),
@@ -250,7 +250,7 @@ schema("/gateways/:name/authentication/users") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(add_user),
-                summary => <<"Add User for a Gateway Authenticator">>,
+                summary => <<"Add user for gateway authenticator">>,
                 parameters => params_gateway_name_in_path(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
                     ref(emqx_authn_api, request_user_create),
@@ -274,7 +274,7 @@ schema("/gateways/:name/authentication/users/:uid") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(get_user),
-                summary => <<"Get User Info for a Gateway Authenticator">>,
+                summary => <<"Get user info for gateway authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_userid_in_path(),
                 responses =>
@@ -291,7 +291,7 @@ schema("/gateways/:name/authentication/users/:uid") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(update_user),
-                summary => <<"Update User Info for a Gateway Authenticator">>,
+                summary => <<"Update user info for gateway authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_userid_in_path(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -312,7 +312,7 @@ schema("/gateways/:name/authentication/users/:uid") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(delete_user),
-                summary => <<"Delete User for a Gateway Authenticator">>,
+                summary => <<"Delete user for gateway authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_userid_in_path(),
                 responses =>
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl
index 705fccf90..68f392923 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl
@@ -126,7 +126,7 @@ schema("/gateways/:name/authentication/import_users") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(emqx_gateway_api_authn, import_users),
-                summary => <<"Import Users">>,
+                summary => <<"Import users">>,
                 parameters => params_gateway_name_in_path(),
                 'requestBody' => emqx_dashboard_swagger:file_schema(filename),
                 responses =>
@@ -140,7 +140,7 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(emqx_gateway_api_listeners, import_users),
-                summary => <<"Import Users">>,
+                summary => <<"Import users">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => emqx_dashboard_swagger:file_schema(filename),
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
index b30de3a3e..e64e918b4 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
@@ -460,7 +460,7 @@ schema("/gateways/:name/clients") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(list_clients),
-                summary => <<"List Gateway's Clients">>,
+                summary => <<"List gateway's clients">>,
                 parameters => params_client_query(),
                 responses =>
                     ?STANDARD_RESP(#{
@@ -478,7 +478,7 @@ schema("/gateways/:name/clients/:clientid") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(get_client),
-                summary => <<"Get Client Info">>,
+                summary => <<"Get client info">>,
                 parameters => params_client_insta(),
                 responses =>
                     ?STANDARD_RESP(#{200 => schema_client()})
@@ -487,7 +487,7 @@ schema("/gateways/:name/clients/:clientid") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(kick_client),
-                summary => <<"Kick out Client">>,
+                summary => <<"Kick out client">>,
                 parameters => params_client_insta(),
                 responses =>
                     ?STANDARD_RESP(#{204 => <<"Kicked">>})
@@ -500,7 +500,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(list_subscriptions),
-                summary => <<"List Client's Subscription">>,
+                summary => <<"List client's subscription">>,
                 parameters => params_client_insta(),
                 responses =>
                     ?STANDARD_RESP(
@@ -516,7 +516,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(add_subscription),
-                summary => <<"Add Subscription for Client">>,
+                summary => <<"Add subscription for client">>,
                 parameters => params_client_insta(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
                     ref(subscription),
@@ -540,7 +540,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions/:topic") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(delete_subscription),
-                summary => <<"Delete Client's Subscription">>,
+                summary => <<"Delete client's subscription">>,
                 parameters => params_topic_name_in_path() ++ params_client_insta(),
                 responses =>
                     ?STANDARD_RESP(#{204 => <<"Unsubscribed">>})
@@ -1020,12 +1020,12 @@ examples_client_list() ->
     #{
         general_client_list =>
             #{
-                summary => <<"General Client List">>,
+                summary => <<"General client list">>,
                 value => [example_general_client()]
             },
         lwm2m_client_list =>
             #{
-                summary => <<"LwM2M Client List">>,
+                summary => <<"LwM2M client list">>,
                 value => [example_lwm2m_client()]
             }
     }.
@@ -1034,12 +1034,12 @@ examples_client() ->
     #{
         general_client =>
             #{
-                summary => <<"General Client Info">>,
+                summary => <<"General client info">>,
                 value => example_general_client()
             },
         lwm2m_client =>
             #{
-                summary => <<"LwM2M Client Info">>,
+                summary => <<"LwM2M client info">>,
                 value => example_lwm2m_client()
             }
     }.
@@ -1048,12 +1048,12 @@ examples_subscription_list() ->
     #{
         general_subscription_list =>
             #{
-                summary => <<"A General Subscription List">>,
+                summary => <<"A general subscription list">>,
                 value => [example_general_subscription()]
             },
         stomp_subscription_list =>
             #{
-                summary => <<"The Stomp Subscription List">>,
+                summary => <<"The STOMP subscription list">>,
                 value => [example_stomp_subscription]
             }
     }.
@@ -1062,12 +1062,12 @@ examples_subscription() ->
     #{
         general_subscription =>
             #{
-                summary => <<"A General Subscription">>,
+                summary => <<"A general subscription">>,
                 value => example_general_subscription()
             },
         stomp_subscription =>
             #{
-                summary => <<"A Stomp Subscription">>,
+                summary => <<"A STOMP subscription">>,
                 value => example_stomp_subscription()
             }
     }.
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
index 43c8156d6..14b80a500 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
@@ -362,7 +362,7 @@ schema("/gateways/:name/listeners") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(list_listeners),
-                summary => <<"List All Listeners">>,
+                summary => <<"List all listeners">>,
                 parameters => params_gateway_name_in_path(),
                 responses =>
                     ?STANDARD_RESP(
@@ -378,7 +378,7 @@ schema("/gateways/:name/listeners") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(add_listener),
-                summary => <<"Add a Listener">>,
+                summary => <<"Add listener">>,
                 parameters => params_gateway_name_in_path(),
                 %% XXX: How to distinguish the different listener supported by
                 %% different types of gateways?
@@ -404,7 +404,7 @@ schema("/gateways/:name/listeners/:id") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(get_listener),
-                summary => <<"Get the Listener Configs">>,
+                summary => <<"Get listener config">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 responses =>
@@ -421,7 +421,7 @@ schema("/gateways/:name/listeners/:id") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(delete_listener),
-                summary => <<"Delete the Listener">>,
+                summary => <<"Delete listener">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 responses =>
@@ -431,7 +431,7 @@ schema("/gateways/:name/listeners/:id") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(update_listener),
-                summary => <<"Update the Listener Configs">>,
+                summary => <<"Update listener config">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -456,7 +456,7 @@ schema("/gateways/:name/listeners/:id/authentication") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(get_listener_authn),
-                summary => <<"Get the Listener's Authenticator">>,
+                summary => <<"Get the listener's authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 responses =>
@@ -471,7 +471,7 @@ schema("/gateways/:name/listeners/:id/authentication") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(add_listener_authn),
-                summary => <<"Create an Authenticator for a Listener">>,
+                summary => <<"Create authenticator for listener">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => schema_authn(),
@@ -482,7 +482,7 @@ schema("/gateways/:name/listeners/:id/authentication") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(update_listener_authn),
-                summary => <<"Update the Listener Authenticator configs">>,
+                summary => <<"Update config of authenticator for listener">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => schema_authn(),
@@ -493,7 +493,7 @@ schema("/gateways/:name/listeners/:id/authentication") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(delete_listener_authn),
-                summary => <<"Delete the Listener's Authenticator">>,
+                summary => <<"Delete the listener's authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 responses =>
@@ -507,7 +507,7 @@ schema("/gateways/:name/listeners/:id/authentication/users") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(list_users),
-                summary => <<"List Authenticator's Users">>,
+                summary => <<"List authenticator's users">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path() ++
                     params_paging_in_qs(),
@@ -525,7 +525,7 @@ schema("/gateways/:name/listeners/:id/authentication/users") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(add_user),
-                summary => <<"Add User for an Authenticator">>,
+                summary => <<"Add user for an authenticator">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path(),
                 'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -550,7 +550,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(get_user),
-                summary => <<"Get User Info">>,
+                summary => <<"Get user info">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path() ++
                     params_userid_in_path(),
@@ -568,7 +568,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(update_user),
-                summary => <<"Update User Info">>,
+                summary => <<"Update user info">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path() ++
                     params_userid_in_path(),
@@ -590,7 +590,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
             #{
                 tags => ?TAGS,
                 desc => ?DESC(delete_user),
-                summary => <<"Delete User">>,
+                summary => <<"Delete user">>,
                 parameters => params_gateway_name_in_path() ++
                     params_listener_id_in_path() ++
                     params_userid_in_path(),
@@ -712,7 +712,7 @@ examples_listener() ->
     #{
         tcp_listener =>
             #{
-                summary => <<"A simple tcp listener example">>,
+                summary => <<"A simple TCP listener example">>,
                 value =>
                     #{
                         name => <<"tcp-def">>,
@@ -738,7 +738,7 @@ examples_listener() ->
             },
         ssl_listener =>
             #{
-                summary => <<"A simple ssl listener example">>,
+                summary => <<"A simple SSL listener example">>,
                 value =>
                     #{
                         name => <<"ssl-def">>,
@@ -771,7 +771,7 @@ examples_listener() ->
             },
         udp_listener =>
             #{
-                summary => <<"A simple udp listener example">>,
+                summary => <<"A simple UDP listener example">>,
                 value =>
                     #{
                         name => <<"udp-def">>,
@@ -789,7 +789,7 @@ examples_listener() ->
             },
         dtls_listener =>
             #{
-                summary => <<"A simple dtls listener example">>,
+                summary => <<"A simple DTLS listener example">>,
                 value =>
                     #{
                         name => <<"dtls-def">>,
@@ -817,7 +817,7 @@ examples_listener() ->
             },
         dtls_listener_with_psk_ciphers =>
             #{
-                summary => <<"A dtls listener with PSK example">>,
+                summary => <<"A DTLS listener with PSK example">>,
                 value =>
                     #{
                         name => <<"dtls-psk">>,
@@ -845,7 +845,7 @@ examples_listener() ->
             },
         lisetner_with_authn =>
             #{
-                summary => <<"A tcp listener with authentication example">>,
+                summary => <<"A TCP listener with authentication example">>,
                 value =>
                     #{
                         name => <<"tcp-with-authn">>,
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index 30de3e8e8..106693a0a 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -180,7 +180,7 @@ schema("/rules") ->
                 ref(emqx_dashboard_swagger, page),
                 ref(emqx_dashboard_swagger, limit)
             ],
-            summary => <<"List Rules">>,
+            summary => <<"List rules">>,
             responses => #{
                 200 =>
                     [
@@ -193,7 +193,7 @@ schema("/rules") ->
         post => #{
             tags => [<<"rules">>],
             description => ?DESC("api2"),
-            summary => <<"Create a Rule">>,
+            summary => <<"Create a rule">>,
             'requestBody' => rule_creation_schema(),
             responses => #{
                 400 => error_schema('BAD_REQUEST', "Invalid Parameters"),
@@ -207,7 +207,7 @@ schema("/rule_events") ->
         get => #{
             tags => [<<"rules">>],
             description => ?DESC("api3"),
-            summary => <<"List Events">>,
+            summary => <<"List rule events">>,
             responses => #{
                 200 => mk(ref(emqx_rule_api_schema, "rule_events"), #{})
             }
@@ -219,7 +219,7 @@ schema("/rules/:id") ->
         get => #{
             tags => [<<"rules">>],
             description => ?DESC("api4"),
-            summary => <<"Get a Rule">>,
+            summary => <<"Get rule">>,
             parameters => param_path_id(),
             responses => #{
                 404 => error_schema('NOT_FOUND', "Rule not found"),
@@ -229,7 +229,7 @@ schema("/rules/:id") ->
         put => #{
             tags => [<<"rules">>],
             description => ?DESC("api5"),
-            summary => <<"Update a Rule">>,
+            summary => <<"Update rule">>,
             parameters => param_path_id(),
             'requestBody' => rule_creation_schema(),
             responses => #{
@@ -240,7 +240,7 @@ schema("/rules/:id") ->
         delete => #{
             tags => [<<"rules">>],
             description => ?DESC("api6"),
-            summary => <<"Delete a Rule">>,
+            summary => <<"Delete rule">>,
             parameters => param_path_id(),
             responses => #{
                 204 => <<"Delete rule successfully">>
@@ -253,7 +253,7 @@ schema("/rules/:id/metrics") ->
         get => #{
             tags => [<<"rules">>],
             description => ?DESC("api4_1"),
-            summary => <<"Get a Rule's Metrics">>,
+            summary => <<"Get rule metrics">>,
             parameters => param_path_id(),
             responses => #{
                 404 => error_schema('NOT_FOUND', "Rule not found"),
@@ -267,7 +267,7 @@ schema("/rules/:id/metrics/reset") ->
         put => #{
             tags => [<<"rules">>],
             description => ?DESC("api7"),
-            summary => <<"Reset a Rule Metrics">>,
+            summary => <<"Reset rule metrics">>,
             parameters => param_path_id(),
             responses => #{
                 404 => error_schema('NOT_FOUND', "Rule not found"),
@@ -281,7 +281,7 @@ schema("/rule_test") ->
         post => #{
             tags => [<<"rules">>],
             description => ?DESC("api8"),
-            summary => <<"Test a Rule">>,
+            summary => <<"Test a rule">>,
             'requestBody' => rule_test_schema(),
             responses => #{
                 400 => error_schema('BAD_REQUEST', "Invalid Parameters"),

From d9b1b1c802b8247dbabb1523c2f757d40f415067 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Tue, 21 Mar 2023 15:23:50 +0300
Subject: [PATCH 26/52] fix(bridge-api): unflatten nodes' bridge lists back

Bridge lists were erroneously flattened in cad6492c. This causes
bridge listing fail in emqx clusters consisting of more than 1 node.
---
 apps/emqx_bridge/src/emqx_bridge_api.erl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index f7a1bb345..f780aa16d 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -71,8 +71,8 @@
 
 -define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME),
     ?NOT_FOUND(
-        <<"Bridge lookup failed: bridge named '", BRIDGE_NAME/binary, "' of type ",
-            (atom_to_binary(BRIDGE_TYPE))/binary, " does not exist.">>
+        <<"Bridge lookup failed: bridge named '", (BRIDGE_NAME)/binary, "' of type ",
+            (bin(BRIDGE_TYPE))/binary, " does not exist.">>
     )
 ).
 
@@ -507,10 +507,10 @@ schema("/bridges_probe") ->
     case is_ok(NodeReplies) of
         {ok, NodeBridges} ->
             AllBridges = [
-                format_resource(Data, Node)
-             || {Node, Bridges} <- lists:zip(Nodes, NodeBridges), Data <- Bridges
+                [format_resource(Data, Node) || Data <- Bridges]
+             || {Node, Bridges} <- lists:zip(Nodes, NodeBridges)
             ],
-            ?OK(zip_bridges([AllBridges]));
+            ?OK(zip_bridges(AllBridges));
         {error, Reason} ->
             ?INTERNAL_ERROR(Reason)
     end.

From e2ff8c1c8f071d549d5041460c956091ea7d2fad Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Tue, 21 Mar 2023 17:22:23 +0300
Subject: [PATCH 27/52] chore: add changelog entry

---
 changes/ce/fix-10190.en.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/ce/fix-10190.en.md

diff --git a/changes/ce/fix-10190.en.md b/changes/ce/fix-10190.en.md
new file mode 100644
index 000000000..bffd9ca00
--- /dev/null
+++ b/changes/ce/fix-10190.en.md
@@ -0,0 +1 @@
+Fix the issue where nodes responses to the list bridges RPC were incorrectly flattened, which caused List Bridges API HTTP handler to crash when there was more than 1 node in the cluster.

From 327b78221027b430db6f9c6658e2b2e2987ab7e9 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Mar 2023 17:04:42 +0100
Subject: [PATCH 28/52] style: lower case in descriptions

---
 .../emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf b/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf
index 623884f31..22f038d4e 100644
--- a/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf
+++ b/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf
@@ -1,7 +1,7 @@
 emqx_topic_metrics_api {
   get_topic_metrics_api {
     desc {
-      en: """List Topic metrics"""
+      en: """List topic metrics"""
       zh: """获取主题监控数据"""
     }
   }
@@ -15,21 +15,21 @@ emqx_topic_metrics_api {
 
   post_topic_metrics_api {
     desc {
-      en: """Create Topic metrics"""
+      en: """Create topic metrics"""
       zh: """创建主题监控数据"""
     }
   }
 
   gat_topic_metrics_data_api {
     desc {
-      en: """Get Topic metrics"""
+      en: """Get topic metrics"""
       zh: """获取主题监控数据"""
     }
   }
 
   delete_topic_metrics_data_api {
     desc {
-      en: """Delete Topic metrics"""
+      en: """Delete topic metrics"""
       zh: """删除主题监控数据"""
     }
   }
@@ -43,7 +43,7 @@ emqx_topic_metrics_api {
 
   topic_metrics_api_response400 {
     desc {
-      en: """Bad Request. Already exists or bad topic name"""
+      en: """Bad request. Already exists or bad topic name"""
       zh: """错误请求。已存在或错误的主题名称"""
     }
   }

From 394a4559035b6855f37a3094a87ded234a5e1773 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Mar 2023 17:07:13 +0100
Subject: [PATCH 29/52] chore: add changelog

---
 changes/ce/fix-10196.en.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/ce/fix-10196.en.md

diff --git a/changes/ce/fix-10196.en.md b/changes/ce/fix-10196.en.md
new file mode 100644
index 000000000..58ff01d8e
--- /dev/null
+++ b/changes/ce/fix-10196.en.md
@@ -0,0 +1 @@
+Use lower-case for schema summaries and descritptions to be used in menu of generated online documentation.

From 7e6f52e8fea00d3f2f5553e49ff24dbb7b1a1ff4 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 21 Mar 2023 17:45:58 -0300
Subject: [PATCH 30/52] test: attempt to fix flaky kafka consumer test

It might need some time for the metrics to be set.
---
 .../test/emqx_bridge_impl_kafka_consumer_SUITE.erl          | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl
index 15b4fbe40..be6494cb2 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl
@@ -1623,7 +1623,11 @@ t_bridge_rule_action_source(Config) ->
                 },
                 emqx_json:decode(RawPayload, [return_maps])
             ),
-            ?assertEqual(1, emqx_resource_metrics:received_get(ResourceId)),
+            ?retry(
+                _Interval = 200,
+                _NAttempts = 20,
+                ?assertEqual(1, emqx_resource_metrics:received_get(ResourceId))
+            ),
             ok
         end
     ),

From 9853d00cadb9ba446c769353d565ceb2c5d54230 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Mon, 13 Mar 2023 11:17:53 +0800
Subject: [PATCH 31/52] feat(bridges): integrate RocketMQ into data bridges

---
 apps/emqx_bridge/src/emqx_bridge.erl          |   3 +-
 .../i18n/emqx_ee_bridge_rocketmq.conf         |  70 ++++
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  17 +-
 .../src/emqx_ee_bridge_rocketmq.erl           | 120 ++++++
 .../i18n/emqx_ee_connector_rocketmq.conf      |  66 ++++
 lib-ee/emqx_ee_connector/rebar.config         |   1 +
 .../src/emqx_ee_connector.app.src             |   3 +-
 .../src/emqx_ee_connector_rocketmq.erl        | 342 ++++++++++++++++++
 8 files changed, 617 insertions(+), 5 deletions(-)
 create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl
 create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf
 create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl

diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl
index 98ce6a8b0..087c72dc3 100644
--- a/apps/emqx_bridge/src/emqx_bridge.erl
+++ b/apps/emqx_bridge/src/emqx_bridge.erl
@@ -67,7 +67,8 @@
     T == timescale;
     T == matrix;
     T == tdengine;
-    T == dynamo
+    T == dynamo;
+    T == rocketmq
 ).
 
 load() ->
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf
new file mode 100644
index 000000000..2e33e6c07
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf
@@ -0,0 +1,70 @@
+emqx_ee_bridge_rocketmq {
+
+    local_topic {
+        desc {
+            en: """The MQTT topic filter to be forwarded to RocketMQ. All MQTT `PUBLISH` messages with the topic
+matching the `local_topic` will be forwarded.</br>
+NOTE: if the bridge is used as a rule action, `local_topic` should be left empty otherwise the messages will be duplicated."""
+            zh: """发送到 'local_topic' 的消息都会转发到 RocketMQ。 </br>
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。"""
+        }
+        label {
+                en: "Local Topic"
+                zh: "本地 Topic"
+            }
+    }
+
+    template {
+        desc {
+            en: """Template, the default value is empty. When this value is empty the whole message will be stored in the RocketMQ"""
+            zh: """模板, 默认为空，为空时将会将整个消息转发给 RocketMQ"""
+            }
+        label {
+            en: "Template"
+            zh: "模板"
+        }
+    }
+    config_enable {
+        desc {
+            en: """Enable or disable this bridge"""
+            zh: """启用/禁用桥接"""
+        }
+        label {
+            en: "Enable Or Disable Bridge"
+            zh: "启用/禁用桥接"
+        }
+        }
+
+    desc_config {
+        desc {
+            en: """Configuration for a RocketMQ bridge."""
+            zh: """RocketMQ 桥接配置"""
+        }
+        label: {
+            en: "RocketMQ Bridge Configuration"
+            zh: "RocketMQ 桥接配置"
+        }
+    }
+
+    desc_type {
+        desc {
+            en: """The Bridge Type"""
+            zh: """Bridge 类型"""
+        }
+        label {
+            en: "Bridge Type"
+            zh: "桥接类型"
+        }
+    }
+
+    desc_name {
+        desc {
+            en: """Bridge name."""
+            zh: """桥接名字"""
+        }
+        label {
+            en: "Bridge Name"
+            zh: "桥接名字"
+        }
+    }
+}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index ec81b7935..3989c3ab2 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -32,7 +32,8 @@ api_schemas(Method) ->
         ref(emqx_ee_bridge_matrix, Method),
         ref(emqx_ee_bridge_tdengine, Method),
         ref(emqx_ee_bridge_clickhouse, Method),
-        ref(emqx_ee_bridge_dynamo, Method)
+        ref(emqx_ee_bridge_dynamo, Method),
+        ref(emqx_ee_bridge_rocketmq, Method)
     ].
 
 schema_modules() ->
@@ -49,7 +50,8 @@ schema_modules() ->
         emqx_ee_bridge_matrix,
         emqx_ee_bridge_tdengine,
         emqx_ee_bridge_clickhouse,
-        emqx_ee_bridge_dynamo
+        emqx_ee_bridge_dynamo,
+        emqx_ee_bridge_rocketmq
     ].
 
 examples(Method) ->
@@ -85,7 +87,8 @@ resource_type(timescale) -> emqx_connector_pgsql;
 resource_type(matrix) -> emqx_connector_pgsql;
 resource_type(tdengine) -> emqx_ee_connector_tdengine;
 resource_type(clickhouse) -> emqx_ee_connector_clickhouse;
-resource_type(dynamo) -> emqx_ee_connector_dynamo.
+resource_type(dynamo) -> emqx_ee_connector_dynamo;
+resource_type(rocketmq) -> emqx_ee_connector_rocketmq.
 
 fields(bridges) ->
     [
@@ -128,6 +131,14 @@ fields(bridges) ->
                     desc => <<"Dynamo Bridge Config">>,
                     required => false
                 }
+            )},
+        {rocketmq,
+            mk(
+                hoconsc:map(name, ref(emqx_ee_bridge_rocketmq, "config")),
+                #{
+                    desc => <<"RocketMQ Bridge Config">>,
+                    required => false
+                }
             )}
     ] ++ kafka_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++
         pgsql_structs() ++ clickhouse_structs().
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl
new file mode 100644
index 000000000..d81ffc54c
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl
@@ -0,0 +1,120 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge_rocketmq).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+-include_lib("emqx_bridge/include/emqx_bridge.hrl").
+-include_lib("emqx_resource/include/emqx_resource.hrl").
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-export([
+    conn_bridge_examples/1,
+    values/1
+]).
+
+-export([
+    namespace/0,
+    roots/0,
+    fields/1,
+    desc/1
+]).
+
+-define(DEFAULT_TEMPLATE, <<>>).
+-define(DEFFAULT_REQ_TIMEOUT, <<"15s">>).
+
+%% -------------------------------------------------------------------------------------------------
+%% api
+
+conn_bridge_examples(Method) ->
+    [
+        #{
+            <<"rocketmq">> => #{
+                summary => <<"RocketMQ Bridge">>,
+                value => values(Method)
+            }
+        }
+    ].
+
+values(get) ->
+    maps:merge(values(post), ?METRICS_EXAMPLE);
+values(post) ->
+    #{
+        enable => true,
+        type => rocketmq,
+        name => <<"foo">>,
+        server => <<"127.0.0.1:9876">>,
+        topic => <<"TopicTest">>,
+        template => ?DEFAULT_TEMPLATE,
+        local_topic => <<"local/topic/#">>,
+        resource_opts => #{
+            worker_pool_size => 1,
+            health_check_interval => ?HEALTHCHECK_INTERVAL_RAW,
+            auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW,
+            batch_size => ?DEFAULT_BATCH_SIZE,
+            batch_time => ?DEFAULT_BATCH_TIME,
+            query_mode => sync,
+            max_queue_bytes => ?DEFAULT_QUEUE_SIZE
+        }
+    };
+values(put) ->
+    values(post).
+
+%% -------------------------------------------------------------------------------------------------
+%% Hocon Schema Definitions
+namespace() -> "bridge_rocketmq".
+
+roots() -> [].
+
+fields("config") ->
+    [
+        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
+        {template,
+            mk(
+                binary(),
+                #{desc => ?DESC("template"), default => ?DEFAULT_TEMPLATE}
+            )},
+        {local_topic,
+            mk(
+                binary(),
+                #{desc => ?DESC("local_topic"), required => false}
+            )},
+        {resource_opts,
+            mk(
+                ref(?MODULE, "creation_opts"),
+                #{
+                    required => false,
+                    default => #{<<"request_timeout">> => ?DEFFAULT_REQ_TIMEOUT},
+                    desc => ?DESC(emqx_resource_schema, <<"resource_opts">>)
+                }
+            )}
+    ] ++
+        (emqx_ee_connector_rocketmq:fields(config) --
+            emqx_connector_schema_lib:prepare_statement_fields());
+fields("creation_opts") ->
+    emqx_resource_schema:fields("creation_opts_sync_only");
+fields("post") ->
+    [type_field(), name_field() | fields("config")];
+fields("put") ->
+    fields("config");
+fields("get") ->
+    emqx_bridge_schema:status_fields() ++ fields("post").
+
+desc("config") ->
+    ?DESC("desc_config");
+desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
+    ["Configuration for RocketMQ using `", string:to_upper(Method), "` method."];
+desc("creation_opts" = Name) ->
+    emqx_resource_schema:desc(Name);
+desc(_) ->
+    undefined.
+
+%% -------------------------------------------------------------------------------------------------
+
+type_field() ->
+    {type, mk(enum([rocketmq]), #{required => true, desc => ?DESC("desc_type")})}.
+
+name_field() ->
+    {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf
new file mode 100644
index 000000000..d4a610212
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf
@@ -0,0 +1,66 @@
+emqx_ee_connector_rocketmq {
+
+    server {
+        desc {
+          en: """
+The IPv4 or IPv6 address or the hostname to connect to.<br/>
+A host entry has the following form: `Host[:Port]`.<br/>
+The RocketMQ default port 9876 is used if `[:Port]` is not specified.
+"""
+          zh: """
+将要连接的 IPv4 或 IPv6 地址，或者主机名。<br/>
+主机名具有以下形式：`Host[:Port]`。<br/>
+如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。
+"""
+        }
+        label: {
+              en: "Server Host"
+              zh: "服务器地址"
+            }
+    }
+
+    topic {
+        desc {
+          en: """RocketMQ Topic"""
+          zh: """RocketMQ 主题"""
+        }
+        label: {
+              en: "RocketMQ Topic"
+              zh: "RocketMQ 主题"
+            }
+    }
+
+    refresh_interval {
+        desc {
+          en: """RocketMQ Topic Route Refresh Interval."""
+          zh: """RocketMQ 主题路由更新间隔。"""
+        }
+        label: {
+              en: "Topic Route Refresh Interval"
+              zh: "主题路由更新间隔"
+            }
+    }
+
+    send_buffer {
+        desc {
+          en: """The socket send buffer size of the RocketMQ driver client."""
+          zh: """RocketMQ 驱动的套字节发送消息的缓冲区大小"""
+        }
+        label: {
+              en: "Send Buffer Size"
+              zh: "发送消息的缓冲区大小"
+            }
+    }
+
+    security_token {
+        desc {
+          en: """RocketMQ Server Security Token"""
+          zh: """RocketMQ 服务器安全令牌"""
+        }
+        label: {
+              en: "Security Token"
+              zh: "安全令牌"
+            }
+    }
+
+}
diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
index 76f6ccfba..96b3df6a3 100644
--- a/lib-ee/emqx_ee_connector/rebar.config
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -5,6 +5,7 @@
   {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.5"}}},
   {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.2"}}},
   {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}},
+  {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}},
   {emqx, {path, "../../apps/emqx"}}
 ]}.
 
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index 6f40f7158..d8921198c 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -11,7 +11,8 @@
         wolff,
         brod,
         clickhouse,
-        erlcloud
+        erlcloud,
+        rocketmq
     ]},
     {env, []},
     {modules, []},
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl
new file mode 100644
index 000000000..d41f83e1d
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl
@@ -0,0 +1,342 @@
+%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_connector_rocketmq).
+
+-behaviour(emqx_resource).
+
+-include_lib("emqx_resource/include/emqx_resource.hrl").
+-include_lib("typerefl/include/types.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+
+-export([roots/0, fields/1]).
+
+%% `emqx_resource' API
+-export([
+    callback_mode/0,
+    is_buffer_supported/0,
+    on_start/2,
+    on_stop/2,
+    on_query/3,
+    on_batch_query/3,
+    on_get_status/2
+]).
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-define(ROCKETMQ_HOST_OPTIONS, #{
+    default_port => 9876
+}).
+
+-ifdef(TEST).
+-export([execute/2]).
+-endif.
+
+%%=====================================================================
+%% Hocon schema
+roots() ->
+    [{config, #{type => hoconsc:ref(?MODULE, config)}}].
+
+fields(config) ->
+    [
+        {server, server()},
+        {topic,
+            mk(
+                binary(),
+                #{default => <<"TopicTest">>, desc => ?DESC(topic)}
+            )},
+        {refresh_interval,
+            mk(
+                emqx_schema:duration(),
+                #{default => <<"3s">>, desc => ?DESC(refresh_interval)}
+            )},
+        {send_buffer,
+            mk(
+                emqx_schema:bytesize(),
+                #{default => <<"1024KB">>, desc => ?DESC(send_buffer)}
+            )},
+        {security_token, mk(binary(), #{default => <<>>, desc => ?DESC(security_token)})}
+        | relational_fields()
+    ].
+
+add_default_username(Fields) ->
+    lists:map(
+        fun
+            ({username, OrigUsernameFn}) ->
+                {username, add_default_fn(OrigUsernameFn, <<"">>)};
+            (Field) ->
+                Field
+        end,
+        Fields
+    ).
+
+add_default_fn(OrigFn, Default) ->
+    fun
+        (default) -> Default;
+        (Field) -> OrigFn(Field)
+    end.
+
+server() ->
+    Meta = #{desc => ?DESC("server")},
+    emqx_schema:servers_sc(Meta, ?ROCKETMQ_HOST_OPTIONS).
+
+relational_fields() ->
+    Fields = [username, password, auto_reconnect],
+    Values = lists:filter(
+        fun({E, _}) -> lists:member(E, Fields) end,
+        emqx_connector_schema_lib:relational_db_fields()
+    ),
+    add_default_username(Values).
+
+%%========================================================================================
+%% `emqx_resource' API
+%%========================================================================================
+
+callback_mode() -> always_sync.
+
+is_buffer_supported() -> false.
+
+on_start(
+    InstanceId,
+    #{server := Server, topic := Topic} = Config1
+) ->
+    ?SLOG(info, #{
+        msg => "starting_rocketmq_connector",
+        connector => InstanceId,
+        config => redact(Config1)
+    }),
+    Config = maps:merge(default_security_info(), Config1),
+    {Host, Port} = emqx_schema:parse_server(Server, ?ROCKETMQ_HOST_OPTIONS),
+
+    Server1 = [{Host, Port}],
+    ClientId = client_id(InstanceId),
+    ClientCfg = #{acl_info => #{}},
+
+    TopicTks = emqx_plugin_libs_rule:preproc_tmpl(Topic),
+    ProducerOpts = make_producer_opts(Config),
+    Templates = parse_template(Config),
+    ProducersMapPID = create_producers_map(ClientId),
+    State = #{
+        client_id => ClientId,
+        topic_tokens => TopicTks,
+        config => Config,
+        templates => Templates,
+        producers_map_pid => ProducersMapPID,
+        producers_opts => ProducerOpts
+    },
+
+    case rocketmq:ensure_supervised_client(ClientId, Server1, ClientCfg) of
+        {ok, _Pid} ->
+            {ok, State};
+        {error, _Reason} = Error ->
+            ?tp(
+                rocketmq_connector_start_failed,
+                #{error => _Reason}
+            ),
+            Error
+    end.
+
+on_stop(InstanceId, #{client_id := ClientId, producers_map_pid := Pid} = _State) ->
+    ?SLOG(info, #{
+        msg => "stopping_rocketmq_connector",
+        connector => InstanceId
+    }),
+    Pid ! ok,
+    ok = rocketmq:stop_and_delete_supervised_client(ClientId).
+
+on_query(InstanceId, Query, State) ->
+    do_query(InstanceId, Query, send_sync, State).
+
+%% We only support batch inserts and all messages must have the same topic
+on_batch_query(InstanceId, [{send_message, _Msg} | _] = Query, State) ->
+    do_query(InstanceId, Query, batch_send_sync, State);
+on_batch_query(_InstanceId, Query, _State) ->
+    {error, {unrecoverable_error, {invalid_request, Query}}}.
+
+on_get_status(_InstanceId, #{client_id := ClientId}) ->
+    case rocketmq_client_sup:find_client(ClientId) of
+        {ok, _Pid} ->
+            connected;
+        _ ->
+            connecting
+    end.
+
+%%========================================================================================
+%% Helper fns
+%%========================================================================================
+
+do_query(
+    InstanceId,
+    Query,
+    QueryFunc,
+    #{
+        templates := Templates,
+        client_id := ClientId,
+        topic_tokens := TopicTks,
+        producers_opts := ProducerOpts,
+        config := #{topic := RawTopic, resource_opts := #{request_timeout := RequestTimeout}}
+    } = State
+) ->
+    ?TRACE(
+        "QUERY",
+        "rocketmq_connector_received",
+        #{connector => InstanceId, query => Query, state => State}
+    ),
+
+    TopicKey = get_topic_key(Query, RawTopic, TopicTks),
+    Data = apply_template(Query, Templates),
+
+    Result = safe_do_produce(
+        InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout
+    ),
+    case Result of
+        {error, Reason} ->
+            ?tp(
+                rocketmq_connector_query_return,
+                #{error => Reason}
+            ),
+            ?SLOG(error, #{
+                msg => "rocketmq_connector_do_query_failed",
+                connector => InstanceId,
+                query => Query,
+                reason => Reason
+            }),
+            Result;
+        _ ->
+            ?tp(
+                rocketmq_connector_query_return,
+                #{result => Result}
+            ),
+            Result
+    end.
+
+safe_do_produce(InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout) ->
+    try
+        Producers = get_producers(ClientId, TopicKey, ProducerOpts),
+        produce(InstanceId, QueryFunc, Producers, Data, RequestTimeout)
+    catch
+        _Type:Reason ->
+            {error, {unrecoverable_error, Reason}}
+    end.
+
+produce(_InstanceId, QueryFunc, Producers, Data, RequestTimeout) ->
+    rocketmq:QueryFunc(Producers, Data, RequestTimeout).
+
+parse_template(Config) ->
+    Templates =
+        case maps:get(template, Config, undefined) of
+            undefined -> #{};
+            <<>> -> #{};
+            Template -> #{send_message => Template}
+        end,
+
+    parse_template(maps:to_list(Templates), #{}).
+
+parse_template([{Key, H} | T], Templates) ->
+    ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(H),
+    parse_template(
+        T,
+        Templates#{Key => ParamsTks}
+    );
+parse_template([], Templates) ->
+    Templates.
+
+get_topic_key({_, Msg}, RawTopic, TopicTks) ->
+    {RawTopic, emqx_plugin_libs_rule:proc_tmpl(TopicTks, Msg)};
+get_topic_key([Query | _], RawTopic, TopicTks) ->
+    get_topic_key(Query, RawTopic, TopicTks).
+
+apply_template({Key, Msg} = _Req, Templates) ->
+    case maps:get(Key, Templates, undefined) of
+        undefined ->
+            emqx_json:encode(Msg);
+        Template ->
+            emqx_plugin_libs_rule:proc_tmpl(Template, Msg)
+    end;
+apply_template([{Key, _} | _] = Reqs, Templates) ->
+    case maps:get(Key, Templates, undefined) of
+        undefined ->
+            [emqx_json:encode(Msg) || {_, Msg} <- Reqs];
+        Template ->
+            [emqx_plugin_libs_rule:proc_tmpl(Template, Msg) || {_, Msg} <- Reqs]
+    end.
+
+client_id(InstanceId) ->
+    Name = emqx_resource_manager:manager_id_to_resource_id(InstanceId),
+    erlang:binary_to_atom(Name, utf8).
+
+redact(Msg) ->
+    emqx_misc:redact(Msg, fun is_sensitive_key/1).
+
+is_sensitive_key(security_token) ->
+    true;
+is_sensitive_key(_) ->
+    false.
+
+make_producer_opts(
+    #{
+        username := Username,
+        password := Password,
+        security_token := SecurityToken,
+        send_buffer := SendBuff,
+        refresh_interval := RefreshInterval
+    }
+) ->
+    ACLInfo = acl_info(Username, Password, SecurityToken),
+    #{
+        tcp_opts => [{sndbuf, SendBuff}],
+        ref_topic_route_interval => RefreshInterval,
+        acl_info => ACLInfo
+    }.
+
+acl_info(<<>>, <<>>, <<>>) ->
+    #{};
+acl_info(Username, Password, <<>>) when is_binary(Username), is_binary(Password) ->
+    #{
+        access_key => Username,
+        secret_key => Password
+    };
+acl_info(Username, Password, SecurityToken) when
+    is_binary(Username), is_binary(Password), is_binary(SecurityToken)
+->
+    #{
+        access_key => Username,
+        secret_key => Password,
+        security_token => SecurityToken
+    };
+acl_info(_, _, _) ->
+    #{}.
+
+create_producers_map(ClientId) ->
+    erlang:spawn(fun() ->
+        case ets:whereis(ClientId) of
+            undefined ->
+                _ = ets:new(ClientId, [public, named_table]),
+                ok;
+            _ ->
+                ok
+        end,
+        receive
+            _Msg ->
+                ok
+        end
+    end).
+
+get_producers(ClientId, {_, Topic1} = TopicKey, ProducerOpts) ->
+    case ets:lookup(ClientId, TopicKey) of
+        [{_, Producers0}] ->
+            Producers0;
+        _ ->
+            ProducerGroup = iolist_to_binary([atom_to_list(ClientId), "_", Topic1]),
+            {ok, Producers0} = rocketmq:ensure_supervised_producers(
+                ClientId, ProducerGroup, Topic1, ProducerOpts
+            ),
+            ets:insert(ClientId, {TopicKey, Producers0}),
+            Producers0
+    end.
+
+default_security_info() ->
+    #{username => <<>>, password => <<>>, security_token => <<>>}.

From 4ad3579966945a40750e9ee2b3c87fe4148a6d89 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Mon, 20 Mar 2023 11:14:25 +0800
Subject: [PATCH 32/52] test(bridges): add test suite for RocketMQ

---
 .../docker-compose-rocketmq.yaml              |  34 +++
 .../docker-compose-toxiproxy.yaml             |   1 +
 .../rocketmq/conf/broker.conf                 |  22 ++
 .../rocketmq/logs/.gitkeep                    |   0
 .../rocketmq/store/.gitkeep                   |   0
 .ci/docker-compose-file/toxiproxy.json        |   6 +
 lib-ee/emqx_ee_bridge/docker-ct               |   1 +
 .../test/emqx_ee_bridge_rocketmq_SUITE.erl    | 267 ++++++++++++++++++
 .../src/emqx_ee_connector_rocketmq.erl        |   4 -
 scripts/ct/run.sh                             |   3 +
 10 files changed, 334 insertions(+), 4 deletions(-)
 create mode 100644 .ci/docker-compose-file/docker-compose-rocketmq.yaml
 create mode 100644 .ci/docker-compose-file/rocketmq/conf/broker.conf
 create mode 100644 .ci/docker-compose-file/rocketmq/logs/.gitkeep
 create mode 100644 .ci/docker-compose-file/rocketmq/store/.gitkeep
 create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl

diff --git a/.ci/docker-compose-file/docker-compose-rocketmq.yaml b/.ci/docker-compose-file/docker-compose-rocketmq.yaml
new file mode 100644
index 000000000..3c872a7c2
--- /dev/null
+++ b/.ci/docker-compose-file/docker-compose-rocketmq.yaml
@@ -0,0 +1,34 @@
+version: '3.9'
+
+services:
+  mqnamesrv:
+    image: apache/rocketmq:4.9.4
+    container_name: rocketmq_namesrv
+#    ports:
+#      - 9876:9876
+    volumes:
+      - ./rocketmq/logs:/opt/logs
+      - ./rocketmq/store:/opt/store
+    command: ./mqnamesrv 
+    networks:
+      - emqx_bridge
+
+  mqbroker:
+    image: apache/rocketmq:4.9.4 
+    container_name: rocketmq_broker
+#    ports:
+#      - 10909:10909
+#      - 10911:10911
+    volumes:
+      - ./rocketmq/logs:/opt/logs
+      - ./rocketmq/store:/opt/store
+      - ./rocketmq/conf/broker.conf:/etc/rocketmq/broker.conf
+    environment:
+        NAMESRV_ADDR: "rocketmq_namesrv:9876"
+        JAVA_OPTS: " -Duser.home=/opt"
+        JAVA_OPT_EXT: "-server -Xms1024m -Xmx1024m -Xmn1024m"
+    command: ./mqbroker -c /etc/rocketmq/broker.conf
+    depends_on:
+      - mqnamesrv
+    networks:
+      - emqx_bridge
diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
index 16f18b6c2..24f1d90b2 100644
--- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
+++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
@@ -22,6 +22,7 @@ services:
       - 15433:5433
       - 16041:6041
       - 18000:8000
+      - 19876:9876
     command:
       - "-host=0.0.0.0"
       - "-config=/config/toxiproxy.json"
diff --git a/.ci/docker-compose-file/rocketmq/conf/broker.conf b/.ci/docker-compose-file/rocketmq/conf/broker.conf
new file mode 100644
index 000000000..c343090e4
--- /dev/null
+++ b/.ci/docker-compose-file/rocketmq/conf/broker.conf
@@ -0,0 +1,22 @@
+brokerClusterName=DefaultCluster
+brokerName=broker-a
+brokerId=0
+
+brokerIP1=rocketmq_broker
+
+defaultTopicQueueNums=4
+autoCreateTopicEnable=true
+autoCreateSubscriptionGroup=true
+
+listenPort=10911
+deleteWhen=04
+
+fileReservedTime=120
+mapedFileSizeCommitLog=1073741824
+mapedFileSizeConsumeQueue=300000
+diskMaxUsedSpaceRatio=100
+maxMessageSize=65536
+
+brokerRole=ASYNC_MASTER
+
+flushDiskType=ASYNC_FLUSH
diff --git a/.ci/docker-compose-file/rocketmq/logs/.gitkeep b/.ci/docker-compose-file/rocketmq/logs/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/.ci/docker-compose-file/rocketmq/store/.gitkeep b/.ci/docker-compose-file/rocketmq/store/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json
index 2f8c4341b..e22735091 100644
--- a/.ci/docker-compose-file/toxiproxy.json
+++ b/.ci/docker-compose-file/toxiproxy.json
@@ -77,5 +77,11 @@
     "listen": "0.0.0.0:9295",
     "upstream": "kafka-1.emqx.net:9295",
     "enabled": true
+  },
+  {
+    "name": "rocketmq",
+    "listen": "0.0.0.0:9876",
+    "upstream": "rocketmq_namesrv:9876",
+    "enabled": true
   }
 ]
diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct
index ac1728ad2..116bc44ad 100644
--- a/lib-ee/emqx_ee_bridge/docker-ct
+++ b/lib-ee/emqx_ee_bridge/docker-ct
@@ -10,3 +10,4 @@ pgsql
 tdengine
 clickhouse
 dynamo
+rocketmq
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl
new file mode 100644
index 000000000..cd02b65d0
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl
@@ -0,0 +1,267 @@
+%%--------------------------------------------------------------------
+% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_bridge_rocketmq_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+% Bridge defaults
+-define(TOPIC, "TopicTest").
+-define(BATCH_SIZE, 10).
+-define(PAYLOAD, <<"HELLO">>).
+
+-define(GET_CONFIG(KEY__, CFG__), proplists:get_value(KEY__, CFG__)).
+
+%%------------------------------------------------------------------------------
+%% CT boilerplate
+%%------------------------------------------------------------------------------
+
+all() ->
+    [
+        {group, with_batch},
+        {group, without_batch}
+    ].
+
+groups() ->
+    TCs = emqx_common_test_helpers:all(?MODULE),
+    [
+        {with_batch, TCs},
+        {without_batch, TCs}
+    ].
+
+init_per_group(with_batch, Config0) ->
+    Config = [{batch_size, ?BATCH_SIZE} | Config0],
+    common_init(Config);
+init_per_group(without_batch, Config0) ->
+    Config = [{batch_size, 1} | Config0],
+    common_init(Config);
+init_per_group(_Group, Config) ->
+    Config.
+
+end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch ->
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    ok;
+end_per_group(_Group, _Config) ->
+    ok.
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_mgmt_api_test_util:end_suite(),
+    ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
+    ok.
+
+init_per_testcase(_Testcase, Config) ->
+    delete_bridge(Config),
+    Config.
+
+end_per_testcase(_Testcase, Config) ->
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    ok = snabbkaffe:stop(),
+    delete_bridge(Config),
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Helper fns
+%%------------------------------------------------------------------------------
+
+common_init(ConfigT) ->
+    BridgeType = <<"rocketmq">>,
+    Host = os:getenv("ROCKETMQ_HOST", "toxiproxy"),
+    Port = list_to_integer(os:getenv("ROCKETMQ_PORT", "9876")),
+
+    Config0 = [
+        {host, Host},
+        {port, Port},
+        {query_mode, sync},
+        {proxy_name, "rocketmq"}
+        | ConfigT
+    ],
+
+    case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of
+        true ->
+            % Setup toxiproxy
+            ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
+            ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
+            emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+            % Ensure EE bridge module is loaded
+            _ = application:load(emqx_ee_bridge),
+            _ = emqx_ee_bridge:module_info(),
+            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+            emqx_mgmt_api_test_util:init_suite(),
+            {Name, RocketMQConf} = rocketmq_config(BridgeType, Config0),
+            Config =
+                [
+                    {rocketmq_config, RocketMQConf},
+                    {rocketmq_bridge_type, BridgeType},
+                    {rocketmq_name, Name},
+                    {proxy_host, ProxyHost},
+                    {proxy_port, ProxyPort}
+                    | Config0
+                ],
+            Config;
+        false ->
+            case os:getenv("IS_CI") of
+                false ->
+                    {skip, no_rocketmq};
+                _ ->
+                    throw(no_rocketmq)
+            end
+    end.
+
+rocketmq_config(BridgeType, Config) ->
+    Port = integer_to_list(?GET_CONFIG(port, Config)),
+    Server = ?GET_CONFIG(host, Config) ++ ":" ++ Port,
+    Name = atom_to_binary(?MODULE),
+    BatchSize = ?config(batch_size, Config),
+    QueryMode = ?config(query_mode, Config),
+    ConfigString =
+        io_lib:format(
+            "bridges.~s.~s {\n"
+            "  enable = true\n"
+            "  server = ~p\n"
+            "  topic = ~p\n"
+            "  resource_opts = {\n"
+            "    request_timeout = 1500ms\n"
+            "    batch_size = ~b\n"
+            "    query_mode = ~s\n"
+            "  }\n"
+            "}",
+            [
+                BridgeType,
+                Name,
+                Server,
+                ?TOPIC,
+                BatchSize,
+                QueryMode
+            ]
+        ),
+    {Name, parse_and_check(ConfigString, BridgeType, Name)}.
+
+parse_and_check(ConfigString, BridgeType, Name) ->
+    {ok, RawConf} = hocon:binary(ConfigString, #{format => map}),
+    hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}),
+    #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf,
+    Config.
+
+create_bridge(Config) ->
+    BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config),
+    Name = ?GET_CONFIG(rocketmq_name, Config),
+    RocketMQConf = ?GET_CONFIG(rocketmq_config, Config),
+    emqx_bridge:create(BridgeType, Name, RocketMQConf).
+
+delete_bridge(Config) ->
+    BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config),
+    Name = ?GET_CONFIG(rocketmq_name, Config),
+    emqx_bridge:remove(BridgeType, Name).
+
+create_bridge_http(Params) ->
+    Path = emqx_mgmt_api_test_util:api_path(["bridges"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of
+        {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
+        Error -> Error
+    end.
+
+send_message(Config, Payload) ->
+    Name = ?GET_CONFIG(rocketmq_name, Config),
+    BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config),
+    BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name),
+    emqx_bridge:send_message(BridgeID, Payload).
+
+query_resource(Config, Request) ->
+    Name = ?GET_CONFIG(rocketmq_name, Config),
+    BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config),
+    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
+    emqx_resource:query(ResourceID, Request, #{timeout => 500}).
+
+%%------------------------------------------------------------------------------
+%% Testcases
+%%------------------------------------------------------------------------------
+
+t_setup_via_config_and_publish(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    SentData = #{payload => ?PAYLOAD},
+    ?check_trace(
+        begin
+            ?wait_async_action(
+                ?assertEqual(ok, send_message(Config, SentData)),
+                #{?snk_kind := rocketmq_connector_query_return},
+                10_000
+            ),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace = ?of_kind(rocketmq_connector_query_return, Trace0),
+            ?assertMatch([#{result := ok}], Trace),
+            ok
+        end
+    ),
+    ok.
+
+t_setup_via_http_api_and_publish(Config) ->
+    BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config),
+    Name = ?GET_CONFIG(rocketmq_name, Config),
+    RocketMQConf = ?GET_CONFIG(rocketmq_config, Config),
+    RocketMQConf2 = RocketMQConf#{
+        <<"name">> => Name,
+        <<"type">> => BridgeType
+    },
+    ?assertMatch(
+        {ok, _},
+        create_bridge_http(RocketMQConf2)
+    ),
+    SentData = #{payload => ?PAYLOAD},
+    ?check_trace(
+        begin
+            ?wait_async_action(
+                ?assertEqual(ok, send_message(Config, SentData)),
+                #{?snk_kind := rocketmq_connector_query_return},
+                10_000
+            ),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace = ?of_kind(rocketmq_connector_query_return, Trace0),
+            ?assertMatch([#{result := ok}], Trace),
+            ok
+        end
+    ),
+    ok.
+
+t_get_status(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+
+    Name = ?GET_CONFIG(rocketmq_name, Config),
+    BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config),
+    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
+
+    ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)),
+    ok.
+
+t_simple_query(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    Request = {send_message, #{message => <<"Hello">>}},
+    Result = query_resource(Config, Request),
+    ?assertEqual(ok, Result),
+    ok.
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl
index d41f83e1d..84f2e2a89 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl
@@ -31,10 +31,6 @@
     default_port => 9876
 }).
 
--ifdef(TEST).
--export([execute/2]).
--endif.
-
 %%=====================================================================
 %% Hocon schema
 roots() ->
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index bf7b2073d..a4eeb366d 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -170,6 +170,9 @@ for dep in ${CT_DEPS}; do
         dynamo)
             FILES+=( '.ci/docker-compose-file/docker-compose-dynamo.yaml' )
             ;;
+        rocketmq)
+            FILES+=( '.ci/docker-compose-file/docker-compose-rocketmq.yaml' )
+            ;; 
         *)
             echo "unknown_ct_dependency $dep"
             exit 1

From 17e207cb71dd5247a2718ff0db6c043b1d038b2c Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Thu, 16 Mar 2023 22:58:47 +0800
Subject: [PATCH 33/52] chore: fix spellcheck && update changes

---
 changes/ee/feat-10143.en.md       | 1 +
 changes/ee/feat-10143.zh.md       | 1 +
 scripts/spellcheck/dicts/emqx.txt | 1 +
 3 files changed, 3 insertions(+)
 create mode 100644 changes/ee/feat-10143.en.md
 create mode 100644 changes/ee/feat-10143.zh.md

diff --git a/changes/ee/feat-10143.en.md b/changes/ee/feat-10143.en.md
new file mode 100644
index 000000000..67fc13dc2
--- /dev/null
+++ b/changes/ee/feat-10143.en.md
@@ -0,0 +1 @@
+Add `RocketMQ` data integration bridge.
diff --git a/changes/ee/feat-10143.zh.md b/changes/ee/feat-10143.zh.md
new file mode 100644
index 000000000..85a13ffa7
--- /dev/null
+++ b/changes/ee/feat-10143.zh.md
@@ -0,0 +1 @@
+为数据桥接增加 `RocketMQ` 支持。
diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt
index b027f92ec..b0d663ead 100644
--- a/scripts/spellcheck/dicts/emqx.txt
+++ b/scripts/spellcheck/dicts/emqx.txt
@@ -271,3 +271,4 @@ nif
 TDengine
 clickhouse
 FormatType
+RocketMQ

From 934e46307242a952db1b108b296ef496c0ef14cc Mon Sep 17 00:00:00 2001
From: Kinplemelon <wujiu529@gmail.com>
Date: Wed, 22 Mar 2023 12:57:39 +0800
Subject: [PATCH 34/52] chore(dashboard): change dashboard repo

---
 scripts/get-dashboard.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh
index c3559865f..ace795aa5 100755
--- a/scripts/get-dashboard.sh
+++ b/scripts/get-dashboard.sh
@@ -20,7 +20,7 @@ case "$VERSION" in
 esac
 
 DASHBOARD_PATH='apps/emqx_dashboard/priv'
-DASHBOARD_REPO='emqx-dashboard-web-new'
+DASHBOARD_REPO='emqx-dashboard5'
 DIRECT_DOWNLOAD_URL="https://github.com/emqx/${DASHBOARD_REPO}/releases/download/${VERSION}/${RELEASE_ASSET_FILE}"
 
 case $(uname) in

From 5d31f85aec0efcfa95f40a0f333fc412ff70c530 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Wed, 22 Mar 2023 11:43:25 +0300
Subject: [PATCH 35/52] chore: remove irrelevant changelog entry

---
 changes/ce/fix-10190.en.md | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 changes/ce/fix-10190.en.md

diff --git a/changes/ce/fix-10190.en.md b/changes/ce/fix-10190.en.md
deleted file mode 100644
index bffd9ca00..000000000
--- a/changes/ce/fix-10190.en.md
+++ /dev/null
@@ -1 +0,0 @@
-Fix the issue where nodes responses to the list bridges RPC were incorrectly flattened, which caused List Bridges API HTTP handler to crash when there was more than 1 node in the cluster.

From 208813330a6f5ea7a997cf104ba9edb0bcadeeea Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Wed, 22 Mar 2023 10:59:20 -0300
Subject: [PATCH 36/52] fix: remove metrics from rocketmq example

---
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl
index d81ffc54c..124e18069 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl
@@ -39,7 +39,7 @@ conn_bridge_examples(Method) ->
     ].
 
 values(get) ->
-    maps:merge(values(post), ?METRICS_EXAMPLE);
+    values(post);
 values(post) ->
     #{
         enable => true,

From 61cb03b45a427fe361a79edc807d5e1e791de132 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Tue, 21 Mar 2023 14:32:50 -0300
Subject: [PATCH 37/52] fix(buffer_worker): change the default
 `resume_interval` value and expose it as hidden config

Also removes the previously added alarm for request timeout.

There are situations where having a short request timeout and a long
health check interval make sense, so we don't want to alarm the user
for those situations.  Instead, we automatically attempt to set a
reasonable `resume_interval` value.
---
 .../i18n/emqx_resource_schema_i18n.conf       |  15 ++-
 apps/emqx_resource/include/emqx_resource.hrl  |   5 +-
 .../src/emqx_resource_buffer_worker.erl       |  49 ++-----
 .../src/emqx_resource_manager.erl             |   1 -
 .../src/schema/emqx_resource_schema.erl       |   7 +
 .../test/emqx_resource_SUITE.erl              | 126 ------------------
 changes/ce/fix-10154.en.md                    |   7 +-
 .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl  |  14 +-
 8 files changed, 50 insertions(+), 174 deletions(-)

diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index 2e5cf96e8..eb4f00ac7 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -45,6 +45,17 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise
     }
   }
 
+  resume_interval {
+    desc {
+      en: """The interval at which a resource will retry inflight requests."""
+      zh: """资源重试机内请求的间隔时间。"""
+    }
+    label {
+      en: """Resume Interval"""
+      zh: """复职时间间隔"""
+    }
+  }
+
   start_after_created {
     desc {
       en: """Whether start the resource right after created."""
@@ -102,8 +113,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise
 
   request_timeout {
     desc {
-      en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.  We recommend setting this timeout to be at least twice the health check interval, so that the buffer has the chance to recover if too many requests get enqueued."""
-      zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。我们建议将这个超时设置为健康检查间隔的至少两倍，这样，如果有太多的请求被排队，缓冲区就有机会恢复。"""
+      en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired."""
+      zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。"""
     }
     label {
       en: """Request Expiry"""
diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 8033ed660..be570e694 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -91,10 +91,7 @@
 -define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024).
 -define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>).
 
-%% Note: this should be greater than the health check timeout;
-%% otherwise, if the buffer worker is ever blocked, than all queued
-%% requests will basically fail without being attempted.
--define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(30)).
+-define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)).
 
 %% count
 -define(DEFAULT_BATCH_SIZE, 1).
diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
index 05622bdd7..648587c25 100644
--- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
@@ -56,8 +56,6 @@
 
 -export([clear_disk_queue_dir/2]).
 
--export([deactivate_bad_request_timeout_alarm/1]).
-
 -elvis([{elvis_style, dont_repeat_yourself, disable}]).
 
 -define(COLLECT_REQ_LIMIT, 1000).
@@ -203,7 +201,8 @@ init({Id, Index, Opts}) ->
     RequestTimeout = maps:get(request_timeout, Opts, ?DEFAULT_REQUEST_TIMEOUT),
     BatchTime0 = maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME),
     BatchTime = adjust_batch_time(Id, RequestTimeout, BatchTime0),
-    maybe_toggle_bad_request_timeout_alarm(Id, RequestTimeout, HealthCheckInterval),
+    DefaultResumeInterval = default_resume_interval(RequestTimeout, HealthCheckInterval),
+    ResumeInterval = maps:get(resume_interval, Opts, DefaultResumeInterval),
     Data = #{
         id => Id,
         index => Index,
@@ -212,7 +211,7 @@ init({Id, Index, Opts}) ->
         batch_size => BatchSize,
         batch_time => BatchTime,
         queue => Queue,
-        resume_interval => maps:get(resume_interval, Opts, HealthCheckInterval),
+        resume_interval => ResumeInterval,
         tref => undefined
     },
     ?tp(buffer_worker_init, #{id => Id, index => Index}),
@@ -1684,38 +1683,16 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) ->
     end,
     BatchTime.
 
-%% The request timeout should be greater than the health check
-%% timeout, health timeout defines how often the buffer worker tries
-%% to unblock. If request timeout is <= health check timeout and the
-%% buffer worker is ever blocked, than all queued requests will
-%% basically fail without being attempted.
--spec maybe_toggle_bad_request_timeout_alarm(
-    resource_id(), request_timeout(), health_check_interval()
-) -> ok.
-maybe_toggle_bad_request_timeout_alarm(Id, _RequestTimeout = infinity, _HealthCheckInterval) ->
-    deactivate_bad_request_timeout_alarm(Id),
-    ok;
-maybe_toggle_bad_request_timeout_alarm(Id, RequestTimeout, HealthCheckInterval) ->
-    case RequestTimeout > HealthCheckInterval of
-        true ->
-            deactivate_bad_request_timeout_alarm(Id),
-            ok;
-        false ->
-            _ = emqx_alarm:activate(
-                bad_request_timeout_alarm_id(Id),
-                #{resource_id => Id, reason => bad_request_timeout},
-                <<"Request timeout should be greater than health check timeout: ", Id/binary>>
-            ),
-            ok
-    end.
-
--spec deactivate_bad_request_timeout_alarm(resource_id()) -> ok.
-deactivate_bad_request_timeout_alarm(Id) ->
-    _ = emqx_alarm:ensure_deactivated(bad_request_timeout_alarm_id(Id)),
-    ok.
-
-bad_request_timeout_alarm_id(Id) ->
-    <<"bad_request_timeout:", Id/binary>>.
+%% The request timeout should be greater than the resume interval, as
+%% it defines how often the buffer worker tries to unblock. If request
+%% timeout is <= resume interval and the buffer worker is ever
+%% blocked, than all queued requests will basically fail without being
+%% attempted.
+-spec default_resume_interval(request_timeout(), health_check_interval()) -> timer:time().
+default_resume_interval(_RequestTimeout = infinity, HealthCheckInterval) ->
+    max(1, HealthCheckInterval);
+default_resume_interval(RequestTimeout, HealthCheckInterval) ->
+    max(1, min(HealthCheckInterval, RequestTimeout div 3)).
 
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 2bdc67a4d..40f9fe1ab 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -506,7 +506,6 @@ handle_remove_event(From, ClearMetrics, Data) ->
         true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id);
         false -> ok
     end,
-    emqx_resource_buffer_worker:deactivate_bad_request_timeout_alarm(Data#data.id),
     {stop_and_reply, {shutdown, removed}, [{reply, From, ok}]}.
 
 start_resource(Data, From) ->
diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
index fdd65bc3c..b9ed176fe 100644
--- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl
+++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl
@@ -55,6 +55,7 @@ fields("creation_opts") ->
     [
         {worker_pool_size, fun worker_pool_size/1},
         {health_check_interval, fun health_check_interval/1},
+        {resume_interval, fun resume_interval/1},
         {start_after_created, fun start_after_created/1},
         {start_timeout, fun start_timeout/1},
         {auto_restart_interval, fun auto_restart_interval/1},
@@ -81,6 +82,12 @@ worker_pool_size(default) -> ?WORKER_POOL_SIZE;
 worker_pool_size(required) -> false;
 worker_pool_size(_) -> undefined.
 
+resume_interval(type) -> emqx_schema:duration_ms();
+resume_interval(hidden) -> true;
+resume_interval(desc) -> ?DESC("resume_interval");
+resume_interval(required) -> false;
+resume_interval(_) -> undefined.
+
 health_check_interval(type) -> emqx_schema:duration_ms();
 health_check_interval(desc) -> ?DESC("health_check_interval");
 health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW;
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index 25f4a6d77..ff7e1d347 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -2554,132 +2554,6 @@ do_t_recursive_flush() ->
     ),
     ok.
 
-%% Check that we raise an alarm if a bad request timeout config is
-%% issued.  Request timeout should be greater than health check
-%% timeout.
-t_bad_request_timeout_alarm(_Config) ->
-    emqx_connector_demo:set_callback_mode(async_if_possible),
-
-    %% 1) Same values.
-    {ok, _} = emqx_resource:create(
-        ?ID,
-        ?DEFAULT_RESOURCE_GROUP,
-        ?TEST_RESOURCE,
-        #{name => test_resource},
-        #{
-            query_mode => async,
-            request_timeout => 1_000,
-            health_check_interval => 1_000,
-            worker_pool_size => 2
-        }
-    ),
-    ExpectedMessage =
-        <<"Request timeout should be greater than health check timeout: ", ?ID/binary>>,
-    ?assertMatch(
-        [
-            #{
-                message := ExpectedMessage,
-                details := #{reason := bad_request_timeout, resource_id := ?ID},
-                deactivate_at := infinity
-            }
-        ],
-        emqx_alarm:get_alarms(activated)
-    ),
-    %% The unexpected termination of one of the buffer workers should
-    %% not turn the alarm off.
-    [Pid, _ | _] = emqx_resource_buffer_worker_sup:worker_pids(?ID),
-    MRef = monitor(process, Pid),
-    exit(Pid, kill),
-    receive
-        {'DOWN', MRef, process, Pid, _} ->
-            ok
-    after 300 ->
-        ct:fail("buffer worker didn't die")
-    end,
-    ?assertMatch(
-        [
-            #{
-                message := ExpectedMessage,
-                details := #{reason := bad_request_timeout, resource_id := ?ID},
-                deactivate_at := infinity
-            }
-        ],
-        emqx_alarm:get_alarms(activated)
-    ),
-    ok = emqx_resource:remove_local(?ID),
-    ?assertEqual([], emqx_alarm:get_alarms(activated)),
-
-    %% 2) Request timeout < health check interval.
-    {ok, _} = emqx_resource:create(
-        ?ID,
-        ?DEFAULT_RESOURCE_GROUP,
-        ?TEST_RESOURCE,
-        #{name => test_resource},
-        #{
-            query_mode => async,
-            request_timeout => 999,
-            health_check_interval => 1_000,
-            worker_pool_size => 2
-        }
-    ),
-    ?assertMatch(
-        [
-            #{
-                message := ExpectedMessage,
-                details := #{reason := bad_request_timeout, resource_id := ?ID},
-                deactivate_at := infinity
-            }
-        ],
-        emqx_alarm:get_alarms(activated)
-    ),
-    ok = emqx_resource:remove_local(?ID),
-    ?assertEqual([], emqx_alarm:get_alarms(activated)),
-
-    %% 2) Request timeout < health check interval.
-    {ok, _} = emqx_resource:create(
-        ?ID,
-        ?DEFAULT_RESOURCE_GROUP,
-        ?TEST_RESOURCE,
-        #{name => test_resource},
-        #{
-            query_mode => async,
-            request_timeout => 999,
-            health_check_interval => 1_000,
-            worker_pool_size => 2
-        }
-    ),
-    ?assertMatch(
-        [
-            #{
-                message := ExpectedMessage,
-                details := #{reason := bad_request_timeout, resource_id := ?ID},
-                deactivate_at := infinity
-            }
-        ],
-        emqx_alarm:get_alarms(activated)
-    ),
-    ok = emqx_resource:remove_local(?ID),
-    ?assertEqual([], emqx_alarm:get_alarms(activated)),
-
-    %% 3) Request timeout > health check interval.
-    {ok, _} = emqx_resource:create(
-        ?ID,
-        ?DEFAULT_RESOURCE_GROUP,
-        ?TEST_RESOURCE,
-        #{name => test_resource},
-        #{
-            query_mode => async,
-            request_timeout => 1_001,
-            health_check_interval => 1_000,
-            worker_pool_size => 2
-        }
-    ),
-    ?assertEqual([], emqx_alarm:get_alarms(activated)),
-    ok = emqx_resource:remove_local(?ID),
-    ?assertEqual([], emqx_alarm:get_alarms(activated)),
-
-    ok.
-
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------
diff --git a/changes/ce/fix-10154.en.md b/changes/ce/fix-10154.en.md
index 83a729360..24bc4bae1 100644
--- a/changes/ce/fix-10154.en.md
+++ b/changes/ce/fix-10154.en.md
@@ -1,7 +1,8 @@
-Change the default `request_timeout` for bridges and connectors to be
-twice the default `health_check_interval`.
+Change the default `resume_interval` for bridges and connectors to be
+the minimum of `health_check_interval` and `request_timeout / 3`.
+Also exposes it as a hidden configuration to allow fine tuning.
 
-Before this change, the default values for those two options meant
+Before this change, the default values for `resume_interval` meant
 that, if a buffer ever got blocked due to resource errors or high
 message volumes, then, by the time the buffer would try to resume its
 normal operations, almost all requests would have timed out.
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl
index 452b7a4d2..55dfa5555 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl
@@ -520,6 +520,7 @@ wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) ->
         #{measurements := #{gauge_set := ExpectedValue}} ->
             ok;
         #{measurements := #{gauge_set := Value}} ->
+            ct:pal("events: ~p", [Events]),
             ct:fail(
                 "gauge ~p didn't reach expected value ~p; last value: ~p",
                 [GaugeName, ExpectedValue, Value]
@@ -972,7 +973,13 @@ t_publish_econnrefused(Config) ->
     ResourceId = ?config(resource_id, Config),
     %% set pipelining to 1 so that one of the 2 requests is `pending'
     %% in ehttpc.
-    {ok, _} = create_bridge(Config, #{<<"pipelining">> => 1}),
+    {ok, _} = create_bridge(
+        Config,
+        #{
+            <<"pipelining">> => 1,
+            <<"resource_opts">> => #{<<"resume_interval">> => <<"15s">>}
+        }
+    ),
     {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config),
     on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end),
     assert_empty_metrics(ResourceId),
@@ -986,7 +993,10 @@ t_publish_timeout(Config) ->
     %% requests are done separately.
     {ok, _} = create_bridge(Config, #{
         <<"pipelining">> => 1,
-        <<"resource_opts">> => #{<<"batch_size">> => 1}
+        <<"resource_opts">> => #{
+            <<"batch_size">> => 1,
+            <<"resume_interval">> => <<"15s">>
+        }
     }),
     {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config),
     on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end),

From 2662f79ec0ce26124664227a357fa3952271c998 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 22 Mar 2023 16:29:51 +0100
Subject: [PATCH 38/52] feat(emqx_dashboard): use label as summary in OpenAPI
 spec

---
 apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
index 77fcd4f76..2290dbedb 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
@@ -457,7 +457,18 @@ trans_description(Spec, Hocon) ->
             Spec;
         Desc ->
             Desc1 = binary:replace(Desc, [<<"\n">>], <<"<br/>">>, [global]),
-            Spec#{description => Desc1}
+            maybe_add_summary_from_label(Spec#{description => Desc1}, Hocon)
+    end.
+
+maybe_add_summary_from_label(Spec, Hocon) ->
+    Label =
+        case desc_struct(Hocon) of
+            ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, undefined);
+            _ -> undefined
+        end,
+    case Label of
+        undefined -> Spec;
+        _ -> Spec#{summary => Label}
     end.
 
 get_i18n(Key, Struct, Default) ->

From 944d456f944ec2b3df863ce16c0cdfeb7d4e119f Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 22 Mar 2023 16:32:34 +0100
Subject: [PATCH 39/52] chore: add changelog

---
 changes/ce/feat-10207.en.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/ce/feat-10207.en.md

diff --git a/changes/ce/feat-10207.en.md b/changes/ce/feat-10207.en.md
new file mode 100644
index 000000000..99ca17944
--- /dev/null
+++ b/changes/ce/feat-10207.en.md
@@ -0,0 +1 @@
+Use 'label' from i18n file as 'summary' in OpenAPI spec.

From 127a075b66c469b0e873934c3b389ed2ebb73aca Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Wed, 22 Mar 2023 14:34:37 -0300
Subject: [PATCH 40/52] test(dynamo): attempt to fix dynamo tests

Those tests in the `flaky` test are really flaky and require lots of
CI retries.

Apparently, the flakiness comes from race conditions from restarting
bridges with the same name too fast between test cases.  Previously,
all test cases were sharing the same bridge name (the module name).
---
 .../src/emqx_resource_manager.erl             |  1 +
 .../test/emqx_ee_bridge_dynamo_SUITE.erl      | 50 +++++++++++++------
 2 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index b21ffcae3..6a4919b41 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -388,6 +388,7 @@ handle_event(state_timeout, health_check, connecting, Data) ->
 handle_event(enter, _OldState, connected = State, Data) ->
     ok = log_state_consistency(State, Data),
     _ = emqx_alarm:deactivate(Data#data.id),
+    ?tp(resource_connected_enter, #{}),
     {keep_state_and_data, health_check_actions(Data)};
 handle_event(state_timeout, health_check, connected, Data) ->
     handle_connected_health_check(Data);
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl
index 26666c6d8..183002e61 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl
@@ -83,9 +83,10 @@ end_per_suite(_Config) ->
     ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
     ok.
 
-init_per_testcase(_Testcase, Config) ->
+init_per_testcase(TestCase, Config) ->
     create_table(Config),
-    Config.
+    ok = snabbkaffe:start_trace(),
+    [{dynamo_name, atom_to_binary(TestCase)} | Config].
 
 end_per_testcase(_Testcase, Config) ->
     ProxyHost = ?config(proxy_host, Config),
@@ -93,7 +94,7 @@ end_per_testcase(_Testcase, Config) ->
     emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
     ok = snabbkaffe:stop(),
     delete_table(Config),
-    delete_bridge(Config),
+    delete_all_bridges(),
     ok.
 
 %%------------------------------------------------------------------------------
@@ -186,15 +187,22 @@ parse_and_check(ConfigString, BridgeType, Name) ->
     Config.
 
 create_bridge(Config) ->
-    BridgeType = ?config(dynamo_bridge_type, Config),
-    Name = ?config(dynamo_name, Config),
-    TDConfig = ?config(dynamo_config, Config),
-    emqx_bridge:create(BridgeType, Name, TDConfig).
+    create_bridge(Config, _Overrides = #{}).
 
-delete_bridge(Config) ->
+create_bridge(Config, Overrides) ->
     BridgeType = ?config(dynamo_bridge_type, Config),
     Name = ?config(dynamo_name, Config),
-    emqx_bridge:remove(BridgeType, Name).
+    DynamoConfig0 = ?config(dynamo_config, Config),
+    DynamoConfig = emqx_map_lib:deep_merge(DynamoConfig0, Overrides),
+    emqx_bridge:create(BridgeType, Name, DynamoConfig).
+
+delete_all_bridges() ->
+    lists:foreach(
+        fun(#{name := Name, type := Type}) ->
+            emqx_bridge:remove(Type, Name)
+        end,
+        emqx_bridge:list()
+    ).
 
 create_bridge_http(Params) ->
     Path = emqx_mgmt_api_test_util:api_path(["bridges"]),
@@ -327,10 +335,12 @@ t_setup_via_http_api_and_publish(Config) ->
     ok.
 
 t_get_status(Config) ->
-    ?assertMatch(
-        {ok, _},
-        create_bridge(Config)
-    ),
+    {{ok, _}, {ok, _}} =
+        ?wait_async_action(
+            create_bridge(Config),
+            #{?snk_kind := resource_connected_enter},
+            20_000
+        ),
 
     ProxyPort = ?config(proxy_port, Config),
     ProxyHost = ?config(proxy_host, Config),
@@ -359,7 +369,12 @@ t_write_failure(Config) ->
     ProxyName = ?config(proxy_name, Config),
     ProxyPort = ?config(proxy_port, Config),
     ProxyHost = ?config(proxy_host, Config),
-    {ok, _} = create_bridge(Config),
+    {{ok, _}, {ok, _}} =
+        ?wait_async_action(
+            create_bridge(Config),
+            #{?snk_kind := resource_connected_enter},
+            20_000
+        ),
     SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD},
     emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
         ?assertMatch(
@@ -372,7 +387,12 @@ t_write_timeout(Config) ->
     ProxyName = ?config(proxy_name, Config),
     ProxyPort = ?config(proxy_port, Config),
     ProxyHost = ?config(proxy_host, Config),
-    {ok, _} = create_bridge(Config),
+    {{ok, _}, {ok, _}} =
+        ?wait_async_action(
+            create_bridge(Config),
+            #{?snk_kind := resource_connected_enter},
+            20_000
+        ),
     SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD},
     emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() ->
         ?assertMatch(

From 8844b22c809ae7495243ed8f9164d84cf6303513 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Wed, 22 Mar 2023 15:32:09 -0300
Subject: [PATCH 41/52] docs: improve descriptions

Co-authored-by: Zaiming (Stone) Shi <zmstone@gmail.com>
---
 apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
index eb4f00ac7..aedcabc70 100644
--- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
+++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf
@@ -47,12 +47,12 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise
 
   resume_interval {
     desc {
-      en: """The interval at which a resource will retry inflight requests."""
-      zh: """资源重试机内请求的间隔时间。"""
+      en: """The interval at which the buffer worker attempts to resend failed requests in the inflight window."""
+      zh: """在发送失败后尝试重传飞行窗口中的请求的时间间隔。"""
     }
     label {
       en: """Resume Interval"""
-      zh: """复职时间间隔"""
+      zh: """重试时间间隔"""
     }
   }
 

From cb65cded8825eba47d5fde6f4542e66af7807b04 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Wed, 22 Mar 2023 15:22:13 -0300
Subject: [PATCH 42/52] fix(last_will_testament): don't publish LWT if client
 is banned when kicked

Fixes https://emqx.atlassian.net/browse/EMQX-9288

Related issue:
https://github.com/emqx/emqx/issues/10192#issuecomment-1478809900
---
 apps/emqx/src/emqx_channel.erl            | 20 ++++---
 apps/emqx_authz/test/emqx_authz_SUITE.erl | 65 +++++++++++++++++++++++
 changes/ce/fix-10209.en.md                |  2 +
 3 files changed, 80 insertions(+), 7 deletions(-)
 create mode 100644 changes/ce/fix-10209.en.md

diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index 9acad4d57..e01a16f83 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -2128,17 +2128,23 @@ publish_will_msg(
     ClientInfo = #{mountpoint := MountPoint},
     Msg = #message{topic = Topic}
 ) ->
-    case emqx_access_control:authorize(ClientInfo, publish, Topic) of
-        allow ->
-            NMsg = emqx_mountpoint:mount(MountPoint, Msg),
-            _ = emqx_broker:publish(NMsg),
-            ok;
-        deny ->
+    PublishingDisallowed = emqx_access_control:authorize(ClientInfo, publish, Topic) =/= allow,
+    ClientBanned = emqx_banned:check(ClientInfo),
+    case PublishingDisallowed orelse ClientBanned of
+        true ->
             ?tp(
                 warning,
                 last_will_testament_publish_denied,
-                #{topic => Topic}
+                #{
+                    topic => Topic,
+                    client_banned => ClientBanned,
+                    publishing_disallowed => PublishingDisallowed
+                }
             ),
+            ok;
+        false ->
+            NMsg = emqx_mountpoint:mount(MountPoint, Msg),
+            _ = emqx_broker:publish(NMsg),
             ok
     end.
 
diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl
index b3ce04f43..84b1d903e 100644
--- a/apps/emqx_authz/test/emqx_authz_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl
@@ -26,6 +26,8 @@
 -include_lib("emqx/include/emqx_placeholder.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
+-import(emqx_common_test_helpers, [on_exit/1]).
+
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
@@ -65,6 +67,7 @@ end_per_suite(_Config) ->
 
 init_per_testcase(TestCase, Config) when
     TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
+    TestCase =:= t_publish_last_will_testament_banned_client_connecting;
     TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
 ->
     {ok, _} = emqx_authz:update(?CMD_REPLACE, []),
@@ -76,11 +79,15 @@ init_per_testcase(_, Config) ->
 
 end_per_testcase(TestCase, _Config) when
     TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
+    TestCase =:= t_publish_last_will_testament_banned_client_connecting;
     TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
 ->
     {ok, _} = emqx:update_config([authorization, deny_action], ignore),
+    {ok, _} = emqx_authz:update(?CMD_REPLACE, []),
+    emqx_common_test_helpers:call_janitor(),
     ok;
 end_per_testcase(_TestCase, _Config) ->
+    emqx_common_test_helpers:call_janitor(),
     ok.
 
 set_special_configs(emqx_authz) ->
@@ -396,5 +403,63 @@ t_publish_last_will_testament_denied_topic(_Config) ->
 
     ok.
 
+%% client is allowed by ACL to publish to its LWT topic, is connected,
+%% and then gets banned and kicked out while connected.  Should not
+%% publish LWT.
+t_publish_last_will_testament_banned_client_connecting(_Config) ->
+    {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE7]),
+    Username = <<"some_client">>,
+    ClientId = <<"some_clientid">>,
+    LWTPayload = <<"should not be published">>,
+    LWTTopic = <<"some_client/lwt">>,
+    ok = emqx:subscribe(<<"some_client/lwt">>),
+    {ok, C} = emqtt:start_link([
+        {clientid, ClientId},
+        {username, Username},
+        {will_topic, LWTTopic},
+        {will_payload, LWTPayload}
+    ]),
+    ?assertMatch({ok, _}, emqtt:connect(C)),
+
+    %% Now we ban the client while it is connected.
+    Now = erlang:system_time(second),
+    Who = {username, Username},
+    emqx_banned:create(#{
+        who => Who,
+        by => <<"test">>,
+        reason => <<"test">>,
+        at => Now,
+        until => Now + 120
+    }),
+    on_exit(fun() -> emqx_banned:delete(Who) end),
+    %% Now kick it as we do in the ban API.
+    process_flag(trap_exit, true),
+    ?check_trace(
+        begin
+            ok = emqx_cm:kick_session(ClientId),
+            receive
+                {deliver, LWTTopic, #message{payload = LWTPayload}} ->
+                    error(lwt_should_not_be_published_to_forbidden_topic)
+            after 2_000 -> ok
+            end,
+            ok
+        end,
+        fun(Trace) ->
+            ?assertMatch(
+                [
+                    #{
+                        client_banned := true,
+                        publishing_disallowed := false
+                    }
+                ],
+                ?of_kind(last_will_testament_publish_denied, Trace)
+            ),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+
+    ok.
+
 stop_apps(Apps) ->
     lists:foreach(fun application:stop/1, Apps).
diff --git a/changes/ce/fix-10209.en.md b/changes/ce/fix-10209.en.md
new file mode 100644
index 000000000..21ce98e44
--- /dev/null
+++ b/changes/ce/fix-10209.en.md
@@ -0,0 +1,2 @@
+Fix bug where a last will testament (LWT) message could be published
+when kicking out a banned client.

From 761847b74aca24f78fe16f10a75c40e154153047 Mon Sep 17 00:00:00 2001
From: Serge Tupchii <serge.tupchii@protonmail.com>
Date: Wed, 22 Mar 2023 15:09:13 +0200
Subject: [PATCH 43/52] ci: update emqx-builders with OTP 24.3.4.2-3 and
 25.1.2-3

These OTP versions implement mnesia_hook:unregister_hook/1
---
 .../docker-compose-kafka.yaml                    |  2 +-
 .ci/docker-compose-file/docker-compose.yaml      |  2 +-
 .../workflows/build_and_push_docker_images.yaml  |  6 +++---
 .github/workflows/build_packages.yaml            | 16 ++++++++--------
 .github/workflows/build_slim_packages.yaml       | 12 ++++++------
 .github/workflows/check_deps_integrity.yaml      |  2 +-
 .github/workflows/code_style_check.yaml          |  2 +-
 .github/workflows/elixir_apps_check.yaml         |  2 +-
 .github/workflows/elixir_deps_check.yaml         |  2 +-
 .github/workflows/elixir_release.yml             |  2 +-
 .github/workflows/release.yaml                   |  2 +-
 .github/workflows/run_emqx_app_tests.yaml        |  6 +++---
 .github/workflows/run_fvt_tests.yaml             | 10 +++++-----
 .github/workflows/run_relup_tests.yaml           |  2 +-
 .github/workflows/run_test_cases.yaml            | 12 ++++++------
 .tool-versions                                   |  2 +-
 deploy/docker/Dockerfile                         |  2 +-
 scripts/relup-test/run-relup-lux.sh              |  4 ++--
 scripts/relup-test/start-relup-test-cluster.sh   |  2 +-
 19 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index d4989bd0b..bbfb4080a 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -18,7 +18,7 @@ services:
       - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret
   kdc:
     hostname: kdc.emqx.net
-    image:  ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04
+    image:  ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu20.04
     container_name: kdc.emqx.net
     expose:
       - 88 # kdc
diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml
index 5c37d971e..2d1c553e9 100644
--- a/.ci/docker-compose-file/docker-compose.yaml
+++ b/.ci/docker-compose-file/docker-compose.yaml
@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04}
+    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu20.04}
     env_file:
       - conf.env
     environment:
diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml
index adf2c2b84..7391adb5c 100644
--- a/.github/workflows/build_and_push_docker_images.yaml
+++ b/.github/workflows/build_and_push_docker_images.yaml
@@ -25,7 +25,7 @@ jobs:
   prepare:
     runs-on: ubuntu-22.04
     # prepare source with any OTP version, no need for a matrix
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04"
 
     outputs:
       PROFILE: ${{ steps.get_profile.outputs.PROFILE }}
@@ -121,9 +121,9 @@ jobs:
         # NOTE: 'otp' and 'elixir' are to configure emqx-builder image
         #       only support latest otp and elixir, not a matrix
         builder:
-          - 5.0-32 # update to latest
+          - 5.0-33 # update to latest
         otp:
-          - 24.3.4.2-2 # switch to 25 once ready to release 5.1
+          - 24.3.4.2-3 # switch to 25 once ready to release 5.1
         elixir:
           - 'no_elixir'
           - '1.13.4' # update to latest
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 3141b77d5..2afe23f67 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -24,7 +24,7 @@ jobs:
   prepare:
     runs-on: ubuntu-22.04
     if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule'
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04
     outputs:
       BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
       IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
@@ -151,7 +151,7 @@ jobs:
         profile:
           - ${{ needs.prepare.outputs.BUILD_PROFILE }}
         otp:
-          - 24.3.4.2-2
+          - 24.3.4.2-3
         os:
           - macos-11
           - macos-12
@@ -203,7 +203,7 @@ jobs:
         profile:
           - ${{ needs.prepare.outputs.BUILD_PROFILE }}
         otp:
-          - 24.3.4.2-2
+          - 24.3.4.2-3
         arch:
           - amd64
           - arm64
@@ -221,7 +221,7 @@ jobs:
           - aws-arm64
           - ubuntu-22.04
         builder:
-          - 5.0-32
+          - 5.0-33
         elixir:
           - 1.13.4
         exclude:
@@ -231,19 +231,19 @@ jobs:
           build_machine: aws-arm64
         include:
           - profile: emqx
-            otp: 25.1.2-2
+            otp: 25.1.2-3
             arch: amd64
             os: ubuntu22.04
             build_machine: ubuntu-22.04
-            builder: 5.0-32
+            builder: 5.0-33
             elixir: 1.13.4
             release_with: elixir
           - profile: emqx
-            otp: 25.1.2-2
+            otp: 25.1.2-3
             arch: amd64
             os: amzn2
             build_machine: ubuntu-22.04
-            builder: 5.0-32
+            builder: 5.0-33
             elixir: 1.13.4
             release_with: elixir
 
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 163956790..30af8bdbc 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -30,12 +30,12 @@ jobs:
       fail-fast: false
       matrix:
         profile:
-          - ["emqx", "24.3.4.2-2", "el7", "erlang"]
-          - ["emqx", "25.1.2-2", "ubuntu22.04", "elixir"]
-          - ["emqx-enterprise", "24.3.4.2-2", "amzn2", "erlang"]
-          - ["emqx-enterprise", "25.1.2-2", "ubuntu20.04", "erlang"]
+          - ["emqx", "24.3.4.2-3", "el7", "erlang"]
+          - ["emqx", "25.1.2-3", "ubuntu22.04", "elixir"]
+          - ["emqx-enterprise", "24.3.4.2-3", "amzn2", "erlang"]
+          - ["emqx-enterprise", "25.1.2-3", "ubuntu20.04", "erlang"]
         builder:
-          - 5.0-32
+          - 5.0-33
         elixir:
           - '1.13.4'
 
@@ -132,7 +132,7 @@ jobs:
         - emqx
         - emqx-enterprise
         otp:
-        - 24.3.4.2-2
+        - 24.3.4.2-3
         os:
         - macos-11
         - macos-12-arm64
diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml
index 58dd06e30..62dfa24ef 100644
--- a/.github/workflows/check_deps_integrity.yaml
+++ b/.github/workflows/check_deps_integrity.yaml
@@ -6,7 +6,7 @@ on:
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-22.04
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04
 
     steps:
       - uses: actions/checkout@v3
diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml
index de05f7e59..97c6b0c88 100644
--- a/.github/workflows/code_style_check.yaml
+++ b/.github/workflows/code_style_check.yaml
@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   code_style_check:
     runs-on: ubuntu-22.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04"
     steps:
       - uses: actions/checkout@v3
         with:
diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml
index 181e81305..247f67a8f 100644
--- a/.github/workflows/elixir_apps_check.yaml
+++ b/.github/workflows/elixir_apps_check.yaml
@@ -9,7 +9,7 @@ jobs:
   elixir_apps_check:
     runs-on: ubuntu-22.04
     # just use the latest builder
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04"
 
     strategy:
       fail-fast: false
diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml
index d753693cc..511639a3c 100644
--- a/.github/workflows/elixir_deps_check.yaml
+++ b/.github/workflows/elixir_deps_check.yaml
@@ -8,7 +8,7 @@ on:
 jobs:
   elixir_deps_check:
     runs-on: ubuntu-22.04
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04
 
     steps:
       - name: Checkout
diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml
index 1647071af..7bd6102ff 100644
--- a/.github/workflows/elixir_release.yml
+++ b/.github/workflows/elixir_release.yml
@@ -17,7 +17,7 @@ jobs:
         profile:
           - emqx
           - emqx-enterprise
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 2f5ddf171..32a45bd51 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -54,7 +54,7 @@ jobs:
           OUTPUT_DIR=${{ steps.profile.outputs.s3dir }}
           aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages
           cd packages
-          DEFAULT_BEAM_PLATFORM='otp24.3.4.2-2'
+          DEFAULT_BEAM_PLATFORM='otp24.3.4.2-3'
           # all packages including full-name and default-name are uploaded to s3
           # but we only upload default-name packages (and elixir) as github artifacts
           # so we rename (overwrite) non-default packages before uploading
diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml
index 52ba13373..0a15f6c0b 100644
--- a/.github/workflows/run_emqx_app_tests.yaml
+++ b/.github/workflows/run_emqx_app_tests.yaml
@@ -12,10 +12,10 @@ jobs:
     strategy:
       matrix:
         builder:
-          - 5.0-32
+          - 5.0-33
         otp:
-          - 24.3.4.2-2
-          - 25.1.2-2
+          - 24.3.4.2-3
+          - 25.1.2-3
         # no need to use more than 1 version of Elixir, since tests
         # run using only Erlang code.  This is needed just to specify
         # the base image.
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index f729c8cbd..bb5aa4a1a 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -17,7 +17,7 @@ jobs:
   prepare:
     runs-on: ubuntu-22.04
     # prepare source with any OTP version, no need for a matrix
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-debian11
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-debian11
 
     steps:
       - uses: actions/checkout@v3
@@ -50,9 +50,9 @@ jobs:
         os:
           - ["debian11", "debian:11-slim"]
         builder:
-          - 5.0-32
+          - 5.0-33
         otp:
-          - 24.3.4.2-2
+          - 24.3.4.2-3
         elixir:
           - 1.13.4
         arch:
@@ -123,9 +123,9 @@ jobs:
         os:
         - ["debian11", "debian:11-slim"]
         builder:
-        - 5.0-32
+        - 5.0-33
         otp:
-        - 24.3.4.2-2
+        - 24.3.4.2-3
         elixir:
         - 1.13.4
         arch:
diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index cd969045d..8727f4d9d 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -15,7 +15,7 @@ concurrency:
 jobs:
   relup_test_plan:
     runs-on: ubuntu-22.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04"
     outputs:
       CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
       OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }}
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 1efe7a4e7..8702cd849 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -31,13 +31,13 @@ jobs:
           MATRIX="$(echo "${APPS}" | jq -c '
             [
               (.[] | select(.profile == "emqx") | . + {
-                builder: "5.0-32",
-                otp: "25.1.2-2",
+                builder: "5.0-33",
+                otp: "25.1.2-3",
                 elixir: "1.13.4"
               }),
               (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.0-32",
-                otp: ["24.3.4.2-2", "25.1.2-2"][],
+                builder: "5.0-33",
+                otp: ["24.3.4.2-3", "25.1.2-3"][],
                 elixir: "1.13.4"
               })
             ]
@@ -230,12 +230,12 @@ jobs:
       - ct
       - ct_docker
     runs-on: ubuntu-22.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04"
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: source-emqx-enterprise-24.3.4.2-2
+          name: source-emqx-enterprise-24.3.4.2-3
           path: .
       - name: unzip source code
         run: unzip -q source.zip
diff --git a/.tool-versions b/.tool-versions
index dcf5945a8..b4d8f8675 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1,2 +1,2 @@
-erlang 24.3.4.2-2
+erlang 24.3.4.2-3
 elixir 1.13.4-otp-24
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index f26926bce..308c26231 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -1,4 +1,4 @@
-ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-debian11
 ARG RUN_FROM=debian:11-slim
 FROM ${BUILD_FROM} AS builder
 
diff --git a/scripts/relup-test/run-relup-lux.sh b/scripts/relup-test/run-relup-lux.sh
index 570e58340..674eadc45 100755
--- a/scripts/relup-test/run-relup-lux.sh
+++ b/scripts/relup-test/run-relup-lux.sh
@@ -45,8 +45,8 @@ fi
 # From now on, no need for the v|e prefix
 OLD_VSN="${old_vsn#[e|v]}"
 
-OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-2-ubuntu20.04-amd64.tar.gz"
-CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-2-ubuntu20.04-amd64.tar.gz"
+OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-3-ubuntu20.04-amd64.tar.gz"
+CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-3-ubuntu20.04-amd64.tar.gz"
 
 if [ ! -f "$OLD_PKG" ]; then
     echo "$OLD_PKG not found"
diff --git a/scripts/relup-test/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh
index 385137dc7..d22c61680 100755
--- a/scripts/relup-test/start-relup-test-cluster.sh
+++ b/scripts/relup-test/start-relup-test-cluster.sh
@@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET"
 BENCH="bench.$NET"
 COOKIE='this-is-a-secret'
 ## Erlang image is needed to run webhook server and emqtt-bench
-ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04"
+ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu20.04"
 # builder has emqtt-bench installed
 BENCH_IMAGE="$ERLANG_IMAGE"
 

From 50bbf4f231433d044569568cb8508f6d4c3da000 Mon Sep 17 00:00:00 2001
From: Serge Tupchii <serge.tupchii@protonmail.com>
Date: Wed, 22 Mar 2023 19:20:27 +0200
Subject: [PATCH 44/52] chore: bump ekka to 0.14.6

ekka 0.14.6 uses mria 0.4.2, which unregisters mnesia_hook upon stopping mria.
This fixes mnesia_hook errors on restarting mria.

Closes: EMQX-9101
---
 apps/emqx/rebar.config      | 2 +-
 changes/ce/feat-10210.en.md | 4 ++++
 mix.exs                     | 2 +-
 rebar.config                | 2 +-
 4 files changed, 7 insertions(+), 3 deletions(-)
 create mode 100644 changes/ce/feat-10210.en.md

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 6875f4003..4c8e2569f 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -27,7 +27,7 @@
     {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
     {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.0"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}},
diff --git a/changes/ce/feat-10210.en.md b/changes/ce/feat-10210.en.md
new file mode 100644
index 000000000..2894ee44e
--- /dev/null
+++ b/changes/ce/feat-10210.en.md
@@ -0,0 +1,4 @@
+Unregister Mnesia post commit hook when Mria is being stopped.
+This fixes hook failures occasionally occurring on stopping/restarting Mria.
+
+[Mria PR](https://github.com/emqx/mria/pull/133)
diff --git a/mix.exs b/mix.exs
index a06bbcfa8..66268e13c 100644
--- a/mix.exs
+++ b/mix.exs
@@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do
       {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
       {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true},
       {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true},
-      {:ekka, github: "emqx/ekka", tag: "0.14.5", override: true},
+      {:ekka, github: "emqx/ekka", tag: "0.14.6", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true},
diff --git a/rebar.config b/rebar.config
index f71da084e..9e874e109 100644
--- a/rebar.config
+++ b/rebar.config
@@ -62,7 +62,7 @@
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}
     , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
     , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}}

From e3541d89479e82d008d2143082281e9f033aa7ce Mon Sep 17 00:00:00 2001
From: Serge Tupchii <serge.tupchii@protonmail.com>
Date: Wed, 22 Mar 2023 19:25:13 +0200
Subject: [PATCH 45/52] test: move dashboard specific cleanup call from
 emqx_common_test_helpers to the relevant test suite

---
 apps/emqx/test/emqx_common_test_helpers.erl              | 1 -
 apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl        | 9 +--------
 .../emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl | 2 ++
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index 658b22c56..d08812075 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -202,7 +202,6 @@ start_apps(Apps, SpecAppConfig, Opts) when is_function(SpecAppConfig) ->
     %% Because, minirest, ekka etc.. application will scan these modules
     lists:foreach(fun load/1, [emqx | Apps]),
     ok = start_ekka(),
-    mnesia:clear_table(emqx_admin),
     ok = emqx_ratelimiter_SUITE:load_conf(),
     lists:foreach(fun(App) -> start_app(App, SpecAppConfig, Opts) end, [emqx | Apps]).
 
diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
index 906d57e9d..18393a40e 100644
--- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
@@ -56,14 +56,6 @@
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
-end_suite() ->
-    end_suite([]).
-
-end_suite(Apps) ->
-    application:unload(emqx_management),
-    mnesia:clear_table(?ADMIN),
-    emqx_common_test_helpers:stop_apps(Apps ++ [emqx_dashboard]).
-
 init_per_suite(Config) ->
     emqx_common_test_helpers:start_apps(
         [emqx_management, emqx_dashboard],
@@ -72,6 +64,7 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
+    mnesia:clear_table(?ADMIN),
     emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_management]),
     mria:stop().
 
diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
index bfbd9b973..fa2373ac3 100644
--- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
@@ -33,10 +33,12 @@ all() ->
 init_per_suite(Config) ->
     application:load(emqx_dashboard),
     mria:start(),
+    mnesia:clear_table(?ADMIN),
     emqx_common_test_helpers:start_apps([emqx_dashboard], fun set_special_configs/1),
     Config.
 
 end_per_suite(Config) ->
+    mnesia:clear_table(?ADMIN),
     emqx_common_test_helpers:stop_apps([emqx_dashboard]),
     Config.
 

From ba49316ec5a8f908a6c09d8ce8b5f9c72c29aeaa Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Mar 2023 19:43:04 +0100
Subject: [PATCH 46/52] fix: use lower-case

---
 apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index c71664c87..080670de6 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -438,7 +438,7 @@ schema("/nodes/:node/bridges/:id/:operation") ->
         'operationId' => '/nodes/:node/bridges/:id/:operation',
         post => #{
             tags => [<<"bridges">>],
-            summary => <<"Stop/Restart bridge">>,
+            summary => <<"Stop/restart bridge">>,
             description => ?DESC("desc_api8"),
             parameters => [
                 param_path_node(),

From 678400bc0a198f88ac1011326d7a053f7475f333 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Mar 2023 16:09:53 +0100
Subject: [PATCH 47/52] fix: add summary to schemas where description has html

---
 apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf   |  4 ++++
 .../i18n/emqx_mgmt_api_publish_i18n.conf          | 14 ++++++++++----
 .../i18n/emqx_mgmt_api_status_i18n.conf           |  4 ++++
 .../emqx_management/src/emqx_mgmt_api_configs.erl |  1 +
 .../emqx_management/src/emqx_mgmt_api_plugins.erl | 15 ++++++++++-----
 .../emqx_management/src/emqx_mgmt_api_publish.erl |  2 ++
 changes/ce/fix-10195.en.md                        |  1 +
 7 files changed, 32 insertions(+), 9 deletions(-)
 create mode 100644 changes/ce/fix-10195.en.md

diff --git a/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf b/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf
index 46854a3db..3ec5367ed 100644
--- a/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf
+++ b/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf
@@ -49,6 +49,10 @@ NOTE: The position should be \"front | rear | before:{name} | after:{name}"""
       zh: """移动 Exhook 服务器顺序。
 注意: 移动的参数只能是：front | rear | before:{name} | after:{name}"""
     }
+    label {
+      en: "Change order of execution for registered Exhook server"
+      zh: "改变已注册的Exhook服务器的执行顺序"
+    }
   }
 
   move_position {
diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf
index 4123ceefd..f91115df5 100644
--- a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf
+++ b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf
@@ -2,8 +2,7 @@
 emqx_mgmt_api_publish {
     publish_api {
         desc {
-            en: """Publish one message.<br/>
-Possible HTTP status response codes are:<br/>
+            en: """Possible HTTP status response codes are:<br/>
 <code>200</code>: The message is delivered to at least one subscriber;<br/>
 <code>202</code>: No matched subscribers;<br/>
 <code>400</code>: Message is invalid. for example bad topic name, or QoS is out of range;<br/>
@@ -16,11 +15,14 @@ Possible HTTP status response codes are:<br/>
 400: 消息编码错误，如非法主题，或 QoS 超出范围等。<br/>
 503: 服务重启等过程中导致转发失败。"""
         }
+        label {
+            en: "Publish a message"
+            zh: "发布一条信息"
+        }
     }
     publish_bulk_api {
         desc {
-            en: """Publish a batch of messages.<br/>
-Possible HTTP response status code are:<br/>
+            en: """Possible HTTP response status code are:<br/>
 200: All messages are delivered to at least one subscriber;<br/>
 202: At least one message was not delivered to any subscriber;<br/>
 400: At least one message is invalid. For example bad topic name, or QoS is out of range;<br/>
@@ -41,6 +43,10 @@ result of each individual message in the batch."""
  <code>/publish</code> 是一样的。
 如果所有的消息都是合法的，那么 HTTP 返回的内容是一个 JSON 数组，每个元素代表了该消息转发的状态。"""
         }
+        label {
+            en: "Publish a batch of messages"
+            zh: "发布一批信息"
+        }
     }
 
     topic_name {
diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf
index fae17b35d..d72fd0998 100644
--- a/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf
+++ b/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf
@@ -22,6 +22,10 @@ emqx_mgmt_api_status {
           "GET `/status`端点（没有`/api/...`前缀）也是这个端点的一个别名，工作方式相同。"
           " 这个别名从v5.0.0开始就有了。"
     }
+    label {
+       en: "Service health check"
+       zh: "服务健康检查"
+    }
   }
 
   get_status_response200 {
diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl
index 2e6aac849..55cc50597 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl
@@ -119,6 +119,7 @@ schema("/configs_reset/:rootname") ->
                     "- For a config entry that has default value, this resets it to the default value;\n"
                     "- For a config entry that has no default value, an error 400 will be returned"
                 >>,
+            summary => <<"Reset config entry">>,
             %% We only return "200" rather than the new configs that has been changed, as
             %% the schema of the changed configs is depends on the request parameter
             %% `conf_path`, it cannot be defined here.
diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl
index 4930e587c..f744ec0d2 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl
@@ -68,10 +68,10 @@ schema("/plugins") ->
     #{
         'operationId' => list_plugins,
         get => #{
+            summary => <<"List all installed plugins">>,
             description =>
-                "List all install plugins.<br/>"
                 "Plugins are launched in top-down order.<br/>"
-                "Using `POST /plugins/{name}/move` to change the boot order.",
+                "Use `POST /plugins/{name}/move` to change the boot order.",
             tags => ?TAGS,
             responses => #{
                 200 => hoconsc:array(hoconsc:ref(plugin))
@@ -82,8 +82,9 @@ schema("/plugins/install") ->
     #{
         'operationId' => upload_install,
         post => #{
+            summary => <<"Install a new plugin">>,
             description =>
-                "Install a plugin(plugin-vsn.tar.gz)."
+                "Upload a plugin tarball (plugin-vsn.tar.gz)."
                 "Follow [emqx-plugin-template](https://github.com/emqx/emqx-plugin-template) "
                 "to develop plugin.",
             tags => ?TAGS,
@@ -112,7 +113,8 @@ schema("/plugins/:name") ->
     #{
         'operationId' => plugin,
         get => #{
-            description => "Describe a plugin according `release.json` and `README.md`.",
+            summary => <<"Get a plugin description">>,
+            description => "Describs plugin according to its `release.json` and `README.md`.",
             tags => ?TAGS,
             parameters => [hoconsc:ref(name)],
             responses => #{
@@ -121,7 +123,8 @@ schema("/plugins/:name") ->
             }
         },
         delete => #{
-            description => "Uninstall a plugin package.",
+            summary => <<"Delete a plugin">>,
+            description => "Uninstalls a previously uploaded plugin package.",
             tags => ?TAGS,
             parameters => [hoconsc:ref(name)],
             responses => #{
@@ -134,6 +137,7 @@ schema("/plugins/:name/:action") ->
     #{
         'operationId' => update_plugin,
         put => #{
+            summary => <<"Trigger action on an installed plugin">>,
             description =>
                 "start/stop a installed plugin.<br/>"
                 "- **start**: start the plugin.<br/>"
@@ -153,6 +157,7 @@ schema("/plugins/:name/move") ->
     #{
         'operationId' => update_boot_order,
         post => #{
+            summary => <<"Move plugin within plugin hiearchy">>,
             description => "Setting the boot order of plugins.",
             tags => ?TAGS,
             parameters => [hoconsc:ref(name)],
diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl
index 245b56c1d..ba486ab89 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl
@@ -50,6 +50,7 @@ schema("/publish") ->
     #{
         'operationId' => publish,
         post => #{
+            summary => <<"Publish a message">>,
             description => ?DESC(publish_api),
             tags => [<<"Publish">>],
             'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, publish_message)),
@@ -65,6 +66,7 @@ schema("/publish/bulk") ->
     #{
         'operationId' => publish_batch,
         post => #{
+            summary => <<"Publish a batch of messages">>,
             description => ?DESC(publish_bulk_api),
             tags => [<<"Publish">>],
             'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message)), #{}),
diff --git a/changes/ce/fix-10195.en.md b/changes/ce/fix-10195.en.md
new file mode 100644
index 000000000..35cc7c082
--- /dev/null
+++ b/changes/ce/fix-10195.en.md
@@ -0,0 +1 @@
+Add labels to API schemas where description contains HTML and breaks formatting of generated documentation otherwise.

From f8d5d53908f71518dc3f16f6d07cc077b0fc3aea Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Wed, 22 Mar 2023 11:26:15 -0300
Subject: [PATCH 48/52] feat(buffer_worker): decouple query mode from
 underlying connector call mode

Fixes https://emqx.atlassian.net/browse/EMQX-9129

Currently, if an user configures a bridge with query mode sync, then
all calls to the underlying driver/connector ("inner calls") will
always be synchronous, regardless of its support for async calls.

Since buffer workers always support async queries ("outer calls"), we
should decouple those two call modes (inner and outer), and avoid
exposing the inner call configuration to user to avoid complexity.

There are two situations when we want to force synchronous calls to
the underlying connector even if it supports async:

1) When using `simple_sync_query`, since we are bypassing the buffer
workers;
2) When retrying the inflight window, to avoid overwhelming the
driver.
---
 apps/emqx_resource/src/emqx_resource.erl      |  2 +-
 .../src/emqx_resource_buffer_worker.erl       | 26 +++---
 .../test/emqx_connector_demo.erl              | 15 ++++
 .../test/emqx_resource_SUITE.erl              | 83 +++++++++++++++++++
 changes/ce/feat-10206.en.md                   |  7 ++
 .../test/emqx_ee_bridge_dynamo_SUITE.erl      |  4 +-
 .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl  | 82 ++++--------------
 .../test/emqx_ee_bridge_influxdb_SUITE.erl    | 40 +++++----
 .../test/emqx_ee_connector_influxdb_SUITE.erl |  4 +-
 9 files changed, 162 insertions(+), 101 deletions(-)
 create mode 100644 changes/ce/feat-10206.en.md

diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 1ccb5ca71..2e72c2a28 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -265,7 +265,7 @@ query(ResId, Request, Opts) ->
             IsBufferSupported = is_buffer_supported(Module),
             case {IsBufferSupported, QM} of
                 {true, _} ->
-                    %% only Kafka so far
+                    %% only Kafka producer so far
                     Opts1 = Opts#{is_buffer_supported => true},
                     emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts1);
                 {false, sync} ->
diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
index 8bfd77e61..4151cf430 100644
--- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
@@ -140,7 +140,7 @@ simple_sync_query(Id, Request) ->
     QueryOpts = simple_query_opts(),
     emqx_resource_metrics:matched_inc(Id),
     Ref = make_request_ref(),
-    Result = call_query(sync, Id, Index, Ref, ?SIMPLE_QUERY(Request), QueryOpts),
+    Result = call_query(force_sync, Id, Index, Ref, ?SIMPLE_QUERY(Request), QueryOpts),
     _ = handle_query_result(Id, Result, _HasBeenSent = false),
     Result.
 
@@ -152,7 +152,7 @@ simple_async_query(Id, Request, QueryOpts0) ->
     QueryOpts = maps:merge(simple_query_opts(), QueryOpts0),
     emqx_resource_metrics:matched_inc(Id),
     Ref = make_request_ref(),
-    Result = call_query(async, Id, Index, Ref, ?SIMPLE_QUERY(Request), QueryOpts),
+    Result = call_query(async_if_possible, Id, Index, Ref, ?SIMPLE_QUERY(Request), QueryOpts),
     _ = handle_query_result(Id, Result, _HasBeenSent = false),
     Result.
 
@@ -377,7 +377,7 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) ->
     } = Data0,
     ?tp(buffer_worker_retry_inflight, #{query_or_batch => QueryOrBatch, ref => Ref}),
     QueryOpts = #{simple_query => false},
-    Result = call_query(sync, Id, Index, Ref, QueryOrBatch, QueryOpts),
+    Result = call_query(force_sync, Id, Index, Ref, QueryOrBatch, QueryOpts),
     ReplyResult =
         case QueryOrBatch of
             ?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt) ->
@@ -566,7 +566,7 @@ do_flush(
     %% unwrap when not batching (i.e., batch size == 1)
     [?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt) = Request] = Batch,
     QueryOpts = #{inflight_tid => InflightTID, simple_query => false},
-    Result = call_query(configured, Id, Index, Ref, Request, QueryOpts),
+    Result = call_query(async_if_possible, Id, Index, Ref, Request, QueryOpts),
     Reply = ?REPLY(ReplyTo, HasBeenSent, Result),
     case reply_caller(Id, Reply, QueryOpts) of
         %% Failed; remove the request from the queue, as we cannot pop
@@ -651,7 +651,7 @@ do_flush(#{queue := Q1} = Data0, #{
         inflight_tid := InflightTID
     } = Data0,
     QueryOpts = #{inflight_tid => InflightTID, simple_query => false},
-    Result = call_query(configured, Id, Index, Ref, Batch, QueryOpts),
+    Result = call_query(async_if_possible, Id, Index, Ref, Batch, QueryOpts),
     case batch_reply_caller(Id, Result, Batch, QueryOpts) of
         %% Failed; remove the request from the queue, as we cannot pop
         %% from it again, but we'll retry it using the inflight table.
@@ -883,17 +883,13 @@ handle_async_worker_down(Data0, Pid) ->
     mark_inflight_items_as_retriable(Data, WorkerMRef),
     {keep_state, Data}.
 
-call_query(QM0, Id, Index, Ref, Query, QueryOpts) ->
-    ?tp(call_query_enter, #{id => Id, query => Query, query_mode => QM0}),
+-spec call_query(force_sync | async_if_possible, _, _, _, _, _) -> _.
+call_query(QM, Id, Index, Ref, Query, QueryOpts) ->
+    ?tp(call_query_enter, #{id => Id, query => Query, query_mode => QM}),
     case emqx_resource_manager:lookup_cached(Id) of
         {ok, _Group, #{status := stopped}} ->
             ?RESOURCE_ERROR(stopped, "resource stopped or disabled");
         {ok, _Group, Resource} ->
-            QM =
-                case QM0 =:= configured of
-                    true -> maps:get(query_mode, Resource);
-                    false -> QM0
-                end,
             do_call_query(QM, Id, Index, Ref, Query, QueryOpts, Resource);
         {error, not_found} ->
             ?RESOURCE_ERROR(not_found, "resource not found")
@@ -1511,9 +1507,9 @@ inc_sent_success(Id, _HasBeenSent = true) ->
 inc_sent_success(Id, _HasBeenSent) ->
     emqx_resource_metrics:success_inc(Id).
 
-call_mode(sync, _) -> sync;
-call_mode(async, always_sync) -> sync;
-call_mode(async, async_if_possible) -> async.
+call_mode(force_sync, _) -> sync;
+call_mode(async_if_possible, always_sync) -> sync;
+call_mode(async_if_possible, async_if_possible) -> async.
 
 assert_ok_result(ok) ->
     true;
diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl
index a863dbb78..a1393c574 100644
--- a/apps/emqx_resource/test/emqx_connector_demo.erl
+++ b/apps/emqx_resource/test/emqx_connector_demo.erl
@@ -146,6 +146,12 @@ on_query(_InstId, {sleep_before_reply, For}, #{pid := Pid}) ->
         {error, timeout}
     end.
 
+on_query_async(_InstId, block, ReplyFun, #{pid := Pid}) ->
+    Pid ! {block, ReplyFun},
+    {ok, Pid};
+on_query_async(_InstId, resume, ReplyFun, #{pid := Pid}) ->
+    Pid ! {resume, ReplyFun},
+    {ok, Pid};
 on_query_async(_InstId, {inc_counter, N}, ReplyFun, #{pid := Pid}) ->
     Pid ! {inc, N, ReplyFun},
     {ok, Pid};
@@ -274,6 +280,10 @@ counter_loop(
             block ->
                 ct:pal("counter recv: ~p", [block]),
                 State#{status => blocked};
+            {block, ReplyFun} ->
+                ct:pal("counter recv: ~p", [block]),
+                apply_reply(ReplyFun, ok),
+                State#{status => blocked};
             {block_now, ReplyFun} ->
                 ct:pal("counter recv: ~p", [block_now]),
                 apply_reply(
@@ -284,6 +294,11 @@ counter_loop(
                 {messages, Msgs} = erlang:process_info(self(), messages),
                 ct:pal("counter recv: ~p, buffered msgs: ~p", [resume, length(Msgs)]),
                 State#{status => running};
+            {resume, ReplyFun} ->
+                {messages, Msgs} = erlang:process_info(self(), messages),
+                ct:pal("counter recv: ~p, buffered msgs: ~p", [resume, length(Msgs)]),
+                apply_reply(ReplyFun, ok),
+                State#{status => running};
             {inc, N, ReplyFun} when Status == running ->
                 %ct:pal("async counter recv: ~p", [{inc, N}]),
                 apply_reply(ReplyFun, ok),
diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index e7c252fa9..ca91ae40d 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -2561,6 +2561,84 @@ do_t_recursive_flush() ->
     ),
     ok.
 
+t_call_mode_uncoupled_from_query_mode(_Config) ->
+    DefaultOpts = #{
+        batch_size => 1,
+        batch_time => 5,
+        worker_pool_size => 1
+    },
+    ?check_trace(
+        begin
+            %% We check that we can call the buffer workers with async
+            %% calls, even if the underlying connector itself only
+            %% supports sync calls.
+            emqx_connector_demo:set_callback_mode(always_sync),
+            {ok, _} = emqx_resource:create(
+                ?ID,
+                ?DEFAULT_RESOURCE_GROUP,
+                ?TEST_RESOURCE,
+                #{name => test_resource},
+                DefaultOpts#{query_mode => async}
+            ),
+            ?tp_span(
+                async_query_sync_driver,
+                #{},
+                ?assertMatch(
+                    {ok, {ok, _}},
+                    ?wait_async_action(
+                        emqx_resource:query(?ID, {inc_counter, 1}),
+                        #{?snk_kind := buffer_worker_flush_ack},
+                        500
+                    )
+                )
+            ),
+            ?assertEqual(ok, emqx_resource:remove_local(?ID)),
+
+            %% And we check the converse: a connector that allows async
+            %% calls can be called synchronously, but the underlying
+            %% call should be async.
+            emqx_connector_demo:set_callback_mode(async_if_possible),
+            {ok, _} = emqx_resource:create(
+                ?ID,
+                ?DEFAULT_RESOURCE_GROUP,
+                ?TEST_RESOURCE,
+                #{name => test_resource},
+                DefaultOpts#{query_mode => sync}
+            ),
+            ?tp_span(
+                sync_query_async_driver,
+                #{},
+                ?assertEqual(ok, emqx_resource:query(?ID, {inc_counter, 2}))
+            ),
+            ?assertEqual(ok, emqx_resource:remove_local(?ID)),
+            ?tp(sync_query_async_driver, #{}),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace1 = trace_between_span(Trace0, async_query_sync_driver),
+            ct:pal("async query calling sync driver\n  ~p", [Trace1]),
+            ?assert(
+                ?strict_causality(
+                    #{?snk_kind := async_query, request := {inc_counter, 1}},
+                    #{?snk_kind := call_query, call_mode := sync},
+                    Trace1
+                )
+            ),
+
+            Trace2 = trace_between_span(Trace0, sync_query_async_driver),
+            ct:pal("sync query calling async driver\n  ~p", [Trace2]),
+            ?assert(
+                ?strict_causality(
+                    #{?snk_kind := sync_query, request := {inc_counter, 2}},
+                    #{?snk_kind := call_query_async},
+                    Trace2
+                )
+            ),
+
+            ok
+        end
+    ).
+
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------
@@ -2742,3 +2820,8 @@ assert_async_retry_fail_then_succeed_inflight(Trace) ->
         )
     ),
     ok.
+
+trace_between_span(Trace0, Marker) ->
+    {Trace1, [_ | _]} = ?split_trace_at(#{?snk_kind := Marker, ?snk_span := {complete, _}}, Trace0),
+    {[_ | _], [_ | Trace2]} = ?split_trace_at(#{?snk_kind := Marker, ?snk_span := start}, Trace1),
+    Trace2.
diff --git a/changes/ce/feat-10206.en.md b/changes/ce/feat-10206.en.md
new file mode 100644
index 000000000..014ea71f2
--- /dev/null
+++ b/changes/ce/feat-10206.en.md
@@ -0,0 +1,7 @@
+Decouple the query mode from the underlying call mode for buffer
+workers.
+
+Prior to this change, setting the query mode of a resource
+such as a bridge to `sync` would force the buffer to call the
+underlying connector in a synchronous way, even if it supports async
+calls.
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl
index 183002e61..c0d58c4f7 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl
@@ -291,7 +291,7 @@ t_setup_via_config_and_publish(Config) ->
         end,
         fun(Trace0) ->
             Trace = ?of_kind(dynamo_connector_query_return, Trace0),
-            ?assertMatch([#{result := {ok, _}}], Trace),
+            ?assertMatch([#{result := ok}], Trace),
             ok
         end
     ),
@@ -328,7 +328,7 @@ t_setup_via_http_api_and_publish(Config) ->
         end,
         fun(Trace0) ->
             Trace = ?of_kind(dynamo_connector_query_return, Trace0),
-            ?assertMatch([#{result := {ok, _}}], Trace),
+            ?assertMatch([#{result := ok}], Trace),
             ok
         end
     ),
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl
index 8424ddff0..709955666 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl
@@ -1013,7 +1013,6 @@ t_publish_timeout(Config) ->
     do_econnrefused_or_timeout_test(Config, timeout).
 
 do_econnrefused_or_timeout_test(Config, Error) ->
-    QueryMode = ?config(query_mode, Config),
     ResourceId = ?config(resource_id, Config),
     TelemetryTable = ?config(telemetry_table, Config),
     Topic = <<"t/topic">>,
@@ -1021,15 +1020,8 @@ do_econnrefused_or_timeout_test(Config, Error) ->
     Message = emqx_message:make(Topic, Payload),
     ?check_trace(
         begin
-            case {QueryMode, Error} of
-                {sync, _} ->
-                    {_, {ok, _}} =
-                        ?wait_async_action(
-                            emqx:publish(Message),
-                            #{?snk_kind := gcp_pubsub_request_failed, recoverable_error := true},
-                            15_000
-                        );
-                {async, econnrefused} ->
+            case Error of
+                econnrefused ->
                     %% at the time of writing, async requests
                     %% are never considered expired by ehttpc
                     %% (even if they arrive late, or never
@@ -1049,7 +1041,7 @@ do_econnrefused_or_timeout_test(Config, Error) ->
                             },
                             15_000
                         );
-                {async, timeout} ->
+                timeout ->
                     %% at the time of writing, async requests
                     %% are never considered expired by ehttpc
                     %% (even if they arrive late, or never
@@ -1067,18 +1059,13 @@ do_econnrefused_or_timeout_test(Config, Error) ->
             end
         end,
         fun(Trace) ->
-            case {QueryMode, Error} of
-                {sync, _} ->
+            case Error of
+                econnrefused ->
                     ?assertMatch(
                         [#{reason := Error, connector := ResourceId} | _],
                         ?of_kind(gcp_pubsub_request_failed, Trace)
                     );
-                {async, econnrefused} ->
-                    ?assertMatch(
-                        [#{reason := Error, connector := ResourceId} | _],
-                        ?of_kind(gcp_pubsub_request_failed, Trace)
-                    );
-                {async, timeout} ->
+                timeout ->
                     ?assertMatch(
                         [_, _ | _],
                         ?of_kind(gcp_pubsub_response, Trace)
@@ -1088,11 +1075,11 @@ do_econnrefused_or_timeout_test(Config, Error) ->
         end
     ),
 
-    case {Error, QueryMode} of
+    case Error of
         %% apparently, async with disabled queue doesn't mark the
         %% message as dropped; and since it never considers the
         %% response expired, this succeeds.
-        {econnrefused, async} ->
+        econnrefused ->
             wait_telemetry_event(TelemetryTable, queuing, ResourceId, #{
                 timeout => 10_000, n_events => 1
             }),
@@ -1114,7 +1101,7 @@ do_econnrefused_or_timeout_test(Config, Error) ->
                 } when Matched >= 1 andalso Inflight + Queueing + Dropped + Failed =< 2,
                 CurrentMetrics
             );
-        {timeout, async} ->
+        timeout ->
             wait_until_gauge_is(inflight, 0, _Timeout = 400),
             wait_until_gauge_is(queuing, 0, _Timeout = 400),
             assert_metrics(
@@ -1129,21 +1116,6 @@ do_econnrefused_or_timeout_test(Config, Error) ->
                     late_reply => 2
                 },
                 ResourceId
-            );
-        {_, sync} ->
-            wait_until_gauge_is(queuing, 0, 500),
-            wait_until_gauge_is(inflight, 1, 500),
-            assert_metrics(
-                #{
-                    dropped => 0,
-                    failed => 0,
-                    inflight => 1,
-                    matched => 1,
-                    queuing => 0,
-                    retried => 0,
-                    success => 0
-                },
-                ResourceId
             )
     end,
 
@@ -1267,7 +1239,6 @@ t_failure_no_body(Config) ->
 
 t_unrecoverable_error(Config) ->
     ResourceId = ?config(resource_id, Config),
-    QueryMode = ?config(query_mode, Config),
     TestPid = self(),
     FailureNoBodyHandler =
         fun(Req0, State) ->
@@ -1298,33 +1269,16 @@ t_unrecoverable_error(Config) ->
     Message = emqx_message:make(Topic, Payload),
     ?check_trace(
         {_, {ok, _}} =
-            case QueryMode of
-                sync ->
-                    ?wait_async_action(
-                        emqx:publish(Message),
-                        #{?snk_kind := gcp_pubsub_request_failed},
-                        5_000
-                    );
-                async ->
-                    ?wait_async_action(
-                        emqx:publish(Message),
-                        #{?snk_kind := gcp_pubsub_response},
-                        5_000
-                    )
-            end,
+            ?wait_async_action(
+                emqx:publish(Message),
+                #{?snk_kind := gcp_pubsub_response},
+                5_000
+            ),
         fun(Trace) ->
-            case QueryMode of
-                sync ->
-                    ?assertMatch(
-                        [#{reason := killed}],
-                        ?of_kind(gcp_pubsub_request_failed, Trace)
-                    );
-                async ->
-                    ?assertMatch(
-                        [#{response := {error, killed}}],
-                        ?of_kind(gcp_pubsub_response, Trace)
-                    )
-            end,
+            ?assertMatch(
+                [#{response := {error, killed}}],
+                ?of_kind(gcp_pubsub_response, Trace)
+            ),
             ok
         end
     ),
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
index 2b2214df0..e8dd970f3 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl
@@ -532,10 +532,12 @@ t_start_ok(Config) ->
     },
     ?check_trace(
         begin
-            ?assertEqual(ok, send_message(Config, SentData)),
             case QueryMode of
-                async -> ct:sleep(500);
-                sync -> ok
+                async ->
+                    ?assertMatch(ok, send_message(Config, SentData)),
+                    ct:sleep(500);
+                sync ->
+                    ?assertMatch({ok, 204, _}, send_message(Config, SentData))
             end,
             PersistedData = query_by_clientid(ClientId, Config),
             Expected = #{
@@ -689,10 +691,12 @@ t_const_timestamp(Config) ->
         <<"payload">> => Payload,
         <<"timestamp">> => erlang:system_time(millisecond)
     },
-    ?assertEqual(ok, send_message(Config, SentData)),
     case QueryMode of
-        async -> ct:sleep(500);
-        sync -> ok
+        async ->
+            ?assertMatch(ok, send_message(Config, SentData)),
+            ct:sleep(500);
+        sync ->
+            ?assertMatch({ok, 204, _}, send_message(Config, SentData))
     end,
     PersistedData = query_by_clientid(ClientId, Config),
     Expected = #{foo => <<"123">>},
@@ -745,7 +749,12 @@ t_boolean_variants(Config) ->
                 <<"timestamp">> => erlang:system_time(millisecond),
                 <<"payload">> => Payload
             },
-            ?assertEqual(ok, send_message(Config, SentData)),
+            case QueryMode of
+                sync ->
+                    ?assertMatch({ok, 204, _}, send_message(Config, SentData));
+                async ->
+                    ?assertMatch(ok, send_message(Config, SentData))
+            end,
             case QueryMode of
                 async -> ct:sleep(500);
                 sync -> ok
@@ -841,10 +850,9 @@ t_bad_timestamp(Config) ->
                     );
                 {sync, false} ->
                     ?assertEqual(
-                        {error,
-                            {unrecoverable_error, [
-                                {error, {bad_timestamp, <<"bad_timestamp">>}}
-                            ]}},
+                        {error, [
+                            {error, {bad_timestamp, <<"bad_timestamp">>}}
+                        ]},
                         Return
                     );
                 {sync, true} ->
@@ -964,7 +972,7 @@ t_write_failure(Config) ->
                                 {error, {resource_error, #{reason := timeout}}},
                                 send_message(Config, SentData)
                             ),
-                            #{?snk_kind := buffer_worker_flush_nack},
+                            #{?snk_kind := handle_async_reply, action := nack},
                             1_000
                         );
                 async ->
@@ -978,13 +986,13 @@ t_write_failure(Config) ->
         fun(Trace0) ->
             case QueryMode of
                 sync ->
-                    Trace = ?of_kind(buffer_worker_flush_nack, Trace0),
+                    Trace = ?of_kind(handle_async_reply, Trace0),
                     ?assertMatch([_ | _], Trace),
                     [#{result := Result} | _] = Trace,
                     ?assert(
                         {error, {error, {closed, "The connection was lost."}}} =:= Result orelse
                             {error, {error, closed}} =:= Result orelse
-                            {error, {recoverable_error, {error, econnrefused}}} =:= Result,
+                            {error, {recoverable_error, econnrefused}} =:= Result,
                         #{got => Result}
                     );
                 async ->
@@ -1006,7 +1014,6 @@ t_write_failure(Config) ->
     ok.
 
 t_missing_field(Config) ->
-    QueryMode = ?config(query_mode, Config),
     BatchSize = ?config(batch_size, Config),
     IsBatch = BatchSize > 1,
     {ok, _} =
@@ -1034,8 +1041,7 @@ t_missing_field(Config) ->
             {ok, _} =
                 snabbkaffe:block_until(
                     ?match_n_events(NEvents, #{
-                        ?snk_kind := influxdb_connector_send_query_error,
-                        mode := QueryMode
+                        ?snk_kind := influxdb_connector_send_query_error
                     }),
                     _Timeout1 = 10_000
                 ),
diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
index 72fc11a67..364821ea0 100644
--- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
+++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
@@ -94,7 +94,7 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
         emqx_resource:get_instance(PoolName),
     ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)),
     % % Perform query as further check that the resource is working as expected
-    ?assertMatch(ok, emqx_resource:query(PoolName, test_query())),
+    ?assertMatch({ok, 204, _}, emqx_resource:query(PoolName, test_query())),
     ?assertEqual(ok, emqx_resource:stop(PoolName)),
     % Resource will be listed still, but state will be changed and healthcheck will fail
     % as the worker no longer exists.
@@ -116,7 +116,7 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
     {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} =
         emqx_resource:get_instance(PoolName),
     ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)),
-    ?assertMatch(ok, emqx_resource:query(PoolName, test_query())),
+    ?assertMatch({ok, 204, _}, emqx_resource:query(PoolName, test_query())),
     % Stop and remove the resource in one go.
     ?assertEqual(ok, emqx_resource:remove_local(PoolName)),
     ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),

From 4d28731176f7d9e8e1b3dd1819580b0ab57a377a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 24 Mar 2023 08:37:00 +0100
Subject: [PATCH 49/52] docs: delete local-override.conf from config template

---
 rel/emqx_conf.template.en.md | 3 +--
 rel/emqx_conf.template.zh.md | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md
index bbeac9489..8740e4319 100644
--- a/rel/emqx_conf.template.en.md
+++ b/rel/emqx_conf.template.en.md
@@ -4,13 +4,12 @@ and a superset of JSON.
 
 ## Layered
 
-EMQX configuration consists of 3 layers.
+EMQX configuration consists of two layers.
 From bottom up:
 
 1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.<br/>
    Changes in this layer require a full node restart to take effect.
 1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf`
-1. Local node overrides: `$EMQX_NODE__DATA_DIR/configs/local-override.conf`
 
 When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir`
 is used.
diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md
index cfb620c0f..9402760a2 100644
--- a/rel/emqx_conf.template.zh.md
+++ b/rel/emqx_conf.template.zh.md
@@ -3,12 +3,11 @@ HOCON（Human-Optimized Config Object Notation）是一个JSON的超集，非常
 
 ## 分层结构
 
-EMQX的配置文件可分为三层，自底向上依次是：
+EMQX的配置文件可分为二层，自底向上依次是：
 
 1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。<br/>
    修改这一层的配置之后，需要重启节点来使之生效。
 1. 集群范围重载层：`$EMQX_NODE__DATA_DIR/configs/cluster-override.conf`
-1. 节点本地重载层：`$EMQX_NODE__DATA_DIR/configs/local-override.conf`
 
 如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置，那么该目录会从 `emqx.conf` 的 `node.data_dir` 配置中读取。
 

From a8cfc1c7a14cb049ea4f4a7978ebfcb9d5b8f07d Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Fri, 24 Mar 2023 11:17:50 +0100
Subject: [PATCH 50/52] fix(emqx_bridge): filter status_reason in config update

---
 apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index c71664c87..f70964e47 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -929,7 +929,7 @@ filter_out_request_body(Conf) ->
         <<"type">>,
         <<"name">>,
         <<"status">>,
-        <<"error">>,
+        <<"status_reason">>,
         <<"node_status">>,
         <<"node_metrics">>,
         <<"metrics">>,

From fbfdaf44e09de51228f97be6aba082437cbdb1bc Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Thu, 23 Mar 2023 17:41:42 +0100
Subject: [PATCH 51/52] fix(emqx_bridge): don't crash on validation error

---
 apps/emqx_bridge/src/emqx_bridge_api.erl      | 25 +++++++++++++---
 .../test/emqx_bridge_api_SUITE.erl            | 30 +++++++++++++++++++
 changes/ce/fix-10226.en.md                    |  1 +
 3 files changed, 52 insertions(+), 4 deletions(-)
 create mode 100644 changes/ce/fix-10226.en.md

diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index f70964e47..9fb8b30c2 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -481,8 +481,7 @@ schema("/bridges_probe") ->
             ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>);
         {error, not_found} ->
             Conf = filter_out_request_body(Conf0),
-            {ok, _} = emqx_bridge:create(BridgeType, BridgeName, Conf),
-            lookup_from_all_nodes(BridgeType, BridgeName, 201)
+            create_bridge(BridgeType, BridgeName, Conf)
     end;
 '/bridges'(get, _Params) ->
     Nodes = mria:running_nodes(),
@@ -508,8 +507,7 @@ schema("/bridges_probe") ->
             {ok, _} ->
                 RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
                 Conf = deobfuscate(Conf1, RawConf),
-                {ok, _} = emqx_bridge:create(BridgeType, BridgeName, Conf),
-                lookup_from_all_nodes(BridgeType, BridgeName, 200);
+                update_bridge(BridgeType, BridgeName, Conf);
             {error, not_found} ->
                 ?BRIDGE_NOT_FOUND(BridgeType, BridgeName)
         end
@@ -609,6 +607,20 @@ lookup_from_local_node(BridgeType, BridgeName) ->
         Error -> Error
     end.
 
+create_bridge(BridgeType, BridgeName, Conf) ->
+    create_or_update_bridge(BridgeType, BridgeName, Conf, 201).
+
+update_bridge(BridgeType, BridgeName, Conf) ->
+    create_or_update_bridge(BridgeType, BridgeName, Conf, 200).
+
+create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) ->
+    case emqx_bridge:create(BridgeType, BridgeName, Conf) of
+        {ok, _} ->
+            lookup_from_all_nodes(BridgeType, BridgeName, HttpStatusCode);
+        {error, #{kind := validation_error} = Reason} ->
+            ?BAD_REQUEST(map_to_json(Reason))
+    end.
+
 '/bridges/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) ->
     ?TRY_PARSE_ID(
         Id,
@@ -1033,3 +1045,8 @@ deobfuscate(NewConf, OldConf) ->
         #{},
         NewConf
     ).
+
+map_to_json(M) ->
+    emqx_json:encode(
+        emqx_map_lib:jsonable_map(M, fun(K, V) -> {K, emqx_map_lib:binary_string(V)} end)
+    ).
diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
index 932d7261a..47a23e71c 100644
--- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
+++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl
@@ -322,6 +322,33 @@ t_http_crud_apis(Config) ->
         end
     ),
 
+    %% Test bad updates
+    {ok, 400, PutFail1} = request(
+        put,
+        uri(["bridges", BridgeID]),
+        maps:remove(<<"url">>, ?HTTP_BRIDGE(URL2, Name))
+    ),
+    ?assertMatch(
+        #{<<"reason">> := <<"required_field">>},
+        emqx_json:decode(maps:get(<<"message">>, emqx_json:decode(PutFail1, [return_maps])), [
+            return_maps
+        ])
+    ),
+    {ok, 400, PutFail2} = request(
+        put,
+        uri(["bridges", BridgeID]),
+        maps:put(<<"curl">>, URL2, maps:remove(<<"url">>, ?HTTP_BRIDGE(URL2, Name)))
+    ),
+    ?assertMatch(
+        #{
+            <<"reason">> := <<"unknown_fields">>,
+            <<"unknown">> := <<"curl">>
+        },
+        emqx_json:decode(maps:get(<<"message">>, emqx_json:decode(PutFail2, [return_maps])), [
+            return_maps
+        ])
+    ),
+
     %% delete the bridge
     {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
@@ -387,6 +414,9 @@ t_http_crud_apis(Config) ->
     ?assert(not maps:is_key(<<"status_reason">>, FixedBridge)),
     ?assert(not maps:is_key(<<"status_reason">>, FixedNodeStatus)),
     {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
+
+    %% Try create bridge with bad characters as name
+    {ok, 400, _} = request(post, uri(["bridges"]), ?HTTP_BRIDGE(URL1, <<"隋达"/utf8>>)),
     ok.
 
 t_http_bridges_local_topic(Config) ->
diff --git a/changes/ce/fix-10226.en.md b/changes/ce/fix-10226.en.md
new file mode 100644
index 000000000..2d833d2dc
--- /dev/null
+++ b/changes/ce/fix-10226.en.md
@@ -0,0 +1 @@
+Don't crash on validation error in `/bridges` API, return `400` instead.

From b02d0088ea1143b620764588d85863147b760755 Mon Sep 17 00:00:00 2001
From: Serge Tupchii <serge.tupchii@protonmail.com>
Date: Thu, 23 Mar 2023 18:27:46 +0200
Subject: [PATCH 52/52] fix(emqx_management): resolve plugin name clashes

Allow installing a plugin if its name matches the beginning of another (already installed) plugin name.
For example: if plugin "emqx_plugin_template_a" is installed, it must not block installing plugin "emqx_plugin_template".

Closes: #10213, EMQX-9290
---
 .../src/emqx_mgmt_api_plugins.erl             |  5 +-
 .../test/emqx_mgmt_api_plugins_SUITE.erl      | 46 ++++++++++++++++++-
 changes/ce/fix-10225.en.md                    |  2 +
 3 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 changes/ce/fix-10225.en.md

diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl
index f744ec0d2..a46584f7f 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl
@@ -48,6 +48,9 @@
 
 -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_.]*$").
 -define(TAGS, [<<"Plugins">>]).
+%% Plugin NameVsn must follow the pattern <app_name>-<vsn>,
+%% app_name must be a snake_case (no '-' allowed).
+-define(VSN_WILDCARD, "-*.tar.gz").
 
 namespace() -> "plugins".
 
@@ -334,7 +337,7 @@ upload_install(post, #{body := #{<<"plugin">> := Plugin}}) when is_map(Plugin) -
             case emqx_plugins:parse_name_vsn(FileName) of
                 {ok, AppName, _Vsn} ->
                     AppDir = filename:join(emqx_plugins:install_dir(), AppName),
-                    case filelib:wildcard(AppDir ++ "*.tar.gz") of
+                    case filelib:wildcard(AppDir ++ ?VSN_WILDCARD) of
                         [] ->
                             do_install_package(FileName, Bin);
                         OtherVsn ->
diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl
index 0cf15d678..24e55494d 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl
@@ -20,6 +20,7 @@
 
 -include_lib("eunit/include/eunit.hrl").
 
+-define(EMQX_PLUGIN_TEMPLATE_NAME, "emqx_plugin_template").
 -define(EMQX_PLUGIN_TEMPLATE_VSN, "5.0.0").
 -define(PACKAGE_SUFFIX, ".tar.gz").
 
@@ -89,6 +90,27 @@ t_plugins(Config) ->
     {ok, []} = uninstall_plugin(NameVsn),
     ok.
 
+t_install_plugin_matching_exisiting_name(Config) ->
+    DemoShDir = proplists:get_value(demo_sh_dir, Config),
+    PackagePath = get_demo_plugin_package(DemoShDir),
+    NameVsn = filename:basename(PackagePath, ?PACKAGE_SUFFIX),
+    ok = emqx_plugins:ensure_uninstalled(NameVsn),
+    ok = emqx_plugins:delete_package(NameVsn),
+    NameVsn1 = ?EMQX_PLUGIN_TEMPLATE_NAME ++ "_a" ++ "-" ++ ?EMQX_PLUGIN_TEMPLATE_VSN,
+    PackagePath1 = create_renamed_package(PackagePath, NameVsn1),
+    NameVsn1 = filename:basename(PackagePath1, ?PACKAGE_SUFFIX),
+    ok = emqx_plugins:ensure_uninstalled(NameVsn1),
+    ok = emqx_plugins:delete_package(NameVsn1),
+    %% First, install plugin "emqx_plugin_template_a", then:
+    %% "emqx_plugin_template" which matches the beginning
+    %% of the previously installed plugin name
+    ok = install_plugin(PackagePath1),
+    ok = install_plugin(PackagePath),
+    {ok, _} = describe_plugins(NameVsn),
+    {ok, _} = describe_plugins(NameVsn1),
+    {ok, _} = uninstall_plugin(NameVsn),
+    {ok, _} = uninstall_plugin(NameVsn1).
+
 t_bad_plugin(Config) ->
     DemoShDir = proplists:get_value(demo_sh_dir, Config),
     PackagePathOrig = get_demo_plugin_package(DemoShDir),
@@ -160,9 +182,31 @@ uninstall_plugin(Name) ->
 
 get_demo_plugin_package(Dir) ->
     #{package := Pkg} = emqx_plugins_SUITE:get_demo_plugin_package(),
-    FileName = "emqx_plugin_template-" ++ ?EMQX_PLUGIN_TEMPLATE_VSN ++ ?PACKAGE_SUFFIX,
+    FileName = ?EMQX_PLUGIN_TEMPLATE_NAME ++ "-" ++ ?EMQX_PLUGIN_TEMPLATE_VSN ++ ?PACKAGE_SUFFIX,
     PluginPath = "./" ++ FileName,
     Pkg = filename:join([Dir, FileName]),
     _ = os:cmd("cp " ++ Pkg ++ " " ++ PluginPath),
     true = filelib:is_regular(PluginPath),
     PluginPath.
+
+create_renamed_package(PackagePath, NewNameVsn) ->
+    {ok, Content} = erl_tar:extract(PackagePath, [compressed, memory]),
+    {ok, NewName, _Vsn} = emqx_plugins:parse_name_vsn(NewNameVsn),
+    NewNameB = atom_to_binary(NewName, utf8),
+    Content1 = lists:map(
+        fun({F, B}) ->
+            [_ | PathPart] = filename:split(F),
+            B1 = update_release_json(PathPart, B, NewNameB),
+            {filename:join([NewNameVsn | PathPart]), B1}
+        end,
+        Content
+    ),
+    NewPackagePath = filename:join(filename:dirname(PackagePath), NewNameVsn ++ ?PACKAGE_SUFFIX),
+    ok = erl_tar:create(NewPackagePath, Content1, [compressed]),
+    NewPackagePath.
+
+update_release_json(["release.json"], FileContent, NewName) ->
+    ContentMap = emqx_json:decode(FileContent, [return_maps]),
+    emqx_json:encode(ContentMap#{<<"name">> => NewName});
+update_release_json(_FileName, FileContent, _NewName) ->
+    FileContent.
diff --git a/changes/ce/fix-10225.en.md b/changes/ce/fix-10225.en.md
new file mode 100644
index 000000000..20f7dfa47
--- /dev/null
+++ b/changes/ce/fix-10225.en.md
@@ -0,0 +1,2 @@
+Allow installing a plugin if its name matches the beginning of another (already installed) plugin name.
+For example: if plugin "emqx_plugin_template_a" is installed, it must not block installing plugin "emqx_plugin_template".