From d1a1e8041d5268440f5a1cafaa678eaf5fd6e745 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.dev>
Date: Wed, 7 Jun 2023 13:57:16 +0300
Subject: [PATCH 01/33] fix(rebalance api): fix error message formatting

---
 .../src/emqx_node_rebalance_api.erl                    | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
index d0526f5d5..713e16463 100644
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
@@ -258,11 +258,11 @@ wrap_rpc(Node, RPCResult) ->
             {200, #{}};
         {error, Reason} ->
             error_response(
-                400, ?BAD_REQUEST, io_lib:format("error on node ~p: ~p", [Node, Reason])
+                400, ?BAD_REQUEST, binfmt("error on node ~p: ~p", [Node, Reason])
             );
         {badrpc, Reason} ->
             error_response(
-                503, ?RPC_ERROR, io_lib:format("RPC error on node ~p: ~p", [Node, Reason])
+                503, ?RPC_ERROR, binfmt("RPC error on node ~p: ~p", [Node, Reason])
             )
     end.
 
@@ -299,9 +299,9 @@ with_nodes_at_key(Key, Params, Fun) ->
         {ok, Params1} ->
             Fun(Params1);
         {error, {unavailable, Nodes}} ->
-            error_response(400, ?NOT_FOUND, io_lib:format("Nodes unavailable: ~p", [Nodes]));
+            error_response(400, ?NOT_FOUND, binfmt("Nodes unavailable: ~p", [Nodes]));
         {error, {invalid, Nodes}} ->
-            error_response(400, ?BAD_REQUEST, io_lib:format("Invalid nodes: ~p", [Nodes]))
+            error_response(400, ?BAD_REQUEST, binfmt("Invalid nodes: ~p", [Nodes]))
     end.
 
 parse_node(Bin) when is_binary(Bin) ->
@@ -331,6 +331,8 @@ without(Keys, Props) ->
         Props
     ).
 
+binfmt(Fmt, Args) -> iolist_to_binary(io_lib:format(Fmt, Args)).
+
 %%------------------------------------------------------------------------------
 %% Schema
 %%------------------------------------------------------------------------------

From 79d430cf0389b151aff9280cad26629d18c612ef Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Wed, 7 Jun 2023 17:45:14 +0300
Subject: [PATCH 02/33] refactor(pluglib): move `emqx_plugin_libs_proto_v1`
 into emqx app

And rename it to `emqx_metrics_proto_v1` in the process.
---
 .../src/proto/emqx_metrics_proto_v1.erl}      |  19 +--
 .../src/emqx_plugin_libs.app.src              |   2 +-
 .../src/emqx_rule_engine_api.erl              | 144 +++++++++---------
 3 files changed, 87 insertions(+), 78 deletions(-)
 rename apps/{emqx_plugin_libs/src/proto/emqx_plugin_libs_proto_v1.erl => emqx/src/proto/emqx_metrics_proto_v1.erl} (71%)

diff --git a/apps/emqx_plugin_libs/src/proto/emqx_plugin_libs_proto_v1.erl b/apps/emqx/src/proto/emqx_metrics_proto_v1.erl
similarity index 71%
rename from apps/emqx_plugin_libs/src/proto/emqx_plugin_libs_proto_v1.erl
rename to apps/emqx/src/proto/emqx_metrics_proto_v1.erl
index 8cd9952e0..c8c92430e 100644
--- a/apps/emqx_plugin_libs/src/proto/emqx_plugin_libs_proto_v1.erl
+++ b/apps/emqx/src/proto/emqx_metrics_proto_v1.erl
@@ -14,25 +14,26 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--module(emqx_plugin_libs_proto_v1).
+-module(emqx_metrics_proto_v1).
 
 -behaviour(emqx_bpapi).
 
 -export([
     introduced_in/0,
 
-    get_metrics/3
+    get_metrics/4
 ]).
 
--include_lib("emqx/include/bpapi.hrl").
+-include("bpapi.hrl").
 
 introduced_in() ->
-    "5.0.0".
+    "5.1.0".
 
 -spec get_metrics(
-    node(),
+    [node()],
     emqx_metrics_worker:handler_name(),
-    emqx_metrics_worker:metric_id()
-) -> emqx_metrics_worker:metrics() | {badrpc, _}.
-get_metrics(Node, HandlerName, MetricId) ->
-    rpc:call(Node, emqx_metrics_worker, get_metrics, [HandlerName, MetricId]).
+    emqx_metrics_worker:metric_id(),
+    timeout()
+) -> emqx_rpc:erpc_multicall(emqx_metrics_worker:metrics()).
+get_metrics(Nodes, HandlerName, MetricId, Timeout) ->
+    erpc:multicall(Nodes, emqx_metrics_worker, get_metrics, [HandlerName, MetricId], Timeout).
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
index 82a95c377..6953acdf1 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_plugin_libs, [
     {description, "EMQX Plugin utility libs"},
-    {vsn, "4.3.11"},
+    {vsn, "4.3.12"},
     {modules, []},
     {applications, [kernel, stdlib]},
     {env, []}
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index 54a739b6d..b00ef387c 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -44,10 +44,13 @@
 %% query callback
 -export([qs2ms/2, run_fuzzy_match/2, format_rule_info_resp/1]).
 
+-define(RPC_GET_METRICS_TIMEOUT, 5000).
+
 -define(ERR_BADARGS(REASON), begin
     R0 = err_msg(REASON),
     <<"Bad Arguments: ", R0/binary>>
 end).
+
 -define(CHECK_PARAMS(PARAMS, TAG, EXPR),
     case emqx_rule_api_schema:check_params(PARAMS, TAG) of
         {ok, CheckedParams} ->
@@ -56,6 +59,7 @@ end).
             {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(REASON)}}
     end
 ).
+
 -define(METRICS(
     MATCH,
     PASS,
@@ -87,6 +91,7 @@ end).
         'matched.rate.last5m' => RATE_5
     }
 ).
+
 -define(metrics(
     MATCH,
     PASS,
@@ -527,74 +532,77 @@ printable_function_name(Mod, Func) ->
     list_to_binary(lists:concat([Mod, ":", Func])).
 
 get_rule_metrics(Id) ->
-    Format = fun
-        (
-            Node,
-            #{
-                counters :=
-                    #{
-                        'matched' := Matched,
-                        'passed' := Passed,
-                        'failed' := Failed,
-                        'failed.exception' := FailedEx,
-                        'failed.no_result' := FailedNoRes,
-                        'actions.total' := OTotal,
-                        'actions.failed' := OFailed,
-                        'actions.failed.out_of_service' := OFailedOOS,
-                        'actions.failed.unknown' := OFailedUnknown,
-                        'actions.success' := OFailedSucc
-                    },
-                rate :=
-                    #{
-                        'matched' :=
-                            #{current := Current, max := Max, last5m := Last5M}
-                    }
-            }
-        ) ->
-            #{
-                metrics => ?METRICS(
-                    Matched,
-                    Passed,
-                    Failed,
-                    FailedEx,
-                    FailedNoRes,
-                    OTotal,
-                    OFailed,
-                    OFailedOOS,
-                    OFailedUnknown,
-                    OFailedSucc,
-                    Current,
-                    Max,
-                    Last5M
-                ),
-                node => Node
-            };
-        (Node, _Metrics) ->
-            %% Empty metrics: can happen when a node joins another and a bridge is not yet
-            %% replicated to it, so the counters map is empty.
-            #{
-                metrics => ?METRICS(
-                    _Matched = 0,
-                    _Passed = 0,
-                    _Failed = 0,
-                    _FailedEx = 0,
-                    _FailedNoRes = 0,
-                    _OTotal = 0,
-                    _OFailed = 0,
-                    _OFailedOOS = 0,
-                    _OFailedUnknown = 0,
-                    _OFailedSucc = 0,
-                    _Current = 0,
-                    _Max = 0,
-                    _Last5M = 0
-                ),
-                node => Node
-            }
-    end,
-    [
-        Format(Node, emqx_plugin_libs_proto_v1:get_metrics(Node, rule_metrics, Id))
-     || Node <- mria:running_nodes()
-    ].
+    Nodes = emqx:running_nodes(),
+    Results = emqx_metrics_proto_v1:get_metrics(Nodes, rule_metrics, Id, ?RPC_GET_METRICS_TIMEOUT),
+    NodeResults = lists:zip(Nodes, Results),
+    NodeMetrics = [format_metrics(Node, Metrics) || {Node, {ok, Metrics}} <- NodeResults],
+    NodeErrors = [Result || Result = {_Node, {NOk, _}} <- NodeResults, NOk =/= ok],
+    NodeErrors == [] orelse
+        ?SLOG(warning, #{
+            msg => "rpc_get_rule_metrics_errors",
+            errors => NodeErrors
+        }),
+    NodeMetrics.
+
+format_metrics(Node, #{
+    counters :=
+        #{
+            'matched' := Matched,
+            'passed' := Passed,
+            'failed' := Failed,
+            'failed.exception' := FailedEx,
+            'failed.no_result' := FailedNoRes,
+            'actions.total' := OTotal,
+            'actions.failed' := OFailed,
+            'actions.failed.out_of_service' := OFailedOOS,
+            'actions.failed.unknown' := OFailedUnknown,
+            'actions.success' := OFailedSucc
+        },
+    rate :=
+        #{
+            'matched' :=
+                #{current := Current, max := Max, last5m := Last5M}
+        }
+}) ->
+    #{
+        metrics => ?METRICS(
+            Matched,
+            Passed,
+            Failed,
+            FailedEx,
+            FailedNoRes,
+            OTotal,
+            OFailed,
+            OFailedOOS,
+            OFailedUnknown,
+            OFailedSucc,
+            Current,
+            Max,
+            Last5M
+        ),
+        node => Node
+    };
+format_metrics(Node, _Metrics) ->
+    %% Empty metrics: can happen when a node joins another and a bridge is not yet
+    %% replicated to it, so the counters map is empty.
+    #{
+        metrics => ?METRICS(
+            _Matched = 0,
+            _Passed = 0,
+            _Failed = 0,
+            _FailedEx = 0,
+            _FailedNoRes = 0,
+            _OTotal = 0,
+            _OFailed = 0,
+            _OFailedOOS = 0,
+            _OFailedUnknown = 0,
+            _OFailedSucc = 0,
+            _Current = 0,
+            _Max = 0,
+            _Last5M = 0
+        ),
+        node => Node
+    }.
 
 aggregate_metrics(AllMetrics) ->
     InitMetrics = ?METRICS(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),

From 39b0d41527e86108161701b9bd728bea9b864312 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Wed, 7 Jun 2023 18:12:26 +0300
Subject: [PATCH 03/33] fix(bpapi): ignore dropped BPAPI module

It doesn't really breaks interoperability.
---
 apps/emqx/priv/bpapi.versions               |  2 +-
 apps/emqx/test/emqx_bpapi_static_checks.erl | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions
index 8623ef04d..4c2fdd3eb 100644
--- a/apps/emqx/priv/bpapi.versions
+++ b/apps/emqx/priv/bpapi.versions
@@ -28,6 +28,7 @@
 {emqx_management,2}.
 {emqx_management,3}.
 {emqx_management,4}.
+{emqx_metrics,1}.
 {emqx_mgmt_api_plugins,1}.
 {emqx_mgmt_api_plugins,2}.
 {emqx_mgmt_cluster,1}.
@@ -38,7 +39,6 @@
 {emqx_node_rebalance_evacuation,1}.
 {emqx_node_rebalance_status,1}.
 {emqx_persistent_session,1}.
-{emqx_plugin_libs,1}.
 {emqx_plugins,1}.
 {emqx_prometheus,1}.
 {emqx_resource,1}.
diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl
index c944293d2..56baf05e8 100644
--- a/apps/emqx/test/emqx_bpapi_static_checks.erl
+++ b/apps/emqx/test/emqx_bpapi_static_checks.erl
@@ -51,8 +51,14 @@
     "gen_rpc, recon, redbug, observer_cli, snabbkaffe, ekka, mria, amqp_client, rabbit_common"
 ).
 -define(IGNORED_MODULES, "emqx_rpc").
--define(FORCE_DELETED_MODULES, [emqx_statsd, emqx_statsd_proto_v1]).
--define(FORCE_DELETED_APIS, [{emqx_statsd, 1}]).
+-define(FORCE_DELETED_MODULES, [
+    emqx_statsd,
+    emqx_statsd_proto_v1
+]).
+-define(FORCE_DELETED_APIS, [
+    {emqx_statsd, 1},
+    {emqx_plugin_libs, 1}
+]).
 %% List of known RPC backend modules:
 -define(RPC_MODULES, "gen_rpc, erpc, rpc, emqx_rpc").
 %% List of known functions also known to do RPC:

From 7f2de66dab7d2d5b2c2ede30f95385273677fbbe Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.dev>
Date: Wed, 7 Jun 2023 18:35:41 +0300
Subject: [PATCH 04/33] fix(rebalance): add wait_health_check timeout to node
 evacuation

Co-authored-by: Thales Macedo Garitezi <thalesmg@gmail.com>
---
 .../src/emqx_node_rebalance.erl               |  6 +-
 .../src/emqx_node_rebalance_api.erl           | 13 +++-
 .../src/emqx_node_rebalance_cli.erl           |  7 +-
 .../src/emqx_node_rebalance_evacuation.erl    | 66 ++++++++++++-------
 ...emqx_node_rebalance_evacuation_persist.erl | 18 ++---
 .../test/emqx_node_rebalance_api_SUITE.erl    |  2 +
 .../emqx_node_rebalance_evacuation_SUITE.erl  | 24 +++++--
 rel/i18n/emqx_node_rebalance_api.hocon        |  2 +-
 8 files changed, 89 insertions(+), 49 deletions(-)

diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl
index 70c022308..9d53841ed 100644
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.erl
@@ -48,8 +48,8 @@
 -type start_opts() :: #{
     conn_evict_rate => pos_integer(),
     sess_evict_rate => pos_integer(),
-    wait_health_check => pos_integer(),
-    wait_takeover => pos_integer(),
+    wait_health_check => number(),
+    wait_takeover => number(),
     abs_conn_threshold => pos_integer(),
     rel_conn_threshold => number(),
     abs_sess_threshold => pos_integer(),
@@ -438,7 +438,7 @@ is_node_available() ->
     node().
 
 all_nodes() ->
-    mria_mnesia:running_nodes().
+    emqx:running_nodes().
 
 seconds(Sec) ->
     round(timer:seconds(Sec)).
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
index 713e16463..abae139ad 100644
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl
@@ -202,10 +202,10 @@ schema("/load_rebalance/:node/evacuation/stop") ->
     }}.
 
 '/load_rebalance/availability_check'(get, #{}) ->
-    case emqx_eviction_agent:status() of
+    case emqx_node_rebalance_status:local_status() of
         disabled ->
             {200, #{}};
-        {enabled, _Stats} ->
+        _ ->
             error_response(503, ?NODE_EVACUATING, <<"Node Evacuating">>)
     end.
 
@@ -434,6 +434,14 @@ fields(rebalance_start) ->
     ];
 fields(rebalance_evacuation_start) ->
     [
+        {"wait_health_check",
+            mk(
+                emqx_schema:timeout_duration_s(),
+                #{
+                    desc => ?DESC(wait_health_check),
+                    required => false
+                }
+            )},
         {"conn_evict_rate",
             mk(
                 pos_integer(),
@@ -714,6 +722,7 @@ rebalance_example() ->
 
 rebalance_evacuation_example() ->
     #{
+        wait_health_check => 10,
         conn_evict_rate => 100,
         sess_evict_rate => 100,
         redirect_to => <<"othernode:1883">>,
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl
index 3bafb9ffe..66f7a1789 100644
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_cli.erl
@@ -103,6 +103,7 @@ cli(_) ->
         [
             {
                 "rebalance start --evacuation \\\n"
+                "    [--wait-health-check Secs] \\\n"
                 "    [--redirect-to \"Host1:Port1 Host2:Port2 ...\"] \\\n"
                 "    [--conn-evict-rate CountPerSec] \\\n"
                 "    [--migrate-to \"node1@host1 node2@host2 ...\"] \\\n"
@@ -182,8 +183,6 @@ collect_args(["--migrate-to", MigrateTo | Args], Map) ->
 %% rebalance
 collect_args(["--nodes", Nodes | Args], Map) ->
     collect_args(Args, Map#{"--nodes" => Nodes});
-collect_args(["--wait-health-check", WaitHealthCheck | Args], Map) ->
-    collect_args(Args, Map#{"--wait-health-check" => WaitHealthCheck});
 collect_args(["--abs-conn-threshold", AbsConnThres | Args], Map) ->
     collect_args(Args, Map#{"--abs-conn-threshold" => AbsConnThres});
 collect_args(["--rel-conn-threshold", RelConnThres | Args], Map) ->
@@ -193,6 +192,8 @@ collect_args(["--abs-sess-threshold", AbsSessThres | Args], Map) ->
 collect_args(["--rel-sess-threshold", RelSessThres | Args], Map) ->
     collect_args(Args, Map#{"--rel-sess-threshold" => RelSessThres});
 %% common
+collect_args(["--wait-health-check", WaitHealthCheck | Args], Map) ->
+    collect_args(Args, Map#{"--wait-health-check" => WaitHealthCheck});
 collect_args(["--conn-evict-rate", ConnEvictRate | Args], Map) ->
     collect_args(Args, Map#{"--conn-evict-rate" => ConnEvictRate});
 collect_args(["--wait-takeover", WaitTakeover | Args], Map) ->
@@ -207,6 +208,8 @@ validate_evacuation([], Map) ->
     {ok, Map};
 validate_evacuation([{"--evacuation", _} | Rest], Map) ->
     validate_evacuation(Rest, Map);
+validate_evacuation([{"--wait-health-check", _} | _] = Opts, Map) ->
+    validate_pos_int(wait_health_check, Opts, Map, fun validate_evacuation/2);
 validate_evacuation([{"--redirect-to", ServerReference} | Rest], Map) ->
     validate_evacuation(Rest, Map#{server_reference => list_to_binary(ServerReference)});
 validate_evacuation([{"--conn-evict-rate", _} | _] = Opts, Map) ->
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl
index 4de362ca9..6b6aa0675 100644
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl
@@ -53,10 +53,11 @@
     server_reference => emqx_eviction_agent:server_reference(),
     conn_evict_rate => pos_integer(),
     sess_evict_rate => pos_integer(),
-    wait_takeover => pos_integer(),
-    migrate_to => migrate_to()
+    wait_takeover => number(),
+    migrate_to => migrate_to(),
+    wait_health_check => number()
 }.
--type start_error() :: already_started | eviction_agent_busy.
+-type start_error() :: already_started.
 -type stats() :: #{
     initial_conns := non_neg_integer(),
     initial_sessions := non_neg_integer(),
@@ -97,7 +98,7 @@ available_nodes(Nodes) when is_list(Nodes) ->
 
 callback_mode() -> handle_event_function.
 
-%% states: disabled, evicting_conns, waiting_takeover, evicting_sessions, prohibiting
+%% states: disabled, waiting_health_check, evicting_conns, waiting_takeover, evicting_sessions, prohibiting
 
 init([]) ->
     case emqx_node_rebalance_evacuation_persist:read(default_opts()) of
@@ -119,25 +120,20 @@ init([]) ->
 %% start
 handle_event(
     {call, From},
-    {start, #{server_reference := ServerReference} = Opts},
+    {start, #{wait_health_check := WaitHealthCheck} = Opts},
     disabled,
     #{} = Data
 ) ->
-    case emqx_eviction_agent:enable(?MODULE, ServerReference) of
-        ok ->
-            NewData = init_data(Data, Opts),
-            ok = emqx_node_rebalance_evacuation_persist:save(Opts),
-            ?SLOG(warning, #{
-                msg => "node_evacuation_started",
-                opts => Opts
-            }),
-            {next_state, evicting_conns, NewData, [
-                {state_timeout, 0, evict_conns},
-                {reply, From, ok}
-            ]};
-        {error, eviction_agent_busy} ->
-            {keep_state_and_data, [{reply, From, {error, eviction_agent_busy}}]}
-    end;
+    ?SLOG(warning, #{
+        msg => "node_evacuation_started",
+        opts => Opts
+    }),
+    NewData = init_data(Data, Opts),
+    ok = emqx_node_rebalance_evacuation_persist:save(Opts),
+    {next_state, waiting_health_check, NewData, [
+        {state_timeout, seconds(WaitHealthCheck), start_eviction},
+        {reply, From, ok}
+    ]};
 handle_event({call, From}, {start, _Opts}, _State, #{}) ->
     {keep_state_and_data, [{reply, From, {error, already_started}}]};
 %% stop
@@ -167,6 +163,27 @@ handle_event({call, From}, status, State, #{migrate_to := MigrateTo} = Data) ->
     {keep_state_and_data, [
         {reply, From, {enabled, Stats#{state => State, migrate_to => migrate_to(MigrateTo)}}}
     ]};
+%% start eviction
+handle_event(
+    state_timeout,
+    start_eviction,
+    waiting_health_check,
+    #{server_reference := ServerReference} = Data
+) ->
+    case emqx_eviction_agent:enable(?MODULE, ServerReference) of
+        ok ->
+            ?tp(debug, eviction_agent_started, #{
+                data => Data
+            }),
+            {next_state, evicting_conns, Data, [
+                {state_timeout, 0, evict_conns}
+            ]};
+        {error, eviction_agent_busy} ->
+            ?tp(warning, eviction_agent_busy, #{
+                data => Data
+            }),
+            {next_state, disabled, deinit(Data)}
+    end;
 %% conn eviction
 handle_event(
     state_timeout,
@@ -270,12 +287,14 @@ default_opts() ->
         conn_evict_rate => ?DEFAULT_CONN_EVICT_RATE,
         sess_evict_rate => ?DEFAULT_SESS_EVICT_RATE,
         wait_takeover => ?DEFAULT_WAIT_TAKEOVER,
+        wait_health_check => ?DEFAULT_WAIT_HEALTH_CHECK,
         migrate_to => undefined
     }.
 
 init_data(Data0, Opts) ->
     Data1 = maps:merge(Data0, Opts),
-    {enabled, #{connections := ConnCount, sessions := SessCount}} = emqx_eviction_agent:status(),
+    ConnCount = emqx_eviction_agent:connection_count(),
+    SessCount = emqx_eviction_agent:session_count(),
     Data1#{
         initial_conns => ConnCount,
         current_conns => ConnCount,
@@ -305,4 +324,7 @@ is_node_available() ->
     node().
 
 all_nodes() ->
-    mria_mnesia:running_nodes() -- [node()].
+    emqx:running_nodes() -- [node()].
+
+seconds(Sec) ->
+    round(timer:seconds(Sec)).
diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl
index 6b145c699..f32bc6ddd 100644
--- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl
+++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation_persist.erl
@@ -21,24 +21,16 @@
 %% APIs
 %%--------------------------------------------------------------------
 
-%% do not persist `migrate_to`:
-%% * after restart there is nothing to migrate
-%% * this value may be invalid after node was offline
--type persisted_start_opts() :: #{
-    server_reference => emqx_eviction_agent:server_reference(),
-    conn_evict_rate => pos_integer(),
-    sess_evict_rate => pos_integer(),
-    wait_takeover => pos_integer()
-}.
 -type start_opts() :: #{
     server_reference => emqx_eviction_agent:server_reference(),
     conn_evict_rate => pos_integer(),
     sess_evict_rate => pos_integer(),
-    wait_takeover => pos_integer(),
-    migrate_to => emqx_node_rebalance_evacuation:migrate_to()
+    wait_takeover => number(),
+    migrate_to => emqx_node_rebalance_evacuation:migrate_to(),
+    wait_health_check => number()
 }.
 
--spec save(persisted_start_opts()) -> ok_or_error(term()).
+-spec save(start_opts()) -> ok_or_error(term()).
 save(
     #{
         server_reference := ServerReference,
@@ -50,7 +42,7 @@ save(
     (is_binary(ServerReference) orelse ServerReference =:= undefined) andalso
         is_integer(ConnEvictRate) andalso ConnEvictRate > 0 andalso
         is_integer(SessEvictRate) andalso SessEvictRate > 0 andalso
-        is_integer(WaitTakeover) andalso WaitTakeover >= 0
+        is_number(WaitTakeover) andalso WaitTakeover >= 0
 ->
     Filepath = evacuation_filepath(),
     case filelib:ensure_dir(Filepath) of
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl
index d8202a33e..119b4a5d9 100644
--- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl
+++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl
@@ -69,6 +69,7 @@ t_start_evacuation_validation(Config) ->
         #{sess_evict_rate => <<"sess">>},
         #{redirect_to => 123},
         #{wait_takeover => <<"wait">>},
+        #{wait_health_check => <<"wait">>},
         #{migrate_to => []},
         #{migrate_to => <<"migrate_to">>},
         #{migrate_to => [<<"bad_node">>]},
@@ -103,6 +104,7 @@ t_start_evacuation_validation(Config) ->
                 conn_evict_rate => 10,
                 sess_evict_rate => 10,
                 wait_takeover => 10,
+                wait_health_check => 10,
                 redirect_to => <<"srv">>,
                 migrate_to => [atom_to_binary(RecipientNode)]
             }
diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl
index 5d774ba7c..b7f1ebb63 100644
--- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl
+++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_evacuation_SUITE.erl
@@ -86,11 +86,13 @@ end_per_testcase(_Case, Config) ->
 
 t_agent_busy(Config) ->
     [{DonorNode, _DonorPort}] = ?config(cluster_nodes, Config),
+
     ok = rpc:call(DonorNode, emqx_eviction_agent, enable, [other_rebalance, undefined]),
 
-    ?assertEqual(
-        {error, eviction_agent_busy},
-        rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)])
+    ?assertWaitEvent(
+        rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]),
+        #{?snk_kind := eviction_agent_busy},
+        5000
     ).
 
 t_already_started(Config) ->
@@ -115,7 +117,12 @@ t_start(Config) ->
 
     [{DonorNode, DonorPort}] = ?config(cluster_nodes, Config),
 
-    ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]),
+    ?assertWaitEvent(
+        rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]),
+        #{?snk_kind := eviction_agent_started},
+        5000
+    ),
+
     ?assertMatch(
         {error, {use_another_server, #{}}},
         emqtt_try_connect([{port, DonorPort}])
@@ -126,7 +133,11 @@ t_persistence(Config) ->
 
     [{DonorNode, DonorPort}] = ?config(cluster_nodes, Config),
 
-    ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]),
+    ?assertWaitEvent(
+        rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]),
+        #{?snk_kind := eviction_agent_started},
+        5000
+    ),
 
     ?assertMatch(
         {error, {use_another_server, #{}}},
@@ -179,7 +190,7 @@ t_conn_evicted(Config) ->
     ?assertWaitEvent(
         ok = rpc:call(DonorNode, emqx_node_rebalance_evacuation, start, [opts(Config)]),
         #{?snk_kind := node_evacuation_evict_conn},
-        1000
+        5000
     ),
 
     ?assertMatch(
@@ -251,6 +262,7 @@ opts(Config) ->
         conn_evict_rate => 10,
         sess_evict_rate => 10,
         wait_takeover => 1,
+        wait_health_check => 1,
         migrate_to => migrate_to(Config)
     }.
 
diff --git a/rel/i18n/emqx_node_rebalance_api.hocon b/rel/i18n/emqx_node_rebalance_api.hocon
index bb67f2aad..8b598134a 100644
--- a/rel/i18n/emqx_node_rebalance_api.hocon
+++ b/rel/i18n/emqx_node_rebalance_api.hocon
@@ -49,7 +49,7 @@ param_node.label:
 """Node name"""
 
 wait_health_check.desc:
-"""Time to wait before starting the rebalance process, in seconds"""
+"""Time to wait before starting the rebalance/evacuation process, in seconds"""
 
 wait_health_check.label:
 """Wait health check"""

From 54f7941329885adfa172fa66b389778e251aa0a2 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.dev>
Date: Wed, 7 Jun 2023 18:54:54 +0300
Subject: [PATCH 05/33] fix(rebalancing): add changelog

Co-authored-by: Thales Macedo Garitezi <thalesmg@gmail.com>
---
 changes/ee/fix-10967.en.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 changes/ee/fix-10967.en.md

diff --git a/changes/ee/fix-10967.en.md b/changes/ee/fix-10967.en.md
new file mode 100644
index 000000000..55cc2b975
--- /dev/null
+++ b/changes/ee/fix-10967.en.md
@@ -0,0 +1,3 @@
+Fixed error message formatting in rebalance API: previously they could be displayed as unclear dumps of internal Erlang structures.
+
+Added `wait_health_check` option to node evacuation CLI and API. This is a time interval when the node reports "unhealthy status" without beginning actual evacuation. We need this to allow a Load Balancer (if any) to remove the evacuated node from balancing and not forward (re)connecting clients to the evacuated node.

From 711cedc282824509f537e8078358b328d50cdf1d Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Thu, 8 Jun 2023 11:46:16 +0200
Subject: [PATCH 06/33] fix: missing 404 response in rule engine api schema

---
 apps/emqx_rule_engine/src/emqx_rule_engine_api.erl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index b00ef387c..2e6952920 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -253,6 +253,7 @@ schema("/rules/:id") ->
             summary => <<"Delete rule">>,
             parameters => param_path_id(),
             responses => #{
+                404 => error_schema('NOT_FOUND', "Rule not found"),
                 204 => <<"Delete rule successfully">>
             }
         }

From b481994e740524a7fe2e1bfd46afa30f474755f6 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 8 Jun 2023 12:30:35 +0200
Subject: [PATCH 07/33] chore: bump to version 5.1.0-alpha.4

---
 apps/emqx/include/emqx_release.hrl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index 6731ea285..cce24f34a 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -32,10 +32,10 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Opensource edition
--define(EMQX_RELEASE_CE, "5.1.0-alpha.3").
+-define(EMQX_RELEASE_CE, "5.1.0-alpha.4").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.1.0-alpha.3").
+-define(EMQX_RELEASE_EE, "5.1.0-alpha.4").
 
 %% The HTTP API version
 -define(EMQX_API_VERSION, "5.0").

From f5e93b1b2b61ec2ccda939c34623951cca7dfedb Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Thu, 8 Jun 2023 13:08:45 +0200
Subject: [PATCH 08/33] docs(emqx_prometheus): leave remark about json being
 deprecated

---
 apps/emqx_prometheus/src/emqx_prometheus_api.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl
index 6fb57483d..c2bfaefc8 100644
--- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl
+++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl
@@ -18,7 +18,6 @@
 
 -behaviour(minirest_api).
 
--include("emqx_prometheus.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
 -import(hoconsc, [ref/2]).
@@ -128,7 +127,8 @@ prometheus_config_example() ->
 
 prometheus_data_schema() ->
     #{
-        description => <<"Get Prometheus Data">>,
+        description =>
+            <<"Get Prometheus Data. Note that support for JSON output is deprecated and will be removed in v5.2.">>,
         content =>
             #{
                 'application/json' =>

From 47fa17b3c17344fa14bd47a3532af4e39c297b15 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Wed, 7 Jun 2023 20:00:17 +0200
Subject: [PATCH 09/33] feat: add sync/async option to the Kafka producer
 bridge

This commit makes it possible to configure if a Kafka bridge should work
in query mode sync or async by setting the resource_opts.query_mode
configuration option.

Fixes:
https://emqx.atlassian.net/browse/EMQX-8631
---
 .../src/emqx_bridge_kafka.erl                 | 37 +++++++++++++++++--
 .../src/emqx_bridge_kafka_impl_producer.erl   | 20 ++++++----
 apps/emqx_resource/src/emqx_resource.erl      |  9 ++++-
 rel/i18n/emqx_bridge_kafka.hocon              | 24 ++++++++++++
 4 files changed, 78 insertions(+), 12 deletions(-)

diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
index 64f2394c4..9cabaf5e9 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
@@ -147,7 +147,7 @@ fields("config_producer") ->
 fields("config_consumer") ->
     fields(kafka_consumer);
 fields(kafka_producer) ->
-    fields("config") ++ fields(producer_opts);
+    fields("config") ++ fields(resource_opts) ++ fields(producer_opts);
 fields(kafka_consumer) ->
     fields("config") ++ fields(consumer_opts);
 fields("config") ->
@@ -294,7 +294,37 @@ fields(producer_kafka_opts) ->
             mk(ref(producer_buffer), #{
                 required => false,
                 desc => ?DESC(producer_buffer)
-            })}
+            })},
+        {query_mode_sync_timeout,
+            mk(
+                emqx_schema:duration_ms(),
+                #{
+                    default => <<"5s">>,
+                    desc => ?DESC(query_mode_sync_timeout)
+                }
+            )}
+    ];
+fields(resource_opts) ->
+    [
+        {resource_opts,
+            mk(
+                ref(?MODULE, resource_opts_fields),
+                #{
+                    required => false,
+                    desc => ?DESC(resource_opts)
+                }
+            )}
+    ];
+fields(resource_opts_fields) ->
+    [
+        {query_mode,
+            mk(
+                enum([async, sync]),
+                #{
+                    default => async,
+                    desc => ?DESC(query_mode)
+                }
+            )}
     ];
 fields(kafka_message) ->
     [
@@ -410,7 +440,8 @@ struct_names() ->
         producer_opts,
         consumer_opts,
         consumer_kafka_opts,
-        consumer_topic_mapping
+        consumer_topic_mapping,
+        resource_opts_fields
     ].
 
 %% -------------------------------------------------------------------------------------------------
diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
index 8b8337b09..8be506efd 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
@@ -43,7 +43,11 @@ on_start(InstId, Config) ->
         bootstrap_hosts := Hosts0,
         bridge_name := BridgeName,
         connect_timeout := ConnTimeout,
-        kafka := KafkaConfig = #{message := MessageTemplate, topic := KafkaTopic},
+        kafka := KafkaConfig = #{
+            message := MessageTemplate,
+            topic := KafkaTopic,
+            query_mode_sync_timeout := QueryModeSyncTimeout
+        },
         metadata_request_timeout := MetaReqTimeout,
         min_metadata_refresh_interval := MinMetaRefreshInterval,
         socket_opts := SocketOpts,
@@ -99,7 +103,8 @@ on_start(InstId, Config) ->
                 client_id => ClientId,
                 kafka_topic => KafkaTopic,
                 producers => Producers,
-                resource_id => ResourceId
+                resource_id => ResourceId,
+                query_mode_sync_timeout => QueryModeSyncTimeout
             }};
         {error, Reason2} ->
             ?SLOG(error, #{
@@ -189,14 +194,15 @@ on_stop(InstanceId, _State) ->
 on_query(
     _InstId,
     {send_message, Message},
-    #{message_template := Template, producers := Producers}
+    #{
+        message_template := Template,
+        producers := Producers,
+        query_mode_sync_timeout := SyncTimeout
+    }
 ) ->
     KafkaMessage = render_message(Template, Message),
-    %% TODO: this function is not used so far,
-    %% timeout should be configurable
-    %% or the on_query/3 should be on_query/4 instead.
     try
-        {_Partition, _Offset} = wolff:send_sync(Producers, [KafkaMessage], 5000),
+        {_Partition, _Offset} = wolff:send_sync(Producers, [KafkaMessage], SyncTimeout),
         ok
     catch
         error:{producer_down, _} = Reason ->
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index 0dbc3067f..c700cfd86 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -276,13 +276,13 @@ query(ResId, Request) ->
     Result :: term().
 query(ResId, Request, Opts) ->
     case emqx_resource_manager:lookup_cached(ResId) of
-        {ok, _Group, #{query_mode := QM, mod := Module}} ->
+        {ok, _Group, #{query_mode := QM, mod := Module} = Config} ->
             IsBufferSupported = is_buffer_supported(Module),
             case {IsBufferSupported, QM} of
                 {true, _} ->
                     %% only Kafka producer so far
                     Opts1 = Opts#{is_buffer_supported => true},
-                    emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts1);
+                    do_query_built_in_buffer(QM, ResId, Request, Opts1);
                 {false, sync} ->
                     emqx_resource_buffer_worker:sync_query(ResId, Request, Opts);
                 {false, async} ->
@@ -292,6 +292,11 @@ query(ResId, Request, Opts) ->
             ?RESOURCE_ERROR(not_found, "resource not found")
     end.
 
+do_query_built_in_buffer(async, ResId, Request, Opts1) ->
+    emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts1);
+do_query_built_in_buffer(sync, ResId, Request, _Opts1) ->
+    emqx_resource_buffer_worker:simple_sync_query(ResId, Request).
+
 -spec simple_sync_query(resource_id(), Request :: term()) -> Result :: term().
 simple_sync_query(ResId, Request) ->
     emqx_resource_buffer_worker:simple_sync_query(ResId, Request).
diff --git a/rel/i18n/emqx_bridge_kafka.hocon b/rel/i18n/emqx_bridge_kafka.hocon
index ef2e27972..740d94984 100644
--- a/rel/i18n/emqx_bridge_kafka.hocon
+++ b/rel/i18n/emqx_bridge_kafka.hocon
@@ -358,4 +358,28 @@ compression.desc:
 compression.label:
 """Compression"""
 
+query_mode.desc:
+"""Query mode. Optional 'sync/async', default 'async'."""
+
+query_mode.label:
+"""Query mode"""
+
+resource_opts.desc:
+"""Resource options."""
+
+resource_opts.label:
+"""Resource Options"""
+
+resource_opts_fields.desc:
+"""Resource options."""
+
+resource_opts_fields.label:
+"""Resource Options"""
+
+query_mode_sync_timeout.desc:
+"""This parameter defines the timeout limit for synchronous queries. It applies only when the bridge query mode is configured to 'sync'."""
+
+query_mode_sync_timeout.label:
+"""Synchronous Query Timeout"""
+
 }

From ae97d516b8f54abc0c9c50f58c8e26e719b0611d Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Wed, 7 Jun 2023 23:25:30 +0300
Subject: [PATCH 10/33] fix(ft): add `enable` flag to every backend

And tolerate multiple configured backends, as long as only one of
them is enabled.
---
 apps/emqx_ft/README.md                        |  3 ++
 apps/emqx_ft/src/emqx_ft_schema.erl           | 43 ++++++++++++++++---
 apps/emqx_ft/src/emqx_ft_storage.erl          |  4 +-
 apps/emqx_ft/src/emqx_ft_storage_exporter.erl |  6 +--
 .../src/emqx_ft_storage_exporter_fs.erl       |  1 +
 apps/emqx_ft/src/emqx_ft_storage_fs.erl       |  1 +
 apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl |  1 +
 apps/emqx_ft/test/emqx_ft_conf_SUITE.erl      |  9 +++-
 .../test/emqx_ft_storage_fs_gc_SUITE.erl      |  3 +-
 apps/emqx_ft/test/emqx_ft_test_helpers.erl    |  9 +++-
 rel/i18n/emqx_ft_schema.hocon                 |  3 ++
 11 files changed, 68 insertions(+), 15 deletions(-)

diff --git a/apps/emqx_ft/README.md b/apps/emqx_ft/README.md
index 019c63dea..eaf47d6df 100644
--- a/apps/emqx_ft/README.md
+++ b/apps/emqx_ft/README.md
@@ -29,6 +29,7 @@ file_transfer {
   enable = true
   storage {
     local {
+      enable = true
       exporter {
         local { root = "/var/lib/emqx/transfers" }
       }
@@ -50,7 +51,9 @@ file_transfer {
   enable = true
   storage {
     local {
+      enable = true
       exporter {
+        enable = true
         s3 {
           host = "s3.us-east-1.amazonaws.com"
           port = "443"
diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl
index c1ee41d0d..d7df09a54 100644
--- a/apps/emqx_ft/src/emqx_ft_schema.erl
+++ b/apps/emqx_ft/src/emqx_ft_schema.erl
@@ -26,6 +26,7 @@
 -export([schema/1]).
 
 -export([translate/1]).
+-export([backend/1]).
 
 -type json_value() ::
     null
@@ -142,7 +143,7 @@ fields(local_storage) ->
                     }
                 }
             )}
-    ];
+    ] ++ common_backend_fields();
 fields(local_storage_segments) ->
     [
         {root,
@@ -190,9 +191,9 @@ fields(local_storage_exporter) ->
                     required => false
                 }
             )}
-    ];
+    ] ++ common_backend_fields();
 fields(s3_exporter) ->
-    emqx_s3_schema:fields(s3);
+    emqx_s3_schema:fields(s3) ++ common_backend_fields();
 fields(local_storage_segments_gc) ->
     [
         {interval,
@@ -229,6 +230,18 @@ fields(local_storage_segments_gc) ->
             )}
     ].
 
+common_backend_fields() ->
+    [
+        {enable,
+            mk(
+                boolean(), #{
+                    desc => ?DESC("backend_enable"),
+                    required => false,
+                    default => true
+                }
+            )}
+    ].
+
 desc(file_transfer) ->
     "File transfer settings";
 desc(local_storage) ->
@@ -275,11 +288,19 @@ validator(filename) ->
     ];
 validator(backend) ->
     fun(Config) ->
-        case maps:keys(Config) of
-            [_Type] ->
+        Enabled = maps:filter(
+            fun(_, Backend) ->
+                maps:get(enable, Backend, false) or maps:get(<<"enable">>, Backend, false)
+            end,
+            Config
+        ),
+        case maps:to_list(Enabled) of
+            [{_Type, _BackendConfig}] ->
                 ok;
             _Conflicts = [_ | _] ->
-                {error, multiple_conflicting_backends}
+                {error, multiple_enabled_backends};
+            _None = [] ->
+                {error, no_enabled_backend}
         end
     end.
 
@@ -319,3 +340,13 @@ translate(Conf) ->
             ?MODULE, #{atom_to_binary(Root) => Conf}, #{atom_key => true}, [Root]
         )
     ).
+
+-spec backend(emqx_config:config()) ->
+    {_Type :: atom(), emqx_config:config()}.
+backend(Config) ->
+    catch maps:foreach(fun emit_enabled/2, Config).
+
+-spec emit_enabled(atom(), emqx_config:config()) ->
+    no_return().
+emit_enabled(Type, BConf = #{enable := Enabled}) ->
+    Enabled andalso throw({Type, BConf}).
diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl
index 007b47db9..e2980c920 100644
--- a/apps/emqx_ft/src/emqx_ft_storage.erl
+++ b/apps/emqx_ft/src/emqx_ft_storage.erl
@@ -182,8 +182,8 @@ on_backend_update(BackendOld, BackendNew) when
 %%--------------------------------------------------------------------
 
 -spec backend(config()) -> backend().
-backend(#{local := Storage}) ->
-    {local, Storage}.
+backend(Config) ->
+    emqx_ft_schema:backend(Config).
 
 on_storage_start({Type, Storage}) ->
     (mod(Type)):start(Storage).
diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl
index 6f2b9bea1..4c9cac67a 100644
--- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl
+++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl
@@ -175,10 +175,10 @@ stop({ExporterMod, ExporterOpts}) ->
 %%------------------------------------------------------------------------------
 
 exporter(Storage) ->
-    case maps:get(exporter, Storage) of
-        #{local := Options} ->
+    case emqx_ft_schema:backend(maps:get(exporter, Storage)) of
+        {local, Options} ->
             {emqx_ft_storage_exporter_fs, Options};
-        #{s3 := Options} ->
+        {s3, Options} ->
             {emqx_ft_storage_exporter_s3, Options}
     end.
 
diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl
index e211cb421..589949bda 100644
--- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl
+++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl
@@ -46,6 +46,7 @@
 -export_type([options/0]).
 
 -type options() :: #{
+    enable := true,
     root => file:name(),
     _ => _
 }.
diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl
index e84d35328..85aa08405 100644
--- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl
+++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl
@@ -102,6 +102,7 @@
 
 -type storage() :: #{
     type := 'local',
+    enable := true,
     segments := segments(),
     exporter := emqx_ft_storage_exporter:exporter()
 }.
diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl
index 1dcc8a79d..50d8579fc 100644
--- a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl
+++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl
@@ -256,6 +256,7 @@ storage(Config) ->
                     },
                     <<"exporter">> => #{
                         <<"local">> => #{
+                            <<"enable">> => true,
                             <<"root">> => ?config(exports_root, Config)
                         }
                     }
diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl
index f235b5ebb..0f949855f 100644
--- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl
+++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl
@@ -77,6 +77,7 @@ t_update_config(_Config) ->
                         },
                         <<"exporter">> => #{
                             <<"local">> => #{
+                                <<"enable">> => true,
                                 <<"root">> => <<"/tmp/exports">>
                             }
                         }
@@ -104,7 +105,10 @@ t_disable_restore_config(Config) ->
         {ok, _},
         emqx_conf:update(
             [file_transfer],
-            #{<<"enable">> => true, <<"storage">> => #{<<"local">> => #{}}},
+            #{
+                <<"enable">> => true,
+                <<"storage">> => #{<<"local">> => #{}}
+            },
             #{}
         )
     ),
@@ -207,6 +211,7 @@ t_switch_exporter(_Config) ->
             [file_transfer, storage, local, exporter],
             #{
                 <<"s3">> => #{
+                    <<"enable">> => true,
                     <<"bucket">> => <<"emqx">>,
                     <<"host">> => <<"https://localhost">>,
                     <<"port">> => 9000,
@@ -234,7 +239,7 @@ t_switch_exporter(_Config) ->
         {ok, _},
         emqx_conf:update(
             [file_transfer, storage, local, exporter],
-            #{<<"local">> => #{}},
+            #{<<"local">> => #{<<"enable">> => true}},
             #{}
         )
     ),
diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl
index 12f91c808..e008bb2b1 100644
--- a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl
+++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl
@@ -45,9 +45,10 @@ init_per_testcase(TC, Config) ->
                 <<"enable">> => true,
                 <<"storage">> => #{
                     <<"local">> => #{
+                        <<"enable">> => true,
                         <<"segments">> => #{<<"root">> => SegmentsRoot},
                         <<"exporter">> => #{
-                            <<"local">> => #{<<"root">> => ExportsRoot}
+                            <<"local">> => #{<<"enable">> => true, <<"root">> => ExportsRoot}
                         }
                     }
                 }
diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl
index 2eb6d84db..a041dcd50 100644
--- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl
+++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl
@@ -55,17 +55,24 @@ local_storage(Config) ->
 local_storage(Config, Opts) ->
     #{
         <<"local">> => #{
+            <<"enable">> => true,
             <<"segments">> => #{<<"root">> => root(Config, node(), [segments])},
             <<"exporter">> => exporter(Config, Opts)
         }
     }.
 
 exporter(Config, #{exporter := local}) ->
-    #{<<"local">> => #{<<"root">> => root(Config, node(), [exports])}};
+    #{
+        <<"local">> => #{
+            <<"enable">> => true,
+            <<"root">> => root(Config, node(), [exports])
+        }
+    };
 exporter(_Config, #{exporter := s3, bucket_name := BucketName}) ->
     BaseConfig = emqx_s3_test_helpers:base_raw_config(tcp),
     #{
         <<"s3">> => BaseConfig#{
+            <<"enable">> => true,
             <<"bucket">> => list_to_binary(BucketName),
             <<"host">> => ?S3_HOST,
             <<"port">> => ?S3_PORT
diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon
index bafda331a..aabdd2032 100644
--- a/rel/i18n/emqx_ft_schema.hocon
+++ b/rel/i18n/emqx_ft_schema.hocon
@@ -18,6 +18,9 @@ store_segment_timeout.desc:
 """Timeout for storing a file segment.<br/>
 After reaching the timeout, message with the segment will be acked with an error"""
 
+backend_enable.desc:
+"""Whether to enable this backend."""
+
 storage_backend.desc:
 """Storage settings for file transfer."""
 

From e3c16be3c4b3bc2691802212f2c5736d981f5bb4 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Thu, 8 Jun 2023 08:13:42 +0300
Subject: [PATCH 11/33] fix(ftschema): ensure `translate` follow `emqx_config`
 behavior

So that atom keys won't unexpectedly get to converters / validators.
---
 apps/emqx_ft/src/emqx_ft_schema.erl | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl
index d7df09a54..6ee7c5e10 100644
--- a/apps/emqx_ft/src/emqx_ft_schema.erl
+++ b/apps/emqx_ft/src/emqx_ft_schema.erl
@@ -288,12 +288,7 @@ validator(filename) ->
     ];
 validator(backend) ->
     fun(Config) ->
-        Enabled = maps:filter(
-            fun(_, Backend) ->
-                maps:get(enable, Backend, false) or maps:get(<<"enable">>, Backend, false)
-            end,
-            Config
-        ),
+        Enabled = maps:filter(fun(_, #{<<"enable">> := E}) -> E end, Config),
         case maps:to_list(Enabled) of
             [{_Type, _BackendConfig}] ->
                 ok;
@@ -334,12 +329,9 @@ ref(Ref) ->
 
 translate(Conf) ->
     [Root] = roots(),
-    maps:get(
-        Root,
-        hocon_tconf:check_plain(
-            ?MODULE, #{atom_to_binary(Root) => Conf}, #{atom_key => true}, [Root]
-        )
-    ).
+    RootRaw = atom_to_binary(Root),
+    ConfChecked = hocon_tconf:check_plain(?MODULE, #{RootRaw => Conf}, #{}, [Root]),
+    emqx_utils_maps:unsafe_atom_key_map(maps:get(RootRaw, ConfChecked)).
 
 -spec backend(emqx_config:config()) ->
     {_Type :: atom(), emqx_config:config()}.

From ce8cc05cc886b37fc9f781aa8999b0482ac888d7 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Thu, 8 Jun 2023 14:47:10 +0300
Subject: [PATCH 12/33] chore(ftschema): mention `translate` is for tests
 purpose only

---
 apps/emqx_ft/src/emqx_ft_schema.erl | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl
index 6ee7c5e10..90b80ea3d 100644
--- a/apps/emqx_ft/src/emqx_ft_schema.erl
+++ b/apps/emqx_ft/src/emqx_ft_schema.erl
@@ -25,9 +25,12 @@
 
 -export([schema/1]).
 
--export([translate/1]).
+%% Utilities
 -export([backend/1]).
 
+%% Test-only helpers
+-export([translate/1]).
+
 -type json_value() ::
     null
     | boolean()
@@ -327,11 +330,7 @@ converter(unicode_string) ->
 ref(Ref) ->
     ref(?MODULE, Ref).
 
-translate(Conf) ->
-    [Root] = roots(),
-    RootRaw = atom_to_binary(Root),
-    ConfChecked = hocon_tconf:check_plain(?MODULE, #{RootRaw => Conf}, #{}, [Root]),
-    emqx_utils_maps:unsafe_atom_key_map(maps:get(RootRaw, ConfChecked)).
+%% Utilities
 
 -spec backend(emqx_config:config()) ->
     {_Type :: atom(), emqx_config:config()}.
@@ -342,3 +341,13 @@ backend(Config) ->
     no_return().
 emit_enabled(Type, BConf = #{enable := Enabled}) ->
     Enabled andalso throw({Type, BConf}).
+
+%% Test-only helpers
+
+-spec translate(emqx_config:raw_config()) ->
+    emqx_config:config().
+translate(Conf) ->
+    [Root] = roots(),
+    RootRaw = atom_to_binary(Root),
+    ConfChecked = hocon_tconf:check_plain(?MODULE, #{RootRaw => Conf}, #{}, [Root]),
+    emqx_utils_maps:unsafe_atom_key_map(maps:get(RootRaw, ConfChecked)).

From abbba7119125e59b7c3a37ca984eb56f0fbb0f80 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Thu, 8 Jun 2023 14:59:49 +0200
Subject: [PATCH 13/33] fix(tls): issue when ssl listner is configured to use
 tls v1.3 only

clients could not connect due to incompatible tls options if ssl listner is configured to use tls v1.3 only
---
 apps/emqx/src/emqx_tls_lib.erl        | 65 +++++++++++++++++++--------
 apps/emqx/test/emqx_tls_lib_tests.erl | 65 +++++++++++++++++++++++++++
 2 files changed, 112 insertions(+), 18 deletions(-)

diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl
index 653f26708..b0115ddc5 100644
--- a/apps/emqx/src/emqx_tls_lib.erl
+++ b/apps/emqx/src/emqx_tls_lib.erl
@@ -485,7 +485,8 @@ to_server_opts(Type, Opts) ->
             cacertfile => Path(cacertfile),
             ciphers => Ciphers,
             versions => Versions
-        })
+        }),
+        Versions
     ).
 
 %% @doc Convert hocon-checked tls client options (map()) to
@@ -510,19 +511,22 @@ to_client_opts(Type, Opts) ->
             SNI = ensure_sni(Get(server_name_indication)),
             Versions = integral_versions(Type, Get(versions)),
             Ciphers = integral_ciphers(Versions, Get(ciphers)),
-            filter([
-                {keyfile, KeyFile},
-                {certfile, CertFile},
-                {cacertfile, CAFile},
-                {verify, Verify},
-                {server_name_indication, SNI},
-                {versions, Versions},
-                {ciphers, Ciphers},
-                {reuse_sessions, Get(reuse_sessions)},
-                {depth, Get(depth)},
-                {password, ensure_str(Get(password))},
-                {secure_renegotiate, Get(secure_renegotiate)}
-            ]);
+            filter(
+                [
+                    {keyfile, KeyFile},
+                    {certfile, CertFile},
+                    {cacertfile, CAFile},
+                    {verify, Verify},
+                    {server_name_indication, SNI},
+                    {versions, Versions},
+                    {ciphers, Ciphers},
+                    {reuse_sessions, Get(reuse_sessions)},
+                    {depth, Get(depth)},
+                    {password, ensure_str(Get(password))},
+                    {secure_renegotiate, Get(secure_renegotiate)}
+                ],
+                Versions
+            );
         false ->
             []
     end.
@@ -552,10 +556,35 @@ resolve_cert_path_for_read_strict(Path) ->
 resolve_cert_path_for_read(Path) ->
     emqx_schema:naive_env_interpolation(Path).
 
-filter([]) -> [];
-filter([{_, undefined} | T]) -> filter(T);
-filter([{_, ""} | T]) -> filter(T);
-filter([H | T]) -> [H | filter(T)].
+filter([], _) ->
+    [];
+filter([{_, undefined} | T], Versions) ->
+    filter(T, Versions);
+filter([{_, ""} | T], Versions) ->
+    filter(T, Versions);
+filter([{K, V} | T], Versions) ->
+    case tls_option_compatible_versions(K) of
+        all ->
+            [{K, V} | filter(T, Versions)];
+        CompatibleVersions ->
+            case CompatibleVersions -- (CompatibleVersions -- Versions) of
+                [] ->
+                    filter(T, Versions);
+                _ ->
+                    [{K, V} | filter(T, Versions)]
+            end
+    end.
+
+tls_option_compatible_versions(reuse_sessions) ->
+    [dtlsv1, 'dtlsv1.2', 'tlsv1', 'tlsv1.1', 'tlsv1.2'];
+tls_option_compatible_versions(secure_renegotiate) ->
+    [dtlsv1, 'dtlsv1.2', 'tlsv1', 'tlsv1.1', 'tlsv1.2'];
+tls_option_compatible_versions(user_lookup_fun) ->
+    [dtlsv1, 'dtlsv1.2', 'tlsv1', 'tlsv1.1', 'tlsv1.2'];
+tls_option_compatible_versions(client_renegotiation) ->
+    [dtlsv1, 'dtlsv1.2', 'tlsv1', 'tlsv1.1', 'tlsv1.2'];
+tls_option_compatible_versions(_) ->
+    all.
 
 -spec fuzzy_map_get(atom() | binary(), map(), any()) -> any().
 fuzzy_map_get(Key, Options, Default) ->
diff --git a/apps/emqx/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl
index 4e8435484..8a412af05 100644
--- a/apps/emqx/test/emqx_tls_lib_tests.erl
+++ b/apps/emqx/test/emqx_tls_lib_tests.erl
@@ -224,6 +224,71 @@ ssl_file_deterministic_names_test() ->
     ),
     _ = file:del_dir_r(filename:join(["/tmp", ?FUNCTION_NAME])).
 
+to_client_opts_test() ->
+    VersionsAll = [tlsv1, 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'],
+    Versions13Only = ['tlsv1.3'],
+    Options = #{
+        enable => true,
+        verify => "Verify",
+        server_name_indication => "SNI",
+        ciphers => "Ciphers",
+        depth => "depth",
+        password => "password",
+        versions => VersionsAll,
+        secure_renegotiate => "secure_renegotiate",
+        reuse_sessions => "reuse_sessions"
+    },
+    Expected1 = lists:usort(maps:keys(Options) -- [enable]),
+    ?assertEqual(
+        Expected1, lists:usort(proplists:get_keys(emqx_tls_lib:to_client_opts(tls, Options)))
+    ),
+    Expected2 =
+        lists:usort(
+            maps:keys(Options) --
+                [enable, reuse_sessions, secure_renegotiate]
+        ),
+    ?assertEqual(
+        Expected2,
+        lists:usort(
+            proplists:get_keys(
+                emqx_tls_lib:to_client_opts(tls, Options#{versions := Versions13Only})
+            )
+        )
+    ),
+    Expected3 = lists:usort(maps:keys(Options) -- [enable, depth, password]),
+    ?assertEqual(
+        Expected3,
+        lists:usort(
+            proplists:get_keys(
+                emqx_tls_lib:to_client_opts(tls, Options#{depth := undefined, password := ""})
+            )
+        )
+    ).
+
+to_server_opts_test() ->
+    VersionsAll = [tlsv1, 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'],
+    Versions13Only = ['tlsv1.3'],
+    Options = #{
+        verify => "Verify",
+        ciphers => "Ciphers",
+        versions => VersionsAll,
+        user_lookup_fun => "funfunfun",
+        client_renegotiation => "client_renegotiation"
+    },
+    Expected1 = lists:usort(maps:keys(Options)),
+    ?assertEqual(
+        Expected1, lists:usort(proplists:get_keys(emqx_tls_lib:to_server_opts(tls, Options)))
+    ),
+    Expected2 = lists:usort(maps:keys(Options) -- [user_lookup_fun, client_renegotiation]),
+    ?assertEqual(
+        Expected2,
+        lists:usort(
+            proplists:get_keys(
+                emqx_tls_lib:to_server_opts(tls, Options#{versions := Versions13Only})
+            )
+        )
+    ).
+
 bin(X) -> iolist_to_binary(X).
 
 test_key() ->

From 0e0d7c10a5f853e2af1af0f2230b6435b3bb6d31 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Thu, 8 Jun 2023 15:08:08 +0200
Subject: [PATCH 14/33] chore: add changelog

---
 changes/ce/fix-10983.en.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 changes/ce/fix-10983.en.md

diff --git a/changes/ce/fix-10983.en.md b/changes/ce/fix-10983.en.md
new file mode 100644
index 000000000..ac5b4834a
--- /dev/null
+++ b/changes/ce/fix-10983.en.md
@@ -0,0 +1,3 @@
+Fix issue when mqtt clients could not connect over TLS if the listener was configured to use TLS v1.3 only.
+
+The problem was that TLS connection was trying to use options incompatible with TLS v1.3.

From 0434e6c6c3af9cb06e55fd2c9940a3030d7fd5ad Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Thu, 8 Jun 2023 15:26:15 +0200
Subject: [PATCH 15/33] docs: update description of TLS options incompatible
 with v1.3

---
 rel/i18n/emqx_schema.hocon | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon
index 78dd2ecaa..f871cfc49 100644
--- a/rel/i18n/emqx_schema.hocon
+++ b/rel/i18n/emqx_schema.hocon
@@ -92,7 +92,8 @@ mqtt_max_topic_alias.label:
 """Max Topic Alias"""
 
 common_ssl_opts_schema_user_lookup_fun.desc:
-"""EMQX-internal callback that is used to lookup pre-shared key (PSK) identity."""
+"""EMQX-internal callback that is used to lookup pre-shared key (PSK) identity.</br>
+Has no effect when TLS version is configured (or negotiated) to 1.3"""
 
 common_ssl_opts_schema_user_lookup_fun.label:
 """SSL PSK user lookup fun"""
@@ -1240,7 +1241,8 @@ The SSL application already takes measures to counter-act such attempts,
 but client-initiated renegotiation can be strictly disabled by setting this option to false.
 The default value is true. Note that disabling renegotiation can result in
 long-lived connections becoming unusable due to limits on
-the number of messages the underlying cipher suite can encipher."""
+the number of messages the underlying cipher suite can encipher.</br>
+Has no effect when TLS version is configured (or negotiated) to 1.3"""
 
 server_ssl_opts_schema_client_renegotiation.label:
 """SSL client renegotiation"""
@@ -1326,7 +1328,8 @@ common_ssl_opts_schema_secure_renegotiate.desc:
 """SSL parameter renegotiation is a feature that allows a client and a server
 to renegotiate the parameters of the SSL connection on the fly.
 RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation,
-you drop support for the insecure renegotiation, prone to MitM attacks."""
+you drop support for the insecure renegotiation, prone to MitM attacks.</br>
+Has no effect when TLS version is configured (or negotiated) to 1.3"""
 
 common_ssl_opts_schema_secure_renegotiate.label:
 """SSL renegotiate"""
@@ -1361,7 +1364,8 @@ mqtt_max_packet_size.label:
 """Max Packet Size"""
 
 common_ssl_opts_schema_reuse_sessions.desc:
-"""Enable TLS session reuse."""
+"""Enable TLS session reuse.</br>
+Has no effect when TLS version is configured (or negotiated) to 1.3"""
 
 common_ssl_opts_schema_reuse_sessions.label:
 """TLS session reuse"""

From ed9e29e769a50164df9e22817200925476da5af5 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Thu, 8 Jun 2023 00:57:42 +0200
Subject: [PATCH 16/33] refactor: refacor query_mode detection code

This commit refactor the query_mode resource detection code according to
a suggestion from @zmstone. This commit should not contain any
functional change except for a change of the Kafka producer bridge
config.
---
 .../src/emqx_bridge_dynamo_connector.erl      |  3 --
 .../src/emqx_bridge_gcp_pubsub_connector.erl  |  5 +-
 .../src/emqx_bridge_kafka.erl                 | 35 ++++--------
 .../src/emqx_bridge_kafka_impl_consumer.erl   | 10 ++--
 .../src/emqx_bridge_kafka_impl_producer.erl   |  8 ++-
 .../src/emqx_bridge_opents_connector.erl      |  3 --
 .../src/emqx_bridge_pulsar_impl_producer.erl  |  9 ++--
 .../src/emqx_bridge_rabbitmq_connector.erl    |  6 ---
 .../src/emqx_bridge_rocketmq_connector.erl    |  3 --
 .../src/emqx_bridge_sqlserver_connector.erl   |  3 --
 .../src/emqx_bridge_tdengine_connector.erl    |  3 --
 apps/emqx_oracle/src/emqx_oracle.erl          |  3 --
 apps/emqx_resource/src/emqx_resource.erl      | 53 ++++++++++---------
 .../src/emqx_resource_manager.erl             | 30 +++++++----
 .../src/emqx_ee_connector.app.src             |  2 +-
 .../src/emqx_ee_connector_mongodb.erl         |  3 --
 16 files changed, 74 insertions(+), 105 deletions(-)

diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl
index 0cc3f993b..214d6b642 100644
--- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl
+++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl
@@ -17,7 +17,6 @@
 %% `emqx_resource' API
 -export([
     callback_mode/0,
-    is_buffer_supported/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -64,8 +63,6 @@ fields(config) ->
 
 callback_mode() -> always_sync.
 
-is_buffer_supported() -> false.
-
 on_start(
     InstanceId,
     #{
diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
index 65a0336ec..18cfca1d9 100644
--- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
+++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
@@ -22,8 +22,7 @@
     on_query_async/4,
     on_batch_query/3,
     on_batch_query_async/4,
-    on_get_status/2,
-    is_buffer_supported/0
+    on_get_status/2
 ]).
 -export([reply_delegator/3]).
 
@@ -56,8 +55,6 @@
 %% emqx_resource API
 %%-------------------------------------------------------------------------------------------------
 
-is_buffer_supported() -> false.
-
 callback_mode() -> async_if_possible.
 
 -spec on_start(resource_id(), config()) -> {ok, state()} | {error, term()}.
diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
index 9cabaf5e9..54cddb735 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
@@ -147,7 +147,7 @@ fields("config_producer") ->
 fields("config_consumer") ->
     fields(kafka_consumer);
 fields(kafka_producer) ->
-    fields("config") ++ fields(resource_opts) ++ fields(producer_opts);
+    fields("config") ++ fields(producer_opts);
 fields(kafka_consumer) ->
     fields("config") ++ fields(consumer_opts);
 fields("config") ->
@@ -295,28 +295,6 @@ fields(producer_kafka_opts) ->
                 required => false,
                 desc => ?DESC(producer_buffer)
             })},
-        {query_mode_sync_timeout,
-            mk(
-                emqx_schema:duration_ms(),
-                #{
-                    default => <<"5s">>,
-                    desc => ?DESC(query_mode_sync_timeout)
-                }
-            )}
-    ];
-fields(resource_opts) ->
-    [
-        {resource_opts,
-            mk(
-                ref(?MODULE, resource_opts_fields),
-                #{
-                    required => false,
-                    desc => ?DESC(resource_opts)
-                }
-            )}
-    ];
-fields(resource_opts_fields) ->
-    [
         {query_mode,
             mk(
                 enum([async, sync]),
@@ -324,6 +302,14 @@ fields(resource_opts_fields) ->
                     default => async,
                     desc => ?DESC(query_mode)
                 }
+            )},
+        {query_mode_sync_timeout,
+            mk(
+                emqx_schema:duration_ms(),
+                #{
+                    default => <<"5s">>,
+                    desc => ?DESC(query_mode_sync_timeout)
+                }
             )}
     ];
 fields(kafka_message) ->
@@ -440,8 +426,7 @@ struct_names() ->
         producer_opts,
         consumer_opts,
         consumer_kafka_opts,
-        consumer_topic_mapping,
-        resource_opts_fields
+        consumer_topic_mapping
     ].
 
 %% -------------------------------------------------------------------------------------------------
diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
index c0de23d94..5f59b4756 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
@@ -8,7 +8,7 @@
 %% `emqx_resource' API
 -export([
     callback_mode/0,
-    is_buffer_supported/0,
+    query_mode/1,
     on_start/2,
     on_stop/2,
     on_get_status/2
@@ -112,11 +112,9 @@
 callback_mode() ->
     async_if_possible.
 
-%% there are no queries to be made to this bridge, so we say that
-%% buffer is supported so we don't spawn unused resource buffer
-%% workers.
-is_buffer_supported() ->
-    true.
+%% consumer bridges don't need resource workers
+query_mode(_Config) ->
+    no_queries.
 
 -spec on_start(resource_id(), config()) -> {ok, state()}.
 on_start(ResourceId, Config) ->
diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
index 8be506efd..eee0cdd99 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
@@ -4,10 +4,11 @@
 -module(emqx_bridge_kafka_impl_producer).
 
 -include_lib("emqx_resource/include/emqx_resource.hrl").
+-include_lib("snabbkaffe/include/trace.hrl").
 
 %% callbacks of behaviour emqx_resource
 -export([
-    is_buffer_supported/0,
+    query_mode/1,
     callback_mode/0,
     on_start/2,
     on_stop/2,
@@ -32,7 +33,10 @@
 %% to hocon; keeping this as just `kafka' for backwards compatibility.
 -define(BRIDGE_TYPE, kafka).
 
-is_buffer_supported() -> true.
+query_mode(#{kafka := #{query_mode := sync}}) ->
+    simple_sync;
+query_mode(_) ->
+    simple_async.
 
 callback_mode() -> async_if_possible.
 
diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl
index 71184e872..887031e2e 100644
--- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl
+++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl
@@ -17,7 +17,6 @@
 %% `emqx_resource' API
 -export([
     callback_mode/0,
-    is_buffer_supported/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -49,8 +48,6 @@ fields(config) ->
 
 callback_mode() -> always_sync.
 
-is_buffer_supported() -> false.
-
 on_start(
     InstanceId,
     #{
diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
index b8157d4fc..1d93ffd0d 100644
--- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
+++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
@@ -11,7 +11,7 @@
 %% `emqx_resource' API
 -export([
     callback_mode/0,
-    is_buffer_supported/0,
+    query_mode/1,
     on_start/2,
     on_stop/2,
     on_get_status/2,
@@ -70,10 +70,9 @@
 
 callback_mode() -> async_if_possible.
 
-%% there are no queries to be made to this bridge, so we say that
-%% buffer is supported so we don't spawn unused resource buffer
-%% workers.
-is_buffer_supported() -> true.
+%% consumer bridges don't need resource workers
+query_mode(_Config) ->
+    no_queries.
 
 -spec on_start(resource_id(), config()) -> {ok, state()}.
 on_start(InstanceId, Config) ->
diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl
index 0dc73b5f6..251158029 100644
--- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl
+++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl
@@ -34,7 +34,6 @@
     %% Optional callbacks
     on_get_status/2,
     on_query/3,
-    is_buffer_supported/0,
     on_batch_query/3
 ]).
 
@@ -187,11 +186,6 @@ callback_mode() -> always_sync.
 
 %% emqx_resource callback
 
--spec is_buffer_supported() -> boolean().
-is_buffer_supported() ->
-    %% We want to make use of EMQX's buffer mechanism
-    false.
-
 %% emqx_resource callback called when the resource is started
 
 -spec on_start(resource_id(), term()) -> {ok, resource_state()} | {error, _}.
diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl
index a7d01960e..d9a9eae3b 100644
--- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl
+++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl
@@ -17,7 +17,6 @@
 %% `emqx_resource' API
 -export([
     callback_mode/0,
-    is_buffer_supported/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -86,8 +85,6 @@ servers() ->
 
 callback_mode() -> always_sync.
 
-is_buffer_supported() -> false.
-
 on_start(
     InstanceId,
     #{servers := BinServers, topic := Topic, sync_timeout := SyncTimeout} = Config
diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
index 52bd910db..d814e6205 100644
--- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
+++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
@@ -30,7 +30,6 @@
 %% callbacks for behaviour emqx_resource
 -export([
     callback_mode/0,
-    is_buffer_supported/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -169,8 +168,6 @@ server() ->
 
 callback_mode() -> always_sync.
 
-is_buffer_supported() -> false.
-
 on_start(
     ResourceId = PoolName,
     #{
diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
index 8fd41443c..97823e0d7 100644
--- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
+++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
@@ -17,7 +17,6 @@
 %% `emqx_resource' API
 -export([
     callback_mode/0,
-    is_buffer_supported/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -79,8 +78,6 @@ server() ->
 
 callback_mode() -> always_sync.
 
-is_buffer_supported() -> false.
-
 on_start(
     InstanceId,
     #{
diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl
index 0f543badd..ed023ef30 100644
--- a/apps/emqx_oracle/src/emqx_oracle.erl
+++ b/apps/emqx_oracle/src/emqx_oracle.erl
@@ -18,7 +18,6 @@
 %% callbacks for behaviour emqx_resource
 -export([
     callback_mode/0,
-    is_buffer_supported/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -68,8 +67,6 @@
 % be sync for now.
 callback_mode() -> always_sync.
 
-is_buffer_supported() -> false.
-
 -spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}.
 on_start(
     InstId,
diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl
index c700cfd86..2f43fd926 100644
--- a/apps/emqx_resource/src/emqx_resource.erl
+++ b/apps/emqx_resource/src/emqx_resource.erl
@@ -100,7 +100,8 @@
     call_health_check/3,
     %% stop the instance
     call_stop/3,
-    is_buffer_supported/1
+    %% get the query mode of the resource
+    query_mode/3
 ]).
 
 %% list all the instances, id only.
@@ -132,7 +133,7 @@
     on_query_async/4,
     on_batch_query_async/4,
     on_get_status/2,
-    is_buffer_supported/0
+    query_mode/1
 ]).
 
 %% when calling emqx_resource:start/1
@@ -173,7 +174,8 @@
     | {resource_status(), resource_state()}
     | {resource_status(), resource_state(), term()}.
 
--callback is_buffer_supported() -> boolean().
+-callback query_mode(Config :: term()) ->
+    simple_sync | simple_async | sync | async | no_queries.
 
 -spec list_types() -> [module()].
 list_types() ->
@@ -276,27 +278,26 @@ query(ResId, Request) ->
     Result :: term().
 query(ResId, Request, Opts) ->
     case emqx_resource_manager:lookup_cached(ResId) of
-        {ok, _Group, #{query_mode := QM, mod := Module} = Config} ->
-            IsBufferSupported = is_buffer_supported(Module),
-            case {IsBufferSupported, QM} of
-                {true, _} ->
-                    %% only Kafka producer so far
+        {ok, _Group, #{query_mode := QM}} ->
+            case QM of
+                simple_async ->
+                    %% TODO(5.1.1): pass Resource instead of ResId to simple APIs
+                    %% so the buffer worker does not need to lookup the cache again
                     Opts1 = Opts#{is_buffer_supported => true},
-                    do_query_built_in_buffer(QM, ResId, Request, Opts1);
-                {false, sync} ->
+                    emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts1);
+                simple_sync ->
+                    %% TODO(5.1.1): pass Resource instead of ResId to simple APIs
+                    %% so the buffer worker does not need to lookup the cache again
+                    emqx_resource_buffer_worker:simple_sync_query(ResId, Request);
+                sync ->
                     emqx_resource_buffer_worker:sync_query(ResId, Request, Opts);
-                {false, async} ->
+                async ->
                     emqx_resource_buffer_worker:async_query(ResId, Request, Opts)
             end;
         {error, not_found} ->
             ?RESOURCE_ERROR(not_found, "resource not found")
     end.
 
-do_query_built_in_buffer(async, ResId, Request, Opts1) ->
-    emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts1);
-do_query_built_in_buffer(sync, ResId, Request, _Opts1) ->
-    emqx_resource_buffer_worker:simple_sync_query(ResId, Request).
-
 -spec simple_sync_query(resource_id(), Request :: term()) -> Result :: term().
 simple_sync_query(ResId, Request) ->
     emqx_resource_buffer_worker:simple_sync_query(ResId, Request).
@@ -372,15 +373,6 @@ list_group_instances(Group) -> emqx_resource_manager:list_group(Group).
 get_callback_mode(Mod) ->
     Mod:callback_mode().
 
--spec is_buffer_supported(module()) -> boolean().
-is_buffer_supported(Module) ->
-    try
-        Module:is_buffer_supported()
-    catch
-        _:_ ->
-            false
-    end.
-
 -spec call_start(resource_id(), module(), resource_config()) ->
     {ok, resource_state()} | {error, Reason :: term()}.
 call_start(ResId, Mod, Config) ->
@@ -417,6 +409,17 @@ call_stop(ResId, Mod, ResourceState) ->
         Res
     end).
 
+-spec query_mode(module(), term(), creation_opts()) ->
+    simple_sync | simple_async | sync | async | no_queries.
+
+query_mode(Mod, Config, Opts) ->
+    case erlang:function_exported(Mod, query_mode, 1) of
+        true ->
+            Mod:query_mode(Config);
+        false ->
+            maps:get(query_mode, Opts, sync)
+    end.
+
 -spec check_config(resource_type(), raw_resource_config()) ->
     {ok, resource_config()} | {error, term()}.
 check_config(ResourceType, Conf) ->
diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index 02dda8021..be58bf0e0 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -144,12 +144,18 @@ create(ResId, Group, ResourceType, Config, Opts) ->
         ],
         [matched]
     ),
-    case emqx_resource:is_buffer_supported(ResourceType) of
-        true ->
-            %% the resource it self supports
-            %% buffer, so there is no need for resource workers
+    QueryMode = emqx_resource:query_mode(ResourceType, Config, Opts),
+    case QueryMode of
+        %% the resource has built-in buffer, so there is no need for resource workers
+        simple_sync ->
             ok;
-        false ->
+        simple_async ->
+            ok;
+        %% The resource is a consumer resource, so there is no need for resource workers
+        no_queries ->
+            ok;
+        _ ->
+            %% start resource workers as the query type requires them
             ok = emqx_resource_buffer_worker_sup:start_workers(ResId, Opts),
             case maps:get(start_after_created, Opts, ?START_AFTER_CREATED) of
                 true ->
@@ -288,16 +294,20 @@ health_check(ResId) ->
 
 %% @doc Function called from the supervisor to actually start the server
 start_link(ResId, Group, ResourceType, Config, Opts) ->
+    QueryMode =
+        case erlang:function_exported(ResourceType, query_mode, 1) of
+            true ->
+                ResourceType:query_mode(Config);
+            false ->
+                maps:get(query_mode, Opts, sync)
+        end,
+
     Data = #data{
         id = ResId,
         group = Group,
         mod = ResourceType,
         callback_mode = emqx_resource:get_callback_mode(ResourceType),
-        %% query_mode = dynamic | sync | async
-        %% TODO:
-        %%  dynamic mode is async mode when things are going well, but becomes sync mode
-        %%  if the resource worker is overloaded
-        query_mode = maps:get(query_mode, Opts, sync),
+        query_mode = QueryMode,
         config = Config,
         opts = Opts,
         state = undefined,
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index 3ed460492..702b9ff09 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -1,6 +1,6 @@
 {application, emqx_ee_connector, [
     {description, "EMQX Enterprise connectors"},
-    {vsn, "0.1.13"},
+    {vsn, "0.1.14"},
     {registered, []},
     {applications, [
         kernel,
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl
index 59f763904..0c5a839e9 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl
@@ -15,7 +15,6 @@
 %% `emqx_resource' API
 -export([
     callback_mode/0,
-    is_buffer_supported/0,
     on_start/2,
     on_stop/2,
     on_query/3,
@@ -28,8 +27,6 @@
 
 callback_mode() -> emqx_connector_mongo:callback_mode().
 
-is_buffer_supported() -> false.
-
 on_start(InstanceId, Config) ->
     case emqx_connector_mongo:on_start(InstanceId, Config) of
         {ok, ConnectorState} ->

From 6f2271e9f0384d88f148d64051b28434f29b3b36 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Thu, 8 Jun 2023 15:43:41 +0200
Subject: [PATCH 17/33] test: add Kafka producer bridge test case for
 query_mode parameter

---
 .../src/emqx_bridge_kafka_impl_producer.erl   |  2 +
 .../emqx_bridge_kafka_impl_producer_SUITE.erl | 75 +++++++++++++++----
 rel/i18n/emqx_bridge_kafka.hocon              | 12 ---
 3 files changed, 62 insertions(+), 27 deletions(-)

diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
index eee0cdd99..b5799a91a 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
@@ -204,6 +204,7 @@ on_query(
         query_mode_sync_timeout := SyncTimeout
     }
 ) ->
+    ?tp(emqx_bridge_kafka_impl_producer_sync_query, #{}),
     KafkaMessage = render_message(Template, Message),
     try
         {_Partition, _Offset} = wolff:send_sync(Producers, [KafkaMessage], SyncTimeout),
@@ -227,6 +228,7 @@ on_query_async(
     AsyncReplyFn,
     #{message_template := Template, producers := Producers}
 ) ->
+    ?tp(emqx_bridge_kafka_impl_producer_async_query, #{}),
     KafkaMessage = render_message(Template, Message),
     %% * Must be a batch because wolff:send and wolff:send_sync are batch APIs
     %% * Must be a single element batch because wolff books calls, but not batch sizes
diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl
index 378dda543..e52f5b07b 100644
--- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl
+++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl
@@ -120,6 +120,34 @@ set_special_configs(emqx_dashboard) ->
     ok;
 set_special_configs(_) ->
     ok.
+
+%%------------------------------------------------------------------------------
+%% Test case for the query_mode parameter
+%%------------------------------------------------------------------------------
+
+t_query_mode(CtConfig) ->
+    %% We need this because on_query_async is in a different group
+    CtConfig1 = [{query_api, none} | CtConfig],
+    ?check_trace(
+        begin
+            publish_with_config_template_parameters(CtConfig1, #{"query_mode" => "sync"})
+        end,
+        fun(RunStageResult, Trace) ->
+            %% We should have a sync Snabbkaffe trace
+            ?assertMatch([_], ?of_kind(emqx_bridge_kafka_impl_producer_sync_query, Trace))
+        end
+    ),
+    ?check_trace(
+        begin
+            publish_with_config_template_parameters(CtConfig1, #{"query_mode" => "async"})
+        end,
+        fun(RunStageResult, Trace) ->
+            %% We should have a sync Snabbkaffe trace
+            ?assertMatch([_], ?of_kind(emqx_bridge_kafka_impl_producer_async_query, Trace))
+        end
+    ),
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Test cases for all combinations of SSL, no SSL and authentication types
 %%------------------------------------------------------------------------------
@@ -473,6 +501,16 @@ do_send(Ref, Config, ResourceId, Msg, State) when is_list(Config) ->
             ok
     end.
 
+publish_with_config_template_parameters(CtConfig, ConfigTemplateParameters) ->
+    publish_helper(
+        CtConfig,
+        #{
+            auth_settings => "none",
+            ssl_settings => #{}
+        },
+        ConfigTemplateParameters
+    ).
+
 publish_with_and_without_ssl(CtConfig, AuthSettings) ->
     publish_with_and_without_ssl(CtConfig, AuthSettings, #{}).
 
@@ -537,21 +575,25 @@ publish_helper(
     {ok, _} = emqx_bridge:create(
         <<?BRIDGE_TYPE>>, list_to_binary(Name), Conf
     ),
-    Time = erlang:unique_integer(),
-    BinTime = integer_to_binary(Time),
     Partition = 0,
-    Msg = #{
-        clientid => BinTime,
-        payload => <<"payload">>,
-        timestamp => Time
-    },
-    {ok, Offset0} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, Partition),
-    ct:pal("base offset before testing ~p", [Offset0]),
-    {ok, _Group, #{state := State}} = emqx_resource:get_instance(InstId),
-    ok = send(CtConfig, InstId, Msg, State),
-    {ok, {_, [KafkaMsg0]}} = brod:fetch(kafka_hosts(), KafkaTopic, Partition, Offset0),
-    ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg0),
-
+    case proplists:get_value(query_api, CtConfig) of
+        none ->
+            ok;
+        _ ->
+            Time = erlang:unique_integer(),
+            BinTime = integer_to_binary(Time),
+            Msg = #{
+                clientid => BinTime,
+                payload => <<"payload">>,
+                timestamp => Time
+            },
+            {ok, Offset0} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, Partition),
+            ct:pal("base offset before testing ~p", [Offset0]),
+            {ok, _Group, #{state := State}} = emqx_resource:get_instance(InstId),
+            ok = send(CtConfig, InstId, Msg, State),
+            {ok, {_, [KafkaMsg0]}} = brod:fetch(kafka_hosts(), KafkaTopic, Partition, Offset0),
+            ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg0)
+    end,
     %% test that it forwards from local mqtt topic as well
     {ok, Offset1} = resolve_kafka_offset(kafka_hosts(), KafkaTopic, Partition),
     ct:pal("base offset before testing (2) ~p", [Offset1]),
@@ -596,13 +638,15 @@ hocon_config(Args) ->
     AuthConfRendered = bbmustache:render(AuthTemplate, AuthConf),
     SSLConf = maps:get("ssl", Args, #{}),
     SSLTemplate = iolist_to_binary(hocon_config_template_ssl(SSLConf)),
+    QueryMode = maps:get("query_mode", Args, <<"async">>),
     SSLConfRendered = bbmustache:render(SSLTemplate, SSLConf),
     Hocon = bbmustache:render(
         iolist_to_binary(hocon_config_template()),
         Args#{
             "authentication" => AuthConfRendered,
             "bridge_name" => Name,
-            "ssl" => SSLConfRendered
+            "ssl" => SSLConfRendered,
+            "query_mode" => QueryMode
         }
     ),
     Hocon.
@@ -630,6 +674,7 @@ bridges.kafka.{{ bridge_name }} {
     }
     partition_strategy = {{ partition_strategy }}
     topic = \"{{ kafka_topic }}\"
+    query_mode = {{ query_mode }}
   }
   metadata_request_timeout = 5s
   min_metadata_refresh_interval = 3s
diff --git a/rel/i18n/emqx_bridge_kafka.hocon b/rel/i18n/emqx_bridge_kafka.hocon
index 740d94984..86acd81fe 100644
--- a/rel/i18n/emqx_bridge_kafka.hocon
+++ b/rel/i18n/emqx_bridge_kafka.hocon
@@ -364,18 +364,6 @@ query_mode.desc:
 query_mode.label:
 """Query mode"""
 
-resource_opts.desc:
-"""Resource options."""
-
-resource_opts.label:
-"""Resource Options"""
-
-resource_opts_fields.desc:
-"""Resource options."""
-
-resource_opts_fields.label:
-"""Resource Options"""
-
 query_mode_sync_timeout.desc:
 """This parameter defines the timeout limit for synchronous queries. It applies only when the bridge query mode is configured to 'sync'."""
 

From d524f8c805cfc7b64031906d7d28c85790593eb1 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Thu, 8 Jun 2023 15:58:19 +0200
Subject: [PATCH 18/33] refactor: rename config parameter

---
 apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl            | 4 ++--
 .../src/emqx_bridge_kafka_impl_producer.erl                 | 6 +++---
 rel/i18n/emqx_bridge_kafka.hocon                            | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
index 54cddb735..5f0dcee72 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
@@ -303,12 +303,12 @@ fields(producer_kafka_opts) ->
                     desc => ?DESC(query_mode)
                 }
             )},
-        {query_mode_sync_timeout,
+        {sync_query_timeout,
             mk(
                 emqx_schema:duration_ms(),
                 #{
                     default => <<"5s">>,
-                    desc => ?DESC(query_mode_sync_timeout)
+                    desc => ?DESC(sync_query_timeout)
                 }
             )}
     ];
diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
index b5799a91a..563e7dd46 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
@@ -50,7 +50,7 @@ on_start(InstId, Config) ->
         kafka := KafkaConfig = #{
             message := MessageTemplate,
             topic := KafkaTopic,
-            query_mode_sync_timeout := QueryModeSyncTimeout
+            sync_query_timeout := SyncQueryTimeout
         },
         metadata_request_timeout := MetaReqTimeout,
         min_metadata_refresh_interval := MinMetaRefreshInterval,
@@ -108,7 +108,7 @@ on_start(InstId, Config) ->
                 kafka_topic => KafkaTopic,
                 producers => Producers,
                 resource_id => ResourceId,
-                query_mode_sync_timeout => QueryModeSyncTimeout
+                sync_query_timeout => SyncQueryTimeout
             }};
         {error, Reason2} ->
             ?SLOG(error, #{
@@ -201,7 +201,7 @@ on_query(
     #{
         message_template := Template,
         producers := Producers,
-        query_mode_sync_timeout := SyncTimeout
+        sync_query_timeout := SyncTimeout
     }
 ) ->
     ?tp(emqx_bridge_kafka_impl_producer_sync_query, #{}),
diff --git a/rel/i18n/emqx_bridge_kafka.hocon b/rel/i18n/emqx_bridge_kafka.hocon
index 86acd81fe..fcb9b0074 100644
--- a/rel/i18n/emqx_bridge_kafka.hocon
+++ b/rel/i18n/emqx_bridge_kafka.hocon
@@ -166,7 +166,7 @@ consumer_offset_reset_policy.label:
 
 partition_count_refresh_interval.desc:
 """The time interval for Kafka producer to discover increased number of partitions.
-After the number of partitions is increased in Kafka, EMQX will start taking the 
+After the number of partitions is increased in Kafka, EMQX will start taking the
 discovered partitions into account when dispatching messages per <code>partition_strategy</code>."""
 
 partition_count_refresh_interval.label:
@@ -364,10 +364,10 @@ query_mode.desc:
 query_mode.label:
 """Query mode"""
 
-query_mode_sync_timeout.desc:
+sync_query_timeout.desc:
 """This parameter defines the timeout limit for synchronous queries. It applies only when the bridge query mode is configured to 'sync'."""
 
-query_mode_sync_timeout.label:
+sync_query_timeout.label:
 """Synchronous Query Timeout"""
 
 }

From b60dbbc792dc85a961d2f64f156d997445a41222 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Thu, 8 Jun 2023 16:04:50 +0200
Subject: [PATCH 19/33] docs: add change log entry for Kafka query mode
 parameter

---
 changes/ee/feat-10970.en.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/ee/feat-10970.en.md

diff --git a/changes/ee/feat-10970.en.md b/changes/ee/feat-10970.en.md
new file mode 100644
index 000000000..d716f6eef
--- /dev/null
+++ b/changes/ee/feat-10970.en.md
@@ -0,0 +1 @@
+A query_mode parameter has been added to the Kafka producer bridge. This parameter allows you to specify if the bridge should use the asynchronous or synchronous mode when sending data to Kafka. The default is asynchronous mode.

From cb3a5fdbd4423b97871757d8e16df0122f392954 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Thu, 8 Jun 2023 16:16:23 +0200
Subject: [PATCH 20/33] style: only callback modules should do dynamic calls

---
 apps/emqx_resource/src/emqx_resource_manager.erl | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl
index be58bf0e0..c1adb8ecd 100644
--- a/apps/emqx_resource/src/emqx_resource_manager.erl
+++ b/apps/emqx_resource/src/emqx_resource_manager.erl
@@ -294,14 +294,11 @@ health_check(ResId) ->
 
 %% @doc Function called from the supervisor to actually start the server
 start_link(ResId, Group, ResourceType, Config, Opts) ->
-    QueryMode =
-        case erlang:function_exported(ResourceType, query_mode, 1) of
-            true ->
-                ResourceType:query_mode(Config);
-            false ->
-                maps:get(query_mode, Opts, sync)
-        end,
-
+    QueryMode = emqx_resource:query_mode(
+        ResourceType,
+        Config,
+        Opts
+    ),
     Data = #data{
         id = ResId,
         group = Group,

From 1c7834e0565bd224b489669df682b593ede98590 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Thu, 8 Jun 2023 16:47:02 +0200
Subject: [PATCH 21/33] fix: fixes due to comments from @zmstone

---
 apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl               | 2 +-
 .../src/emqx_bridge_pulsar_impl_producer.erl                   | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
index 5f0dcee72..31307fd16 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl
@@ -305,7 +305,7 @@ fields(producer_kafka_opts) ->
             )},
         {sync_query_timeout,
             mk(
-                emqx_schema:duration_ms(),
+                emqx_schema:timeout_duration_ms(),
                 #{
                     default => <<"5s">>,
                     desc => ?DESC(sync_query_timeout)
diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
index 1d93ffd0d..0dcf1bf66 100644
--- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
+++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
@@ -70,9 +70,8 @@
 
 callback_mode() -> async_if_possible.
 
-%% consumer bridges don't need resource workers
 query_mode(_Config) ->
-    no_queries.
+    simple_async.
 
 -spec on_start(resource_id(), config()) -> {ok, state()}.
 on_start(InstanceId, Config) ->

From 38e767b8085f6cef4849929847cfbdccd997506e Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 9 Jun 2023 09:14:55 +0200
Subject: [PATCH 22/33] fix(schema): map new duration types to dashboard's type
 name

---
 apps/emqx_conf/src/emqx_conf.erl | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl
index cc56d5e46..d2e45581d 100644
--- a/apps/emqx_conf/src/emqx_conf.erl
+++ b/apps/emqx_conf/src/emqx_conf.erl
@@ -320,10 +320,16 @@ typename_to_spec("atom()", _Mod) ->
     #{type => string};
 typename_to_spec("duration()", _Mod) ->
     #{type => duration};
+typename_to_spec("timeout_duration()", _Mod) ->
+    #{type => duration};
 typename_to_spec("duration_s()", _Mod) ->
     #{type => duration};
+typename_to_spec("timeout_duration_s()", _Mod) ->
+    #{type => duration};
 typename_to_spec("duration_ms()", _Mod) ->
     #{type => duration};
+typename_to_spec("timeout_duration_ms()", _Mod) ->
+    #{type => duration};
 typename_to_spec("percent()", _Mod) ->
     #{type => percent};
 typename_to_spec("file()", _Mod) ->

From 2671e8ecf9f35f7ab0ff774ca6e482996762ce3a Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 9 Jun 2023 11:00:05 +0200
Subject: [PATCH 23/33] fix: dialyzer type problem

---
 apps/emqx_resource/include/emqx_resource.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl
index 562e18d52..8e17ac15c 100644
--- a/apps/emqx_resource/include/emqx_resource.hrl
+++ b/apps/emqx_resource/include/emqx_resource.hrl
@@ -22,7 +22,7 @@
 -type resource_state() :: term().
 -type resource_status() :: connected | disconnected | connecting | stopped.
 -type callback_mode() :: always_sync | async_if_possible.
--type query_mode() :: async | sync | dynamic.
+-type query_mode() :: async | sync | simple_async | simple_sync | dynamic.
 -type result() :: term().
 -type reply_fun() :: {fun((result(), Args :: term()) -> any()), Args :: term()} | undefined.
 -type query_opts() :: #{

From d6c1ee183f2e685c4ac83e8d755d40e2f24e0f35 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Wed, 7 Jun 2023 21:19:52 +0300
Subject: [PATCH 24/33] refactor(pluglib): move `emqx_placeholder` to utils app

Also make user that existing code calls it directly.
---
 apps/emqx_authn/src/emqx_authn_utils.erl      | 12 ++--
 apps/emqx_authz/src/emqx_authz_utils.erl      | 12 ++--
 .../emqx_authz/test/emqx_authz_rule_SUITE.erl |  6 +-
 .../src/emqx_bridge_cassandra_connector.erl   |  8 +--
 .../src/emqx_bridge_clickhouse_connector.erl  | 10 +--
 .../src/emqx_bridge_dynamo_connector.erl      |  2 +-
 .../emqx_bridge_dynamo_connector_client.erl   |  2 +-
 .../src/emqx_bridge_gcp_pubsub_connector.erl  |  6 +-
 .../src/emqx_bridge_influxdb_connector.erl    | 20 +++---
 .../src/emqx_bridge_iotdb_impl.erl            | 16 ++---
 .../src/emqx_bridge_kafka_impl_consumer.erl   |  8 +--
 .../src/emqx_bridge_kafka_impl_producer.erl   |  4 +-
 .../src/emqx_bridge_mqtt_msg.erl              |  4 +-
 .../src/emqx_bridge_pulsar_impl_producer.erl  |  8 +--
 .../src/emqx_bridge_rabbitmq_connector.erl    |  4 +-
 .../src/emqx_bridge_rocketmq_connector.erl    | 10 +--
 .../src/emqx_bridge_sqlserver_connector.erl   |  6 +-
 .../src/emqx_bridge_tdengine_connector.erl    | 16 ++---
 .../test/emqx_bridge_tdengine_SUITE.erl       |  4 +-
 .../src/emqx_connector_http.erl               | 24 +++----
 .../src/emqx_connector_mysql.erl              |  6 +-
 .../src/emqx_connector_pgsql.erl              |  6 +-
 .../src/emqx_mqttsn_channel.erl               |  4 +-
 .../src/emqx_stomp_channel.erl                |  4 +-
 apps/emqx_oracle/src/emqx_oracle.erl          |  6 +-
 .../src/emqx_plugin_libs_rule.erl             | 69 +------------------
 .../src/emqx_rule_actions.erl                 | 14 ++--
 .../src/emqx_placeholder.erl                  | 43 +++++++++---
 .../test/emqx_placeholder_SUITE.erl           |  0
 .../src/emqx_ee_connector_hstreamdb.erl       |  8 +--
 .../src/emqx_ee_connector_mongodb.erl         |  8 +--
 .../src/emqx_ee_connector_redis.erl           |  4 +-
 32 files changed, 151 insertions(+), 203 deletions(-)
 rename apps/{emqx_plugin_libs => emqx_utils}/src/emqx_placeholder.erl (92%)
 rename apps/{emqx_plugin_libs => emqx_utils}/test/emqx_placeholder_SUITE.erl (100%)

diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl
index 12520251e..8e168bb5d 100644
--- a/apps/emqx_authn/src/emqx_authn_utils.erl
+++ b/apps/emqx_authn/src/emqx_authn_utils.erl
@@ -225,21 +225,19 @@ without_password(Credential, [Name | Rest]) ->
             without_password(Credential, Rest)
     end.
 
-urlencode_var({var, _} = Var, Value) ->
-    emqx_http_lib:uri_encode(handle_var(Var, Value));
 urlencode_var(Var, Value) ->
-    handle_var(Var, Value).
+    emqx_http_lib:uri_encode(handle_var(Var, Value)).
 
-handle_var({var, _Name}, undefined) ->
+handle_var(_Name, undefined) ->
     <<>>;
-handle_var({var, <<"peerhost">>}, PeerHost) ->
+handle_var([<<"peerhost">>], PeerHost) ->
     emqx_placeholder:bin(inet:ntoa(PeerHost));
 handle_var(_, Value) ->
     emqx_placeholder:bin(Value).
 
-handle_sql_var({var, _Name}, undefined) ->
+handle_sql_var(_Name, undefined) ->
     <<>>;
-handle_sql_var({var, <<"peerhost">>}, PeerHost) ->
+handle_sql_var([<<"peerhost">>], PeerHost) ->
     emqx_placeholder:bin(inet:ntoa(PeerHost));
 handle_sql_var(_, Value) ->
     emqx_placeholder:sql_data(Value).
diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl
index c01505680..ec112070e 100644
--- a/apps/emqx_authz/src/emqx_authz_utils.erl
+++ b/apps/emqx_authz/src/emqx_authz_utils.erl
@@ -188,21 +188,19 @@ convert_client_var({dn, DN}) -> {cert_subject, DN};
 convert_client_var({protocol, Proto}) -> {proto_name, Proto};
 convert_client_var(Other) -> Other.
 
-urlencode_var({var, _} = Var, Value) ->
-    emqx_http_lib:uri_encode(handle_var(Var, Value));
 urlencode_var(Var, Value) ->
-    handle_var(Var, Value).
+    emqx_http_lib:uri_encode(handle_var(Var, Value)).
 
-handle_var({var, _Name}, undefined) ->
+handle_var(_Name, undefined) ->
     <<>>;
-handle_var({var, <<"peerhost">>}, IpAddr) ->
+handle_var([<<"peerhost">>], IpAddr) ->
     inet_parse:ntoa(IpAddr);
 handle_var(_Name, Value) ->
     emqx_placeholder:bin(Value).
 
-handle_sql_var({var, _Name}, undefined) ->
+handle_sql_var(_Name, undefined) ->
     <<>>;
-handle_sql_var({var, <<"peerhost">>}, IpAddr) ->
+handle_sql_var([<<"peerhost">>], IpAddr) ->
     inet_parse:ntoa(IpAddr);
 handle_sql_var(_Name, Value) ->
     emqx_placeholder:sql_data(Value).
diff --git a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl
index 76e5677ce..fbfb84785 100644
--- a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl
@@ -81,7 +81,7 @@ t_compile(_) ->
                 {{127, 0, 0, 1}, {127, 0, 0, 1}, 32},
                 {{192, 168, 1, 0}, {192, 168, 1, 255}, 24}
             ]},
-            subscribe, [{pattern, [{var, {var, <<"clientid">>}}]}]},
+            subscribe, [{pattern, [{var, [<<"clientid">>]}]}]},
         emqx_authz_rule:compile(?SOURCE3)
     ),
 
@@ -99,14 +99,14 @@ t_compile(_) ->
                 {clientid, {re_pattern, _, _, _, _}}
             ]},
             publish, [
-                {pattern, [{var, {var, <<"username">>}}]}, {pattern, [{var, {var, <<"clientid">>}}]}
+                {pattern, [{var, [<<"username">>]}]}, {pattern, [{var, [<<"clientid">>]}]}
             ]},
         emqx_authz_rule:compile(?SOURCE5)
     ),
 
     ?assertEqual(
         {allow, {username, {eq, <<"test">>}}, publish, [
-            {pattern, [{str, <<"t/foo">>}, {var, {var, <<"username">>}}, {str, <<"boo">>}]}
+            {pattern, [{str, <<"t/foo">>}, {var, [<<"username">>]}, {str, <<"boo">>}]}
         ]},
         emqx_authz_rule:compile(?SOURCE6)
     ),
diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl
index 285825714..ad41329d2 100644
--- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl
+++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl
@@ -273,10 +273,10 @@ proc_cql_params(
     %% assert
     _PreparedKey = maps:get(PreparedKey0, Prepares),
     Tokens = maps:get(PreparedKey0, ParamsTokens),
-    {PreparedKey0, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))};
+    {PreparedKey0, assign_type_for_params(emqx_placeholder:proc_sql(Tokens, Params))};
 proc_cql_params(query, SQL, Params, _State) ->
-    {SQL1, Tokens} = emqx_plugin_libs_rule:preproc_sql(SQL, '?'),
-    {SQL1, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}.
+    {SQL1, Tokens} = emqx_placeholder:preproc_sql(SQL, '?'),
+    {SQL1, assign_type_for_params(emqx_placeholder:proc_sql(Tokens, Params))}.
 
 exec_cql_query(InstId, PoolName, Type, Async, PreparedKey, Data) when
     Type == query; Type == prepared_query
@@ -403,7 +403,7 @@ parse_prepare_cql(_) ->
     #{prepare_cql => #{}, params_tokens => #{}}.
 
 parse_prepare_cql([{Key, H} | T], Prepares, Tokens) ->
-    {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H, '?'),
+    {PrepareSQL, ParamsTokens} = emqx_placeholder:preproc_sql(H, '?'),
     parse_prepare_cql(
         T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens}
     );
diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
index 211e41174..002460c95 100644
--- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
+++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
@@ -193,7 +193,7 @@ prepare_sql_templates(#{
     batch_value_separator := Separator
 }) ->
     InsertTemplate =
-        emqx_plugin_libs_rule:preproc_tmpl(Template),
+        emqx_placeholder:preproc_tmpl(Template),
     BulkExtendInsertTemplate =
         prepare_sql_bulk_extend_template(Template, Separator),
     #{
@@ -210,7 +210,7 @@ prepare_sql_bulk_extend_template(Template, Separator) ->
     %% Add separator before ValuesTemplate so that one can append it
     %% to an insert template
     ExtendParamTemplate = iolist_to_binary([Separator, ValuesTemplate]),
-    emqx_plugin_libs_rule:preproc_tmpl(ExtendParamTemplate).
+    emqx_placeholder:preproc_tmpl(ExtendParamTemplate).
 
 %% This function is similar to emqx_plugin_libs_rule:split_insert_sql/1 but can
 %% also handle Clickhouse's SQL extension for INSERT statments that allows the
@@ -363,7 +363,7 @@ on_query(
     transform_and_log_clickhouse_result(ClickhouseResult, ResourceID, SQL).
 
 get_sql(send_message, #{send_message_template := PreparedSQL}, Data) ->
-    emqx_plugin_libs_rule:proc_tmpl(PreparedSQL, Data);
+    emqx_placeholder:proc_tmpl(PreparedSQL, Data);
 get_sql(_, _, SQL) ->
     SQL.
 
@@ -421,10 +421,10 @@ objects_to_sql(
     }
 ) ->
     %% Prepare INSERT-statement and the first row after VALUES
-    InsertStatementHead = emqx_plugin_libs_rule:proc_tmpl(InsertTemplate, FirstObject),
+    InsertStatementHead = emqx_placeholder:proc_tmpl(InsertTemplate, FirstObject),
     FormatObjectDataFunction =
         fun(Object) ->
-            emqx_plugin_libs_rule:proc_tmpl(BulkExtendInsertTemplate, Object)
+            emqx_placeholder:proc_tmpl(BulkExtendInsertTemplate, Object)
         end,
     InsertStatementTail = lists:map(FormatObjectDataFunction, RemainingObjects),
     CompleteStatement = erlang:iolist_to_binary([InsertStatementHead, InsertStatementTail]),
diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl
index 0cc3f993b..b0e145e63 100644
--- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl
+++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl
@@ -200,7 +200,7 @@ parse_template(Config) ->
     parse_template(maps:to_list(Templates), #{}).
 
 parse_template([{Key, H} | T], Templates) ->
-    ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(H),
+    ParamsTks = emqx_placeholder:preproc_tmpl(H),
     parse_template(
         T,
         Templates#{Key => ParamsTks}
diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl
index faaef9df4..1b379298f 100644
--- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl
+++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl
@@ -144,7 +144,7 @@ apply_template({Key, Msg} = Req, Templates) ->
         undefined ->
             Req;
         Template ->
-            {Key, emqx_plugin_libs_rule:proc_tmpl(Template, Msg)}
+            {Key, emqx_placeholder:proc_tmpl(Template, Msg)}
     end;
 %% now there is no batch delete, so
 %% 1. we can simply replace the `send_message` to `put`
diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
index 65a0336ec..58a3f6612 100644
--- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
+++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl
@@ -40,7 +40,7 @@
     connect_timeout := timer:time(),
     jwt_config := emqx_connector_jwt:jwt_config(),
     max_retries := non_neg_integer(),
-    payload_template := emqx_plugin_libs_rule:tmpl_token(),
+    payload_template := emqx_placeholder:tmpl_token(),
     pool_name := binary(),
     project_id := binary(),
     pubsub_topic := binary(),
@@ -104,7 +104,7 @@ on_start(
         connect_timeout => ConnectTimeout,
         jwt_config => JWTConfig,
         max_retries => MaxRetries,
-        payload_template => emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate),
+        payload_template => emqx_placeholder:preproc_tmpl(PayloadTemplate),
         pool_name => ResourceId,
         project_id => ProjectId,
         pubsub_topic => PubSubTopic,
@@ -294,7 +294,7 @@ encode_payload(_State = #{payload_template := PayloadTemplate}, Selected) ->
     Interpolated =
         case PayloadTemplate of
             [] -> emqx_utils_json:encode(Selected);
-            _ -> emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Selected)
+            _ -> emqx_placeholder:proc_tmpl(PayloadTemplate, Selected)
         end,
     #{data => base64:encode(Interpolated)}.
 
diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
index 05e7c11b2..71ab85c58 100644
--- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
+++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
@@ -436,7 +436,7 @@ to_config([Item0 | Rest], Acc, Precision) ->
     Ts0 = maps:get(timestamp, Item0, undefined),
     {Ts, FromPrecision, ToPrecision} = preproc_tmpl_timestamp(Ts0, Precision),
     Item = #{
-        measurement => emqx_plugin_libs_rule:preproc_tmpl(maps:get(measurement, Item0)),
+        measurement => emqx_placeholder:preproc_tmpl(maps:get(measurement, Item0)),
         timestamp => Ts,
         precision => {FromPrecision, ToPrecision},
         tags => to_kv_config(maps:get(tags, Item0)),
@@ -458,18 +458,18 @@ preproc_tmpl_timestamp(Ts, Precision) when is_integer(Ts) ->
 preproc_tmpl_timestamp(Ts, Precision) when is_list(Ts) ->
     preproc_tmpl_timestamp(iolist_to_binary(Ts), Precision);
 preproc_tmpl_timestamp(<<?DEFAULT_TIMESTAMP_TMPL>> = Ts, Precision) ->
-    {emqx_plugin_libs_rule:preproc_tmpl(Ts), ms, Precision};
+    {emqx_placeholder:preproc_tmpl(Ts), ms, Precision};
 preproc_tmpl_timestamp(Ts, Precision) when is_binary(Ts) ->
     %% a placehold is in use. e.g. ${payload.my_timestamp}
     %% we can only hope it the value will be of the same precision in the configs
-    {emqx_plugin_libs_rule:preproc_tmpl(Ts), Precision, Precision}.
+    {emqx_placeholder:preproc_tmpl(Ts), Precision, Precision}.
 
 to_kv_config(KVfields) ->
     maps:fold(fun to_maps_config/3, #{}, proplists:to_map(KVfields)).
 
 to_maps_config(K, V, Res) ->
-    NK = emqx_plugin_libs_rule:preproc_tmpl(bin(K)),
-    NV = emqx_plugin_libs_rule:preproc_tmpl(bin(V)),
+    NK = emqx_placeholder:preproc_tmpl(bin(K)),
+    NV = emqx_placeholder:preproc_tmpl(bin(V)),
     Res#{NK => NV}.
 
 %% -------------------------------------------------------------------------------------------------
@@ -505,7 +505,7 @@ parse_batch_data(InstId, BatchData, SyntaxLines) ->
         fields := [{binary(), binary()}],
         measurement := binary(),
         tags := [{binary(), binary()}],
-        timestamp := emqx_plugin_libs_rule:tmpl_token() | integer(),
+        timestamp := emqx_placeholder:tmpl_token() | integer(),
         precision := {From :: ts_precision(), To :: ts_precision()}
     }
 ]) -> {ok, [map()]} | {error, term()}.
@@ -526,7 +526,7 @@ lines_to_points(Data, [#{timestamp := Ts} = Item | Rest], ResultPointsAcc, Error
     is_list(Ts)
 ->
     TransOptions = #{return => rawlist, var_trans => fun data_filter/1},
-    case parse_timestamp(emqx_plugin_libs_rule:proc_tmpl(Ts, Data, TransOptions)) of
+    case parse_timestamp(emqx_placeholder:proc_tmpl(Ts, Data, TransOptions)) of
         {ok, TsInt} ->
             Item1 = Item#{timestamp => TsInt},
             continue_lines_to_points(Data, Item1, Rest, ResultPointsAcc, ErrorPointsAcc);
@@ -573,7 +573,7 @@ line_to_point(
     {_, EncodedTags} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags),
     {_, EncodedFields} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields),
     maps:without([precision], Item#{
-        measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Data),
+        measurement => emqx_placeholder:proc_tmpl(Measurement, Data),
         tags => EncodedTags,
         fields => EncodedFields,
         timestamp => maybe_convert_time_unit(Ts, Precision)
@@ -590,8 +590,8 @@ time_unit(ns) -> nanosecond.
 maps_config_to_data(K, V, {Data, Res}) ->
     KTransOptions = #{return => rawlist, var_trans => fun key_filter/1},
     VTransOptions = #{return => rawlist, var_trans => fun data_filter/1},
-    NK0 = emqx_plugin_libs_rule:proc_tmpl(K, Data, KTransOptions),
-    NV = emqx_plugin_libs_rule:proc_tmpl(V, Data, VTransOptions),
+    NK0 = emqx_placeholder:proc_tmpl(K, Data, KTransOptions),
+    NV = emqx_placeholder:proc_tmpl(V, Data, VTransOptions),
     case {NK0, NV} of
         {[undefined], _} ->
             {Data, Res};
diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl
index 0e1934525..087906549 100644
--- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl
+++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_impl.erl
@@ -185,7 +185,7 @@ preproc_data(
             timestamp => maybe_preproc_tmpl(
                 maps:get(<<"timestamp">>, Data, <<"now">>)
             ),
-            measurement => emqx_plugin_libs_rule:preproc_tmpl(Measurement),
+            measurement => emqx_placeholder:preproc_tmpl(Measurement),
             data_type => DataType,
             value => maybe_preproc_tmpl(Value)
         }
@@ -203,7 +203,7 @@ preproc_data(_NoMatch, Acc) ->
     Acc.
 
 maybe_preproc_tmpl(Value) when is_binary(Value) ->
-    emqx_plugin_libs_rule:preproc_tmpl(Value);
+    emqx_placeholder:preproc_tmpl(Value);
 maybe_preproc_tmpl(Value) ->
     Value.
 
@@ -225,7 +225,7 @@ proc_data(PreProcessedData, Msg) ->
         ) ->
             #{
                 timestamp => iot_timestamp(TimestampTkn, Msg, Nows),
-                measurement => emqx_plugin_libs_rule:proc_tmpl(Measurement, Msg),
+                measurement => emqx_placeholder:proc_tmpl(Measurement, Msg),
                 data_type => DataType,
                 value => proc_value(DataType, ValueTkn, Msg)
             }
@@ -236,7 +236,7 @@ proc_data(PreProcessedData, Msg) ->
 iot_timestamp(Timestamp, _, _) when is_integer(Timestamp) ->
     Timestamp;
 iot_timestamp(TimestampTkn, Msg, Nows) ->
-    iot_timestamp(emqx_plugin_libs_rule:proc_tmpl(TimestampTkn, Msg), Nows).
+    iot_timestamp(emqx_placeholder:proc_tmpl(TimestampTkn, Msg), Nows).
 
 iot_timestamp(Timestamp, #{now_ms := NowMs}) when
     Timestamp =:= <<"now">>; Timestamp =:= <<"now_ms">>; Timestamp =:= <<>>
@@ -250,7 +250,7 @@ iot_timestamp(Timestamp, _) when is_binary(Timestamp) ->
     binary_to_integer(Timestamp).
 
 proc_value(<<"TEXT">>, ValueTkn, Msg) ->
-    case emqx_plugin_libs_rule:proc_tmpl(ValueTkn, Msg) of
+    case emqx_placeholder:proc_tmpl(ValueTkn, Msg) of
         <<"undefined">> -> null;
         Val -> Val
     end;
@@ -262,7 +262,7 @@ proc_value(Int, ValueTkn, Msg) when Int =:= <<"FLOAT">>; Int =:= <<"DOUBLE">> ->
     convert_float(replace_var(ValueTkn, Msg)).
 
 replace_var(Tokens, Data) when is_list(Tokens) ->
-    [Val] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}),
+    [Val] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}),
     Val;
 replace_var(Val, _Data) ->
     Val.
@@ -410,8 +410,8 @@ device_id(Message, Payloads, State) ->
             %% [FIXME] there could be conflicting device-ids in the Payloads
             maps:get(<<"device_id">>, hd(Payloads), undefined);
         DeviceId ->
-            DeviceIdTkn = emqx_plugin_libs_rule:preproc_tmpl(DeviceId),
-            emqx_plugin_libs_rule:proc_tmpl(DeviceIdTkn, Message)
+            DeviceIdTkn = emqx_placeholder:preproc_tmpl(DeviceId),
+            emqx_placeholder:proc_tmpl(DeviceIdTkn, Message)
     end.
 
 handle_response({ok, 200, _Headers, Body} = Resp) ->
diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
index c0de23d94..9d03d53f9 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
@@ -68,7 +68,7 @@
     resource_id := resource_id(),
     topic_mapping := #{
         kafka_topic() := #{
-            payload_template := emqx_plugin_libs_rule:tmpl_token(),
+            payload_template := emqx_placeholder:tmpl_token(),
             mqtt_topic => emqx_types:topic(),
             qos => emqx_types:qos()
         }
@@ -82,7 +82,7 @@
     resource_id := resource_id(),
     topic_mapping := #{
         kafka_topic() := #{
-            payload_template := emqx_plugin_libs_rule:tmpl_token(),
+            payload_template := emqx_placeholder:tmpl_token(),
             mqtt_topic => emqx_types:topic(),
             qos => emqx_types:qos()
         }
@@ -536,7 +536,7 @@ convert_topic_mapping(TopicMappingList) ->
                 qos := QoS,
                 payload_template := PayloadTemplate0
             } = Fields,
-            PayloadTemplate = emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate0),
+            PayloadTemplate = emqx_placeholder:preproc_tmpl(PayloadTemplate0),
             Acc#{
                 KafkaTopic => #{
                     payload_template => PayloadTemplate,
@@ -559,7 +559,7 @@ render(FullMessage, PayloadTemplate) ->
                 emqx_plugin_libs_rule:bin(X)
         end
     },
-    emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, FullMessage, Opts).
+    emqx_placeholder:proc_tmpl(PayloadTemplate, FullMessage, Opts).
 
 encode(Value, none) ->
     Value;
diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
index 8b8337b09..cb8427cc5 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
@@ -240,7 +240,7 @@ compile_message_template(T) ->
     }.
 
 preproc_tmpl(Tmpl) ->
-    emqx_plugin_libs_rule:preproc_tmpl(Tmpl).
+    emqx_placeholder:preproc_tmpl(Tmpl).
 
 render_message(
     #{key := KeyTemplate, value := ValueTemplate, timestamp := TimestampTemplate}, Message
@@ -259,7 +259,7 @@ render(Template, Message) ->
         end,
         return => full_binary
     },
-    emqx_plugin_libs_rule:proc_tmpl(Template, Message, Opts).
+    emqx_placeholder:proc_tmpl(Template, Message, Opts).
 
 render_timestamp(Template, Message) ->
     try
diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl
index 8a8cffe55..48cae70d7 100644
--- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl
+++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl
@@ -21,7 +21,7 @@
 
 -export_type([msgvars/0]).
 
--type template() :: emqx_plugin_libs_rule:tmpl_token().
+-type template() :: emqx_placeholder:tmpl_token().
 
 -type msgvars() :: #{
     topic => template(),
@@ -48,7 +48,7 @@ parse(Conf) ->
 parse_field(Key, Conf, Acc) ->
     case Conf of
         #{Key := Val} when is_binary(Val) ->
-            Acc#{Key => emqx_plugin_libs_rule:preproc_tmpl(Val)};
+            Acc#{Key => emqx_placeholder:preproc_tmpl(Val)};
         #{Key := Val} ->
             Acc#{Key => Val};
         #{} ->
diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
index b8157d4fc..0b5885c16 100644
--- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
+++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
@@ -34,8 +34,8 @@
     value := binary()
 }.
 -type message_template() :: #{
-    key := emqx_plugin_libs_rule:tmpl_token(),
-    value := emqx_plugin_libs_rule:tmpl_token()
+    key := emqx_placeholder:tmpl_token(),
+    value := emqx_placeholder:tmpl_token()
 }.
 -type config() :: #{
     authentication := _,
@@ -421,7 +421,7 @@ compile_message_template(TemplateOpts) ->
     }.
 
 preproc_tmpl(Template) ->
-    emqx_plugin_libs_rule:preproc_tmpl(Template).
+    emqx_placeholder:preproc_tmpl(Template).
 
 render_message(
     Message, #{key := KeyTemplate, value := ValueTemplate}
@@ -439,7 +439,7 @@ render(Message, Template) ->
         end,
         return => full_binary
     },
-    emqx_plugin_libs_rule:proc_tmpl(Template, Message, Opts).
+    emqx_placeholder:proc_tmpl(Template, Message, Opts).
 
 get_producer_status(Producers) ->
     case pulsar_producers:all_connected(Producers) of
diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl
index 0dc73b5f6..c7aca7633 100644
--- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl
+++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl
@@ -225,7 +225,7 @@ on_start(
         {pool_size, PoolSize},
         {pool, InstanceID}
     ],
-    ProcessedTemplate = emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate),
+    ProcessedTemplate = emqx_placeholder:preproc_tmpl(PayloadTemplate),
     State = #{
         poolname => InstanceID,
         processed_payload_template => ProcessedTemplate,
@@ -547,7 +547,7 @@ is_send_message_atom(_) ->
 format_data([], Msg) ->
     emqx_utils_json:encode(Msg);
 format_data(Tokens, Msg) ->
-    emqx_plugin_libs_rule:proc_tmpl(Tokens, Msg).
+    emqx_placeholder:proc_tmpl(Tokens, Msg).
 
 handle_result({error, ecpool_empty}) ->
     {error, {recoverable_error, ecpool_empty}};
diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl
index a7d01960e..6167b7749 100644
--- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl
+++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl
@@ -102,7 +102,7 @@ on_start(
         emqx_schema:parse_servers(BinServers, ?ROCKETMQ_HOST_OPTIONS)
     ),
     ClientId = client_id(InstanceId),
-    TopicTks = emqx_plugin_libs_rule:preproc_tmpl(Topic),
+    TopicTks = emqx_placeholder:preproc_tmpl(Topic),
     #{acl_info := AclInfo} = ProducerOpts = make_producer_opts(Config),
     ClientCfg = #{acl_info => AclInfo},
     Templates = parse_template(Config),
@@ -240,7 +240,7 @@ parse_template(Config) ->
     parse_template(maps:to_list(Templates), #{}).
 
 parse_template([{Key, H} | T], Templates) ->
-    ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(H),
+    ParamsTks = emqx_placeholder:preproc_tmpl(H),
     parse_template(
         T,
         Templates#{Key => ParamsTks}
@@ -249,7 +249,7 @@ parse_template([], Templates) ->
     Templates.
 
 get_topic_key({_, Msg}, TopicTks) ->
-    emqx_plugin_libs_rule:proc_tmpl(TopicTks, Msg);
+    emqx_placeholder:proc_tmpl(TopicTks, Msg);
 get_topic_key([Query | _], TopicTks) ->
     get_topic_key(Query, TopicTks).
 
@@ -258,14 +258,14 @@ apply_template({Key, Msg} = _Req, Templates) ->
         undefined ->
             emqx_utils_json:encode(Msg);
         Template ->
-            emqx_plugin_libs_rule:proc_tmpl(Template, Msg)
+            emqx_placeholder:proc_tmpl(Template, Msg)
     end;
 apply_template([{Key, _} | _] = Reqs, Templates) ->
     case maps:get(Key, Templates, undefined) of
         undefined ->
             [emqx_utils_json:encode(Msg) || {_, Msg} <- Reqs];
         Template ->
-            [emqx_plugin_libs_rule:proc_tmpl(Template, Msg) || {_, Msg} <- Reqs]
+            [emqx_placeholder:proc_tmpl(Template, Msg) || {_, Msg} <- Reqs]
     end.
 
 client_id(ResourceId) ->
diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
index 52bd910db..edd8770f5 100644
--- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
+++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
@@ -455,9 +455,7 @@ parse_sql_template([{Key, H} | T], BatchInsertTks) ->
                             Key =>
                                 #{
                                     ?BATCH_INSERT_PART => InsertSQL,
-                                    ?BATCH_PARAMS_TOKENS => emqx_plugin_libs_rule:preproc_tmpl(
-                                        Params
-                                    )
+                                    ?BATCH_PARAMS_TOKENS => emqx_placeholder:preproc_tmpl(Params)
                                 }
                         }
                     );
@@ -478,7 +476,7 @@ parse_sql_template([], BatchInsertTks) ->
 apply_template(
     {?ACTION_SEND_MESSAGE = _Key, _Msg} = Query, Templates
 ) ->
-    %% TODO: fix emqx_plugin_libs_rule:proc_tmpl/2
+    %% TODO: fix emqx_placeholder:proc_tmpl/2
     %% it won't add single quotes for string
     apply_template([Query], Templates);
 %% batch inserts
diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
index 8fd41443c..6c1d06429 100644
--- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
+++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
@@ -126,8 +126,8 @@ on_query(InstanceId, {query, SQL}, State) ->
     do_query(InstanceId, SQL, State);
 on_query(InstanceId, {Key, Data}, #{insert_tokens := InsertTksMap} = State) ->
     case maps:find(Key, InsertTksMap) of
-        {ok, Tokens} ->
-            SQL = emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Data),
+        {ok, Tokens} when is_map(Data) ->
+            SQL = emqx_placeholder:proc_sql_param_str(Tokens, Data),
             do_query(InstanceId, SQL, State);
         _ ->
             {error, {unrecoverable_error, invalid_request}}
@@ -136,7 +136,7 @@ on_query(InstanceId, {Key, Data}, #{insert_tokens := InsertTksMap} = State) ->
 %% aggregate the batch queries to one SQL is a heavy job, we should put it in the worker process
 on_batch_query(
     InstanceId,
-    [{Key, _} | _] = BatchReq,
+    [{Key, _Data = #{}} | _] = BatchReq,
     #{batch_tokens := BatchTksMap, query_opts := Opts} = State
 ) ->
     case maps:find(Key, BatchTksMap) of
@@ -231,8 +231,8 @@ do_batch_insert(Conn, Tokens, BatchReqs, Opts) ->
 aggregate_query({InsertPartTks, ParamsPartTks}, BatchReqs) ->
     lists:foldl(
         fun({_, Data}, Acc) ->
-            InsertPart = emqx_plugin_libs_rule:proc_sql_param_str(InsertPartTks, Data),
-            ParamsPart = emqx_plugin_libs_rule:proc_sql_param_str(ParamsPartTks, Data),
+            InsertPart = emqx_placeholder:proc_sql_param_str(InsertPartTks, Data),
+            ParamsPart = emqx_placeholder:proc_sql_param_str(ParamsPartTks, Data),
             Values = maps:get(InsertPart, Acc, []),
             maps:put(InsertPart, [ParamsPart | Values], Acc)
         end,
@@ -260,12 +260,12 @@ parse_batch_prepare_sql([{Key, H} | T], InsertTksMap, BatchTksMap) ->
         {ok, select} ->
             parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap);
         {ok, insert} ->
-            InsertTks = emqx_plugin_libs_rule:preproc_tmpl(H),
+            InsertTks = emqx_placeholder:preproc_tmpl(H),
             H1 = string:trim(H, trailing, ";"),
             case split_insert_sql(H1) of
                 [_InsertStr, InsertPart, _ValuesStr, ParamsPart] ->
-                    InsertPartTks = emqx_plugin_libs_rule:preproc_tmpl(InsertPart),
-                    ParamsPartTks = emqx_plugin_libs_rule:preproc_tmpl(ParamsPart),
+                    InsertPartTks = emqx_placeholder:preproc_tmpl(InsertPart),
+                    ParamsPartTks = emqx_placeholder:preproc_tmpl(ParamsPart),
                     parse_batch_prepare_sql(
                         T,
                         InsertTksMap#{Key => InsertTks},
diff --git a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl
index 55f8bb69e..7644921f0 100644
--- a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl
+++ b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl
@@ -496,7 +496,7 @@ t_simple_sql_query(Config) ->
         ),
     case EnableBatch of
         true ->
-            ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result);
+            ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result);
         false ->
             ?assertMatch({ok, #{<<"code">> := 0, <<"data">> := [[1]]}}, Result)
     end,
@@ -535,7 +535,7 @@ t_bad_sql_parameter(Config) ->
             2_000
         ),
 
-    ?assertMatch({error, #{<<"code">> := _}}, Result),
+    ?assertMatch({error, {unrecoverable_error, invalid_request}}, Result),
     ok.
 
 t_nasty_sql_string(Config) ->
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index c44212cc3..149704f76 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -464,8 +464,8 @@ preprocess_request(
     } = Req
 ) ->
     #{
-        method => emqx_plugin_libs_rule:preproc_tmpl(to_bin(Method)),
-        path => emqx_plugin_libs_rule:preproc_tmpl(Path),
+        method => emqx_placeholder:preproc_tmpl(to_bin(Method)),
+        path => emqx_placeholder:preproc_tmpl(Path),
         body => maybe_preproc_tmpl(body, Req),
         headers => wrap_auth_header(preproc_headers(Headers)),
         request_timeout => maps:get(request_timeout, Req, ?DEFAULT_REQUEST_TIMEOUT_MS),
@@ -477,8 +477,8 @@ preproc_headers(Headers) when is_map(Headers) ->
         fun(K, V, Acc) ->
             [
                 {
-                    emqx_plugin_libs_rule:preproc_tmpl(to_bin(K)),
-                    emqx_plugin_libs_rule:preproc_tmpl(to_bin(V))
+                    emqx_placeholder:preproc_tmpl(to_bin(K)),
+                    emqx_placeholder:preproc_tmpl(to_bin(V))
                 }
                 | Acc
             ]
@@ -490,8 +490,8 @@ preproc_headers(Headers) when is_list(Headers) ->
     lists:map(
         fun({K, V}) ->
             {
-                emqx_plugin_libs_rule:preproc_tmpl(to_bin(K)),
-                emqx_plugin_libs_rule:preproc_tmpl(to_bin(V))
+                emqx_placeholder:preproc_tmpl(to_bin(K)),
+                emqx_placeholder:preproc_tmpl(to_bin(V))
             }
         end,
         Headers
@@ -530,7 +530,7 @@ try_bin_to_lower(Bin) ->
 maybe_preproc_tmpl(Key, Conf) ->
     case maps:get(Key, Conf, undefined) of
         undefined -> undefined;
-        Val -> emqx_plugin_libs_rule:preproc_tmpl(Val)
+        Val -> emqx_placeholder:preproc_tmpl(Val)
     end.
 
 process_request(
@@ -544,8 +544,8 @@ process_request(
     Msg
 ) ->
     Conf#{
-        method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg)),
-        path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg),
+        method => make_method(emqx_placeholder:proc_tmpl(MethodTks, Msg)),
+        path => emqx_placeholder:proc_tmpl(PathTks, Msg),
         body => process_request_body(BodyTks, Msg),
         headers => proc_headers(HeadersTks, Msg),
         request_timeout => ReqTimeout
@@ -554,14 +554,14 @@ process_request(
 process_request_body(undefined, Msg) ->
     emqx_utils_json:encode(Msg);
 process_request_body(BodyTks, Msg) ->
-    emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg).
+    emqx_placeholder:proc_tmpl(BodyTks, Msg).
 
 proc_headers(HeaderTks, Msg) ->
     lists:map(
         fun({K, V}) ->
             {
-                emqx_plugin_libs_rule:proc_tmpl(K, Msg),
-                emqx_plugin_libs_rule:proc_tmpl(emqx_secret:unwrap(V), Msg)
+                emqx_placeholder:proc_tmpl(K, Msg),
+                emqx_placeholder:proc_tmpl(emqx_secret:unwrap(V), Msg)
             }
         end,
         HeaderTks
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index 25db669ce..a46b21d92 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -344,7 +344,7 @@ parse_prepare_sql(Config) ->
     parse_prepare_sql(maps:to_list(SQL), #{}, #{}, #{}, #{}).
 
 parse_prepare_sql([{Key, H} | _] = L, Prepares, Tokens, BatchInserts, BatchTks) ->
-    {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H),
+    {PrepareSQL, ParamsTokens} = emqx_placeholder:preproc_sql(H),
     parse_batch_prepare_sql(
         L, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens}, BatchInserts, BatchTks
     );
@@ -363,7 +363,7 @@ parse_batch_prepare_sql([{Key, H} | T], Prepares, Tokens, BatchInserts, BatchTks
         {ok, insert} ->
             case emqx_plugin_libs_rule:split_insert_sql(H) of
                 {ok, {InsertSQL, Params}} ->
-                    ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(Params),
+                    ParamsTks = emqx_placeholder:preproc_tmpl(Params),
                     parse_prepare_sql(
                         T,
                         Prepares,
@@ -389,7 +389,7 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens})
         undefined ->
             {SQLOrData, Params};
         Tokens ->
-            {TypeOrKey, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)}
+            {TypeOrKey, emqx_placeholder:proc_sql(Tokens, SQLOrData)}
     end.
 
 on_batch_insert(InstId, BatchReqs, InsertPart, Tokens, State) ->
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index cbe95cb33..8fb60f102 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -188,7 +188,7 @@ on_batch_query(
                     {error, {unrecoverable_error, batch_prepare_not_implemented}};
                 TokenList ->
                     {_, Datas} = lists:unzip(BatchReq),
-                    Datas2 = [emqx_plugin_libs_rule:proc_sql(TokenList, Data) || Data <- Datas],
+                    Datas2 = [emqx_placeholder:proc_sql(TokenList, Data) || Data <- Datas],
                     St = maps:get(BinKey, Sts),
                     case on_sql_query(InstId, PoolName, execute_batch, St, Datas2) of
                         {error, _Error} = Result ->
@@ -218,7 +218,7 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens})
         undefined ->
             {SQLOrData, Params};
         Tokens ->
-            {Key, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)}
+            {Key, emqx_placeholder:proc_sql(Tokens, SQLOrData)}
     end.
 
 on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) ->
@@ -350,7 +350,7 @@ parse_prepare_sql(Config) ->
     parse_prepare_sql(maps:to_list(SQL), #{}, #{}).
 
 parse_prepare_sql([{Key, H} | T], Prepares, Tokens) ->
-    {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H, '$n'),
+    {PrepareSQL, ParamsTokens} = emqx_placeholder:preproc_sql(H, '$n'),
     parse_prepare_sql(
         T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens}
     );
diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl
index 84334875b..71a891850 100644
--- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl
+++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl
@@ -301,8 +301,8 @@ feedvar(Override, Packet, ConnInfo, ClientInfo) ->
     },
     maps:map(
         fun(_K, V) ->
-            Tokens = emqx_plugin_libs_rule:preproc_tmpl(V),
-            emqx_plugin_libs_rule:proc_tmpl(Tokens, Envs)
+            Tokens = emqx_placeholder:preproc_tmpl(V),
+            emqx_placeholder:proc_tmpl(Tokens, Envs)
         end,
         Override
     ).
diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl
index 316432dea..854ccad88 100644
--- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl
+++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl
@@ -283,8 +283,8 @@ feedvar(Override, Packet, ConnInfo, ClientInfo) ->
     },
     maps:map(
         fun(_K, V) ->
-            Tokens = emqx_plugin_libs_rule:preproc_tmpl(V),
-            emqx_plugin_libs_rule:proc_tmpl(Tokens, Envs)
+            Tokens = emqx_placeholder:preproc_tmpl(V),
+            emqx_placeholder:proc_tmpl(Tokens, Envs)
         end,
         Override
     ).
diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl
index 0f543badd..5d80c0d56 100644
--- a/apps/emqx_oracle/src/emqx_oracle.erl
+++ b/apps/emqx_oracle/src/emqx_oracle.erl
@@ -168,7 +168,7 @@ on_batch_query(
                     {error, {unrecoverable_error, batch_prepare_not_implemented}};
                 TokenList ->
                     {_, Datas} = lists:unzip(BatchReq),
-                    Datas2 = [emqx_plugin_libs_rule:proc_sql(TokenList, Data) || Data <- Datas],
+                    Datas2 = [emqx_placeholder:proc_sql(TokenList, Data) || Data <- Datas],
                     St = maps:get(BinKey, Sts),
                     case
                         on_sql_query(InstId, PoolName, execute_batch, ?SYNC_QUERY_MODE, St, Datas2)
@@ -204,7 +204,7 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{
                 undefined ->
                     {SQLOrData, Params};
                 Sql ->
-                    {Sql, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)}
+                    {Sql, emqx_placeholder:proc_sql(Tokens, SQLOrData)}
             end
     end.
 
@@ -305,7 +305,7 @@ parse_prepare_sql(Config) ->
     parse_prepare_sql(maps:to_list(SQL), #{}, #{}).
 
 parse_prepare_sql([{Key, H} | T], Prepares, Tokens) ->
-    {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H, ':n'),
+    {PrepareSQL, ParamsTokens} = emqx_placeholder:preproc_sql(H, ':n'),
     parse_prepare_sql(
         T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens}
     );
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
index 3bfac1ec4..b3e3507f0 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
@@ -19,17 +19,6 @@
 
 %% preprocess and process template string with place holders
 -export([
-    preproc_tmpl/1,
-    proc_tmpl/2,
-    proc_tmpl/3,
-    preproc_cmd/1,
-    proc_cmd/2,
-    proc_cmd/3,
-    preproc_sql/1,
-    preproc_sql/2,
-    proc_sql/2,
-    proc_sql_param_str/2,
-    proc_cql_param_str/2,
     split_insert_sql/1,
     detect_sql_type/1,
     proc_batch_sql/3,
@@ -61,68 +50,15 @@
 ]).
 
 -export([
-    now_ms/0,
     can_topic_match_oneof/2
 ]).
 
--export_type([tmpl_token/0]).
-
 -compile({no_auto_import, [float/1]}).
 
 -type uri_string() :: iodata().
 
 -type tmpl_token() :: list({var, binary()} | {str, binary()}).
 
--type tmpl_cmd() :: list(tmpl_token()).
-
--type prepare_statement_key() :: binary().
-
-%% preprocess template string with place holders
--spec preproc_tmpl(binary()) -> tmpl_token().
-preproc_tmpl(Str) ->
-    emqx_placeholder:preproc_tmpl(Str).
-
--spec proc_tmpl(tmpl_token(), map()) -> binary().
-proc_tmpl(Tokens, Data) ->
-    emqx_placeholder:proc_tmpl(Tokens, Data).
-
--spec proc_tmpl(tmpl_token(), map(), map()) -> binary() | list().
-proc_tmpl(Tokens, Data, Opts) ->
-    emqx_placeholder:proc_tmpl(Tokens, Data, Opts).
-
--spec preproc_cmd(binary()) -> tmpl_cmd().
-preproc_cmd(Str) ->
-    emqx_placeholder:preproc_cmd(Str).
-
--spec proc_cmd([tmpl_token()], map()) -> binary() | list().
-proc_cmd(Tokens, Data) ->
-    emqx_placeholder:proc_cmd(Tokens, Data).
--spec proc_cmd([tmpl_token()], map(), map()) -> list().
-proc_cmd(Tokens, Data, Opts) ->
-    emqx_placeholder:proc_cmd(Tokens, Data, Opts).
-
-%% preprocess SQL with place holders
--spec preproc_sql(Sql :: binary()) -> {prepare_statement_key(), tmpl_token()}.
-preproc_sql(Sql) ->
-    emqx_placeholder:preproc_sql(Sql).
-
--spec preproc_sql(Sql :: binary(), ReplaceWith :: '?' | '$n' | ':n') ->
-    {prepare_statement_key(), tmpl_token()}.
-preproc_sql(Sql, ReplaceWith) ->
-    emqx_placeholder:preproc_sql(Sql, ReplaceWith).
-
--spec proc_sql(tmpl_token(), map()) -> list().
-proc_sql(Tokens, Data) ->
-    emqx_placeholder:proc_sql(Tokens, Data).
-
--spec proc_sql_param_str(tmpl_token(), map()) -> binary().
-proc_sql_param_str(Tokens, Data) ->
-    emqx_placeholder:proc_sql_param_str(Tokens, Data).
-
--spec proc_cql_param_str(tmpl_token(), map()) -> binary().
-proc_cql_param_str(Tokens, Data) ->
-    emqx_placeholder:proc_cql_param_str(Tokens, Data).
-
 %% SQL = <<"INSERT INTO \"abc\" (c1,c2,c3) VALUES (${1}, ${1}, ${1})">>
 -spec split_insert_sql(binary()) -> {ok, {InsertSQL, Params}} | {error, atom()} when
     InsertSQL :: binary(),
@@ -169,7 +105,7 @@ detect_sql_type(SQL) ->
 proc_batch_sql(BatchReqs, InsertPart, Tokens) ->
     ValuesPart = erlang:iolist_to_binary(
         lists:join($,, [
-            proc_sql_param_str(Tokens, Msg)
+            emqx_placeholder:proc_sql_param_str(Tokens, Msg)
          || {_, Msg} <- BatchReqs
         ])
     ),
@@ -353,9 +289,6 @@ number_to_list(Int) when is_integer(Int) ->
 number_to_list(Float) when is_float(Float) ->
     float_to_list(Float, [{decimals, 10}, compact]).
 
-now_ms() ->
-    erlang:system_time(millisecond).
-
 can_topic_match_oneof(Topic, Filters) ->
     lists:any(
         fun(Fltr) ->
diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
index b562af09d..a5ad35066 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl
@@ -65,10 +65,10 @@ pre_process_action_args(
 ) ->
     Args#{
         preprocessed_tmpl => #{
-            topic => emqx_plugin_libs_rule:preproc_tmpl(Topic),
+            topic => emqx_placeholder:preproc_tmpl(Topic),
             qos => preproc_vars(QoS),
             retain => preproc_vars(Retain),
-            payload => emqx_plugin_libs_rule:preproc_tmpl(Payload),
+            payload => emqx_placeholder:preproc_tmpl(Payload),
             user_properties => preproc_user_properties(UserProperties)
         }
     };
@@ -110,7 +110,7 @@ republish(
         }
     }
 ) ->
-    Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected),
+    Topic = emqx_placeholder:proc_tmpl(TopicTks, Selected),
     Payload = format_msg(PayloadTks, Selected),
     QoS = replace_simple_var(QoSTks, Selected, 0),
     Retain = replace_simple_var(RetainTks, Selected, false),
@@ -189,7 +189,7 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) ->
     emqx_metrics:inc_msg(Msg).
 
 preproc_vars(Data) when is_binary(Data) ->
-    emqx_plugin_libs_rule:preproc_tmpl(Data);
+    emqx_placeholder:preproc_tmpl(Data);
 preproc_vars(Data) ->
     Data.
 
@@ -201,13 +201,13 @@ preproc_user_properties(<<"${pub_props.'User-Property'}">>) ->
     ?ORIGINAL_USER_PROPERTIES;
 preproc_user_properties(<<"${", _/binary>> = V) ->
     %% use a variable
-    emqx_plugin_libs_rule:preproc_tmpl(V);
+    emqx_placeholder:preproc_tmpl(V);
 preproc_user_properties(_) ->
     %% invalid, discard
     undefined.
 
 replace_simple_var(Tokens, Data, Default) when is_list(Tokens) ->
-    [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}),
+    [Var] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}),
     case Var of
         %% cannot find the variable from Data
         undefined -> Default;
@@ -219,7 +219,7 @@ replace_simple_var(Val, _Data, _Default) ->
 format_msg([], Selected) ->
     emqx_utils_json:encode(Selected);
 format_msg(Tokens, Selected) ->
-    emqx_plugin_libs_rule:proc_tmpl(Tokens, Selected).
+    emqx_placeholder:proc_tmpl(Tokens, Selected).
 
 format_pub_props(UserPropertiesTks, Selected, Env) ->
     UserProperties =
diff --git a/apps/emqx_plugin_libs/src/emqx_placeholder.erl b/apps/emqx_utils/src/emqx_placeholder.erl
similarity index 92%
rename from apps/emqx_plugin_libs/src/emqx_placeholder.erl
rename to apps/emqx_utils/src/emqx_placeholder.erl
index dcd666f5b..babac8a84 100644
--- a/apps/emqx_plugin_libs/src/emqx_placeholder.erl
+++ b/apps/emqx_utils/src/emqx_placeholder.erl
@@ -46,7 +46,7 @@
     quote_mysql/1
 ]).
 
--include_lib("emqx/include/emqx_placeholder.hrl").
+-define(PH_VAR_THIS, '$this').
 
 -define(EX_PLACE_HOLDER, "(\\$\\{[a-zA-Z0-9\\._]+\\})").
 
@@ -55,7 +55,7 @@
 %% Space and CRLF
 -define(EX_WITHE_CHARS, "\\s").
 
--type tmpl_token() :: list({var, binary()} | {str, binary()}).
+-type tmpl_token() :: list({var, ?PH_VAR_THIS | [binary()]} | {str, binary()}).
 
 -type tmpl_cmd() :: list(tmpl_token()).
 
@@ -123,11 +123,11 @@ proc_tmpl(Tokens, Data, Opts = #{return := rawlist}) ->
             ({str, Str}) ->
                 Str;
             ({var, Phld}) when is_function(Trans, 1) ->
-                Trans(get_phld_var(Phld, Data));
+                Trans(lookup_var(Phld, Data));
             ({var, Phld}) when is_function(Trans, 2) ->
-                Trans(Phld, get_phld_var(Phld, Data));
+                Trans(Phld, lookup_var(Phld, Data));
             ({var, Phld}) ->
-                get_phld_var(Phld, Data)
+                lookup_var(Phld, Data)
         end,
         Tokens
     ).
@@ -264,13 +264,35 @@ quote_mysql(Str) when is_binary(Str) ->
 quote_mysql(Str) ->
     quote_escape(Str, fun escape_mysql/1).
 
+lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] ->
+    Value;
+lookup_var([Prop | Rest], Data) ->
+    case lookup(Prop, Data) of
+        {ok, Value} ->
+            lookup_var(Rest, Value);
+        {error, _} ->
+            undefined
+    end.
+
+lookup(Prop, Data) when is_binary(Prop) ->
+    case maps:get(Prop, Data, undefined) of
+        undefined ->
+            try
+                {ok, maps:get(binary_to_existing_atom(Prop, utf8), Data)}
+            catch
+                error:{badkey, _} ->
+                    {error, undefined};
+                error:badarg ->
+                    {error, undefined}
+            end;
+        Value ->
+            {ok, Value}
+    end.
+
 %%------------------------------------------------------------------------------
 %% Internal functions
 %%------------------------------------------------------------------------------
 
-get_phld_var(Phld, Data) ->
-    emqx_rule_maps:nested_get(Phld, Data).
-
 preproc_var_re(#{placeholders := PHs, strip_double_quote := true}) ->
     Res = [ph_to_re(PH) || PH <- PHs],
     QuoteRes = ["\"" ++ Re ++ "\"" || Re <- Res],
@@ -340,9 +362,8 @@ parse_nested(<<".", R/binary>>) ->
     parse_nested(R);
 parse_nested(Attr) ->
     case string:split(Attr, <<".">>, all) of
-        [<<>>] -> {var, ?PH_VAR_THIS};
-        [Attr] -> {var, Attr};
-        Nested -> {path, [{key, P} || P <- Nested]}
+        [<<>>] -> ?PH_VAR_THIS;
+        Nested -> Nested
     end.
 
 unwrap(<<"\"${", Val/binary>>, _StripDoubleQuote = true) ->
diff --git a/apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl b/apps/emqx_utils/test/emqx_placeholder_SUITE.erl
similarity index 100%
rename from apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl
rename to apps/emqx_utils/test/emqx_placeholder_SUITE.erl
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
index 0ae9bb2dc..70eca83d7 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl
@@ -225,9 +225,9 @@ start_producer(
                 msg => "hstreamdb connector: producer started"
             }),
             EnableBatch = maps:get(enable_batch, Options, false),
-            Payload = emqx_plugin_libs_rule:preproc_tmpl(PayloadBin),
+            Payload = emqx_placeholder:preproc_tmpl(PayloadBin),
             OrderingKeyBin = maps:get(ordering_key, Options, <<"">>),
-            OrderingKey = emqx_plugin_libs_rule:preproc_tmpl(OrderingKeyBin),
+            OrderingKey = emqx_placeholder:preproc_tmpl(OrderingKeyBin),
             State = #{
                 client => Client,
                 producer => Producer,
@@ -254,8 +254,8 @@ start_producer(
     end.
 
 to_record(OrderingKeyTmpl, PayloadTmpl, Data) ->
-    OrderingKey = emqx_plugin_libs_rule:proc_tmpl(OrderingKeyTmpl, Data),
-    Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTmpl, Data),
+    OrderingKey = emqx_placeholder:proc_tmpl(OrderingKeyTmpl, Data),
+    Payload = emqx_placeholder:proc_tmpl(PayloadTmpl, Data),
     to_record(OrderingKey, Payload).
 
 to_record(OrderingKey, Payload) when is_binary(OrderingKey) ->
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl
index 59f763904..20985e961 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl
@@ -57,7 +57,7 @@ on_query(InstanceId, {send_message, Message0}, State) ->
         connector_state := ConnectorState
     } = State,
     NewConnectorState = ConnectorState#{
-        collection => emqx_plugin_libs_rule:proc_tmpl(CollectionTemplate, Message0)
+        collection => emqx_placeholder:proc_tmpl(CollectionTemplate, Message0)
     },
     Message = render_message(PayloadTemplate, Message0),
     Res = emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, NewConnectorState),
@@ -76,7 +76,7 @@ on_get_status(InstanceId, _State = #{connector_state := ConnectorState}) ->
 preprocess_template(undefined = _PayloadTemplate) ->
     undefined;
 preprocess_template(PayloadTemplate) ->
-    emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate).
+    emqx_placeholder:preproc_tmpl(PayloadTemplate).
 
 render_message(undefined = _PayloadTemplate, Message) ->
     Message;
@@ -102,14 +102,14 @@ format_data(PayloadTks, Msg) ->
     case maps:size(PreparedTupleMap) of
         % If no tuples were found simply proceed with the json decoding and be done with it
         0 ->
-            emqx_utils_json:decode(emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Msg), [return_maps]);
+            emqx_utils_json:decode(emqx_placeholder:proc_tmpl(PayloadTks, Msg), [return_maps]);
         _ ->
             % If tuples were found, replace the tuple values with the references created, run
             % the modified message through the json parser, and then at the end replace the
             % references with the actual tuple values.
             ProcessedMessage = replace_message_values_with_references(Msg, PreparedTupleMap),
             DecodedMap = emqx_utils_json:decode(
-                emqx_plugin_libs_rule:proc_tmpl(PayloadTks, ProcessedMessage), [return_maps]
+                emqx_placeholder:proc_tmpl(PayloadTks, ProcessedMessage), [return_maps]
             ),
             populate_map_with_tuple_values(PreparedTupleMap, DecodedMap)
     end.
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_redis.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_redis.erl
index 4ce96d5c7..9caba2beb 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_redis.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_redis.erl
@@ -126,13 +126,13 @@ process_batch_data(BatchData, CommandTemplate) ->
 proc_command_template(CommandTemplate, Msg) ->
     lists:map(
         fun(ArgTks) ->
-            emqx_plugin_libs_rule:proc_tmpl(ArgTks, Msg, #{return => full_binary})
+            emqx_placeholder:proc_tmpl(ArgTks, Msg, #{return => full_binary})
         end,
         CommandTemplate
     ).
 
 preproc_command_template(CommandTemplate) ->
     lists:map(
-        fun emqx_plugin_libs_rule:preproc_tmpl/1,
+        fun emqx_placeholder:preproc_tmpl/1,
         CommandTemplate
     ).

From 7d0abb6146f976be25437179593d1e74dc880656 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Thu, 8 Jun 2023 08:56:24 +0300
Subject: [PATCH 25/33] feat(emqx): add `emqx_topic:match_any/2` utility

---
 apps/emqx/src/emqx_topic.erl                        |  7 +++++++
 apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl | 12 ------------
 apps/emqx_rule_engine/src/emqx_rule_engine.erl      |  2 +-
 apps/emqx_rule_engine/src/emqx_rule_sqltester.erl   |  3 +--
 4 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl
index 62dca99c7..c1515e14b 100644
--- a/apps/emqx/src/emqx_topic.erl
+++ b/apps/emqx/src/emqx_topic.erl
@@ -21,6 +21,7 @@
 %% APIs
 -export([
     match/2,
+    match_any/2,
     validate/1,
     validate/2,
     levels/1,
@@ -86,6 +87,12 @@ match([_H1 | _], []) ->
 match([], [_H | _T2]) ->
     false.
 
+-spec match_any(Name, [Filter]) -> boolean() when
+    Name :: topic() | words(),
+    Filter :: topic() | words().
+match_any(Topic, Filters) ->
+    lists:any(fun(Filter) -> match(Topic, Filter) end, Filters).
+
 %% @doc Validate topic name or filter
 -spec validate(topic() | {name | filter, topic()}) -> true.
 validate(Topic) when is_binary(Topic) ->
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
index b3e3507f0..fd792310b 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
@@ -49,10 +49,6 @@
     tcp_connectivity/3
 ]).
 
--export([
-    can_topic_match_oneof/2
-]).
-
 -compile({no_auto_import, [float/1]}).
 
 -type uri_string() :: iodata().
@@ -288,11 +284,3 @@ number_to_list(Int) when is_integer(Int) ->
     integer_to_list(Int);
 number_to_list(Float) when is_float(Float) ->
     float_to_list(Float, [{decimals, 10}, compact]).
-
-can_topic_match_oneof(Topic, Filters) ->
-    lists:any(
-        fun(Fltr) ->
-            emqx_topic:match(Topic, Fltr)
-        end,
-        Filters
-    ).
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
index 24ad2c5f0..ff6636b9a 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
@@ -206,7 +206,7 @@ get_rules_for_topic(Topic) ->
     [
         Rule
      || Rule = #{from := From} <- get_rules(),
-        emqx_plugin_libs_rule:can_topic_match_oneof(Topic, From)
+        emqx_topic:match_any(Topic, From)
     ].
 
 -spec get_rules_with_same_event(Topic :: binary()) -> [rule()].
diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
index 455efe389..f3b4e2790 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl
@@ -14,7 +14,6 @@
 
 -module(emqx_rule_sqltester).
 
--include("rule_engine.hrl").
 -include_lib("emqx/include/logger.hrl").
 
 -export([
@@ -31,7 +30,7 @@ test(#{sql := Sql, context := Context}) ->
             case lists:all(fun is_publish_topic/1, EventTopics) of
                 true ->
                     %% test if the topic matches the topic filters in the rule
-                    case emqx_plugin_libs_rule:can_topic_match_oneof(InTopic, EventTopics) of
+                    case emqx_topic:match_any(InTopic, EventTopics) of
                         true -> test_rule(Sql, Select, Context, EventTopics);
                         false -> {error, nomatch}
                     end;

From a51baaa206bbbb084a7b9568e1f391fc4722762e Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Thu, 8 Jun 2023 14:42:20 +0300
Subject: [PATCH 26/33] refactor(pluglib): move conversion utils to
 `emqx_utils_conv`

---
 .../src/emqx_bridge_influxdb_connector.erl    |   4 +-
 .../src/emqx_bridge_kafka_impl_consumer.erl   |   2 +-
 .../src/emqx_bridge_kafka_impl_producer.erl   |   2 +-
 .../src/emqx_bridge_pulsar_impl_producer.erl  |   2 +-
 .../src/emqx_bridge_sqlserver_connector.erl   |   2 +-
 apps/emqx_oracle/src/emqx_oracle.erl          |  10 +-
 .../src/emqx_plugin_libs_rule.erl             |  41 +-----
 .../test/emqx_plugin_libs_rule_SUITE.erl      |  23 +---
 apps/emqx_rule_engine/src/emqx_rule_funcs.erl |  40 +++---
 apps/emqx_rule_engine/src/emqx_rule_maps.erl  |   2 +-
 .../test/emqx_rule_funcs_SUITE.erl            |   6 +-
 apps/emqx_utils/src/emqx_placeholder.erl      |   4 +-
 apps/emqx_utils/src/emqx_utils_conv.erl       | 125 ++++++++++++++++++
 .../emqx_utils/test/emqx_utils_conv_tests.erl |  44 ++++++
 14 files changed, 215 insertions(+), 92 deletions(-)
 create mode 100644 apps/emqx_utils/src/emqx_utils_conv.erl
 create mode 100644 apps/emqx_utils/test/emqx_utils_conv_tests.erl

diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
index 71ab85c58..adf61918a 100644
--- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
+++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl
@@ -637,7 +637,7 @@ value_type(Val) ->
     Val.
 
 key_filter(undefined) -> undefined;
-key_filter(Value) -> emqx_plugin_libs_rule:bin(Value).
+key_filter(Value) -> emqx_utils_conv:bin(Value).
 
 data_filter(undefined) -> undefined;
 data_filter(Int) when is_integer(Int) -> Int;
@@ -645,7 +645,7 @@ data_filter(Number) when is_number(Number) -> Number;
 data_filter(Bool) when is_boolean(Bool) -> Bool;
 data_filter(Data) -> bin(Data).
 
-bin(Data) -> emqx_plugin_libs_rule:bin(Data).
+bin(Data) -> emqx_utils_conv:bin(Data).
 
 %% helper funcs
 log_error_points(InstId, Errs) ->
diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
index 9d03d53f9..a95e67b8d 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl
@@ -556,7 +556,7 @@ render(FullMessage, PayloadTemplate) ->
             (undefined) ->
                 <<>>;
             (X) ->
-                emqx_plugin_libs_rule:bin(X)
+                emqx_utils_conv:bin(X)
         end
     },
     emqx_placeholder:proc_tmpl(PayloadTemplate, FullMessage, Opts).
diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
index cb8427cc5..2d48c788b 100644
--- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
+++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl
@@ -255,7 +255,7 @@ render(Template, Message) ->
     Opts = #{
         var_trans => fun
             (undefined) -> <<"">>;
-            (X) -> emqx_plugin_libs_rule:bin(X)
+            (X) -> emqx_utils_conv:bin(X)
         end,
         return => full_binary
     },
diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
index 0b5885c16..133b98710 100644
--- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
+++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl
@@ -435,7 +435,7 @@ render(Message, Template) ->
     Opts = #{
         var_trans => fun
             (undefined) -> <<"">>;
-            (X) -> emqx_plugin_libs_rule:bin(X)
+            (X) -> emqx_utils_conv:bin(X)
         end,
         return => full_binary
     },
diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
index edd8770f5..cb0c0e16e 100644
--- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
+++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
@@ -44,7 +44,7 @@
 %% Internal exports used to execute code with ecpool worker
 -export([do_get_status/1, worker_do_insert/3]).
 
--import(emqx_plugin_libs_rule, [str/1]).
+-import(emqx_utils_conv, [str/1]).
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
 -define(ACTION_SEND_MESSAGE, send_message).
diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl
index 5d80c0d56..d6fd51847 100644
--- a/apps/emqx_oracle/src/emqx_oracle.erl
+++ b/apps/emqx_oracle/src/emqx_oracle.erl
@@ -93,14 +93,14 @@ on_start(
     ServiceName =
         case maps:get(service_name, Config, undefined) of
             undefined -> undefined;
-            ServiceName0 -> emqx_plugin_libs_rule:str(ServiceName0)
+            ServiceName0 -> emqx_utils_conv:str(ServiceName0)
         end,
     Options = [
         {host, Host},
         {port, Port},
-        {user, emqx_plugin_libs_rule:str(User)},
+        {user, emqx_utils_conv:str(User)},
         {password, jamdb_secret:wrap(maps:get(password, Config, ""))},
-        {sid, emqx_plugin_libs_rule:str(Sid)},
+        {sid, emqx_utils_conv:str(Sid)},
         {service_name, ServiceName},
         {pool_size, maps:get(<<"pool_size">>, Config, ?DEFAULT_POOL_SIZE)},
         {timeout, ?OPT_TIMEOUT},
@@ -268,14 +268,14 @@ connect(Opts) ->
     jamdb_oracle:start_link(Opts).
 
 sql_query_to_str(SqlQuery) ->
-    emqx_plugin_libs_rule:str(SqlQuery).
+    emqx_utils_conv:str(SqlQuery).
 
 sql_params_to_str(Params) when is_list(Params) ->
     lists:map(
         fun
             (false) -> "0";
             (true) -> "1";
-            (Value) -> emqx_plugin_libs_rule:str(Value)
+            (Value) -> emqx_utils_conv:str(Value)
         end,
         Params
     ).
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
index fd792310b..aa9e38316 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
@@ -27,8 +27,6 @@
 
 %% type converting
 -export([
-    str/1,
-    bin/1,
     bool/1,
     int/1,
     float/1,
@@ -36,7 +34,6 @@
     map/1,
     utf8_bin/1,
     utf8_str/1,
-    number_to_binary/1,
     atom_key/1,
     unsafe_atom_key/1
 ]).
@@ -172,39 +169,15 @@ tcp_connectivity(Host, Port, Timeout) ->
             {error, Reason}
     end.
 
-str(Bin) when is_binary(Bin) -> binary_to_list(Bin);
-str(Num) when is_number(Num) -> number_to_list(Num);
-str(Atom) when is_atom(Atom) -> atom_to_list(Atom);
-str(Map) when is_map(Map) -> binary_to_list(emqx_utils_json:encode(Map));
-str(List) when is_list(List) ->
-    case io_lib:printable_list(List) of
-        true -> List;
-        false -> binary_to_list(emqx_utils_json:encode(List))
-    end;
-str(Data) ->
-    error({invalid_str, Data}).
-
 utf8_bin(Str) when is_binary(Str); is_list(Str) ->
     unicode:characters_to_binary(Str);
 utf8_bin(Str) ->
-    unicode:characters_to_binary(bin(Str)).
+    unicode:characters_to_binary(emqx_utils_conv:bin(Str)).
 
 utf8_str(Str) when is_binary(Str); is_list(Str) ->
     unicode:characters_to_list(Str);
 utf8_str(Str) ->
-    unicode:characters_to_list(str(Str)).
-
-bin(Bin) when is_binary(Bin) -> Bin;
-bin(Num) when is_number(Num) -> number_to_binary(Num);
-bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
-bin(Map) when is_map(Map) -> emqx_utils_json:encode(Map);
-bin(List) when is_list(List) ->
-    case io_lib:printable_list(List) of
-        true -> list_to_binary(List);
-        false -> emqx_utils_json:encode(List)
-    end;
-bin(Data) ->
-    error({invalid_bin, Data}).
+    unicode:characters_to_list(emqx_utils_conv:str(Str)).
 
 int(List) when is_list(List) ->
     try
@@ -274,13 +247,3 @@ bool(Bool) when
     false;
 bool(Bool) ->
     error({invalid_boolean, Bool}).
-
-number_to_binary(Int) when is_integer(Int) ->
-    integer_to_binary(Int);
-number_to_binary(Float) when is_float(Float) ->
-    float_to_binary(Float, [{decimals, 10}, compact]).
-
-number_to_list(Int) when is_integer(Int) ->
-    integer_to_list(Int);
-number_to_list(Float) when is_float(Float) ->
-    float_to_list(Float, [{decimals, 10}, compact]).
diff --git a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl
index bf509f362..bb1ba838b 100644
--- a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl
+++ b/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl
@@ -28,11 +28,11 @@ all() -> emqx_common_test_helpers:all(?MODULE).
 t_http_connectivity(_) ->
     {ok, Socket} = gen_tcp:listen(?PORT, []),
     ok = emqx_plugin_libs_rule:http_connectivity(
-        "http://127.0.0.1:" ++ emqx_plugin_libs_rule:str(?PORT), 1000
+        "http://127.0.0.1:" ++ integer_to_list(?PORT), 1000
     ),
     gen_tcp:close(Socket),
     {error, _} = emqx_plugin_libs_rule:http_connectivity(
-        "http://127.0.0.1:" ++ emqx_plugin_libs_rule:str(?PORT), 1000
+        "http://127.0.0.1:" ++ integer_to_list(?PORT), 1000
     ).
 
 t_tcp_connectivity(_) ->
@@ -41,25 +41,6 @@ t_tcp_connectivity(_) ->
     gen_tcp:close(Socket),
     {error, _} = emqx_plugin_libs_rule:tcp_connectivity("127.0.0.1", ?PORT, 1000).
 
-t_str(_) ->
-    ?assertEqual("abc", emqx_plugin_libs_rule:str("abc")),
-    ?assertEqual("abc", emqx_plugin_libs_rule:str(abc)),
-    ?assertEqual("{\"a\":1}", emqx_plugin_libs_rule:str(#{a => 1})),
-    ?assertEqual("1", emqx_plugin_libs_rule:str(1)),
-    ?assertEqual("2.0", emqx_plugin_libs_rule:str(2.0)),
-    ?assertEqual("true", emqx_plugin_libs_rule:str(true)),
-    ?assertError(_, emqx_plugin_libs_rule:str({a, v})).
-
-t_bin(_) ->
-    ?assertEqual(<<"abc">>, emqx_plugin_libs_rule:bin("abc")),
-    ?assertEqual(<<"abc">>, emqx_plugin_libs_rule:bin(abc)),
-    ?assertEqual(<<"{\"a\":1}">>, emqx_plugin_libs_rule:bin(#{a => 1})),
-    ?assertEqual(<<"[{\"a\":1}]">>, emqx_plugin_libs_rule:bin([#{a => 1}])),
-    ?assertEqual(<<"1">>, emqx_plugin_libs_rule:bin(1)),
-    ?assertEqual(<<"2.0">>, emqx_plugin_libs_rule:bin(2.0)),
-    ?assertEqual(<<"true">>, emqx_plugin_libs_rule:bin(true)),
-    ?assertError(_, emqx_plugin_libs_rule:bin({a, v})).
-
 t_atom_key(_) ->
     _ = erlang,
     _ = port,
diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
index a94aa0c8a..de9bf0485 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
@@ -17,7 +17,6 @@
 -module(emqx_rule_funcs).
 
 -include("rule_engine.hrl").
--include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
 
 -elvis([{elvis_style, god_modules, disable}]).
@@ -266,8 +265,6 @@
     ]}
 ).
 
--define(is_var(X), is_binary(X)).
-
 %% @doc "msgid()" Func
 msgid() ->
     fun
@@ -631,29 +628,42 @@ do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"signed">>, <<"little">>) ->
 %%------------------------------------------------------------------------------
 
 str(Data) ->
-    emqx_plugin_libs_rule:bin(Data).
+    emqx_utils_conv:bin(Data).
 
+str_utf8(Data) when is_binary(Data); is_list(Data) ->
+    unicode:characters_to_binary(Data);
 str_utf8(Data) ->
-    emqx_plugin_libs_rule:utf8_bin(Data).
+    unicode:characters_to_binary(str(Data)).
 
 bool(Data) ->
-    emqx_plugin_libs_rule:bool(Data).
+    emqx_utils_conv:bool(Data).
 
 int(Data) ->
-    emqx_plugin_libs_rule:int(Data).
+    emqx_utils_conv:int(Data).
 
 float(Data) ->
-    emqx_plugin_libs_rule:float(Data).
+    emqx_utils_conv:float(Data).
 
 float(Data, Decimals) when Decimals > 0 ->
-    Data1 = ?MODULE:float(Data),
+    Data1 = emqx_utils_conv:float(Data),
     list_to_float(float_to_list(Data1, [{decimals, Decimals}])).
 
 float2str(Float, Precision) ->
-    emqx_plugin_libs_rule:float2str(Float, Precision).
+    float_to_binary(Float, [{decimals, Precision}, compact]).
 
+map(Bin) when is_binary(Bin) ->
+    case emqx_utils_json:decode(Bin) of
+        Map = #{} ->
+            Map;
+        _ ->
+            error(badarg, [Bin])
+    end;
+map(List) when is_list(List) ->
+    maps:from_list(List);
+map(Map = #{}) ->
+    Map;
 map(Data) ->
-    emqx_plugin_libs_rule:map(Data).
+    error(badarg, [Data]).
 
 bin2hexstr(Bin) when is_binary(Bin) ->
     emqx_utils:bin_to_hexstr(Bin, upper).
@@ -895,7 +905,7 @@ mget(Key, Map, Default) ->
             Val;
         error when is_atom(Key) ->
             %% the map may have an equivalent binary-form key
-            BinKey = emqx_plugin_libs_rule:bin(Key),
+            BinKey = emqx_utils_conv:bin(Key),
             case maps:find(BinKey, Map) of
                 {ok, Val} -> Val;
                 error -> Default
@@ -922,7 +932,7 @@ mput(Key, Val, Map) ->
             maps:put(Key, Val, Map);
         error when is_atom(Key) ->
             %% the map may have an equivalent binary-form key
-            BinKey = emqx_plugin_libs_rule:bin(Key),
+            BinKey = emqx_utils_conv:bin(Key),
             case maps:find(BinKey, Map) of
                 {ok, _} -> maps:put(BinKey, Val, Map);
                 error -> maps:put(Key, Val, Map)
@@ -1053,7 +1063,7 @@ unix_ts_to_rfc3339(Epoch) ->
     unix_ts_to_rfc3339(Epoch, <<"second">>).
 
 unix_ts_to_rfc3339(Epoch, Unit) when is_integer(Epoch) ->
-    emqx_plugin_libs_rule:bin(
+    emqx_utils_conv:bin(
         calendar:system_time_to_rfc3339(
             Epoch, [{unit, time_unit(Unit)}]
         )
@@ -1090,7 +1100,7 @@ format_date(TimeUnit, Offset, FormatString) ->
 
 format_date(TimeUnit, Offset, FormatString, TimeEpoch) ->
     Unit = time_unit(TimeUnit),
-    emqx_plugin_libs_rule:bin(
+    emqx_utils_conv:bin(
         lists:concat(
             emqx_calendar:format(TimeEpoch, Unit, Offset, FormatString)
         )
diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/apps/emqx_rule_engine/src/emqx_rule_maps.erl
index 3dfffca46..baf83ff6f 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_maps.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_maps.erl
@@ -97,7 +97,7 @@ general_find({key, Key}, Map, _OrgData, Handler) when is_map(Map) ->
             Handler({found, {{key, Key}, Val}});
         error when is_atom(Key) ->
             %% the map may have an equivalent binary-form key
-            BinKey = emqx_plugin_libs_rule:bin(Key),
+            BinKey = atom_to_binary(Key),
             case maps:find(BinKey, Map) of
                 {ok, Val} -> Handler({equivalent, {{key, BinKey}, Val}});
                 error -> Handler(not_found)
diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
index 12110b206..6e67f6cd1 100644
--- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
+++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
@@ -126,7 +126,7 @@ t_int(_) ->
     ?assertEqual(1, emqx_rule_funcs:int(1.0001)),
     ?assertEqual(1, emqx_rule_funcs:int(true)),
     ?assertEqual(0, emqx_rule_funcs:int(false)),
-    ?assertError({invalid_number, {a, v}}, emqx_rule_funcs:int({a, v})),
+    ?assertError(badarg, emqx_rule_funcs:int({a, v})),
     ?assertError(_, emqx_rule_funcs:int("a")).
 
 t_float(_) ->
@@ -137,7 +137,7 @@ t_float(_) ->
     ?assertEqual(1.9, emqx_rule_funcs:float(1.9)),
     ?assertEqual(1.0001, emqx_rule_funcs:float(1.0001)),
     ?assertEqual(1.0000000001, emqx_rule_funcs:float(1.0000000001)),
-    ?assertError({invalid_number, {a, v}}, emqx_rule_funcs:float({a, v})),
+    ?assertError(badarg, emqx_rule_funcs:float({a, v})),
     ?assertError(_, emqx_rule_funcs:float("a")).
 
 t_map(_) ->
@@ -158,7 +158,7 @@ t_bool(_) ->
     ?assertEqual(true, emqx_rule_funcs:bool(<<"true">>)),
     ?assertEqual(false, emqx_rule_funcs:bool(false)),
     ?assertEqual(false, emqx_rule_funcs:bool(<<"false">>)),
-    ?assertError({invalid_boolean, _}, emqx_rule_funcs:bool(3)).
+    ?assertError(badarg, emqx_rule_funcs:bool(3)).
 
 t_proc_dict_put_get_del(_) ->
     ?assertEqual(undefined, emqx_rule_funcs:proc_dict_get(<<"abc">>)),
diff --git a/apps/emqx_utils/src/emqx_placeholder.erl b/apps/emqx_utils/src/emqx_placeholder.erl
index babac8a84..861548f93 100644
--- a/apps/emqx_utils/src/emqx_placeholder.erl
+++ b/apps/emqx_utils/src/emqx_placeholder.erl
@@ -112,7 +112,7 @@ proc_tmpl(Tokens, Data) ->
 
 -spec proc_tmpl(tmpl_token(), map(), proc_tmpl_opts()) -> binary() | list().
 proc_tmpl(Tokens, Data, Opts = #{return := full_binary}) ->
-    Trans = maps:get(var_trans, Opts, fun emqx_plugin_libs_rule:bin/1),
+    Trans = maps:get(var_trans, Opts, fun emqx_utils_conv:bin/1),
     list_to_binary(
         proc_tmpl(Tokens, Data, #{return => rawlist, var_trans => Trans})
     );
@@ -243,7 +243,7 @@ sql_data(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
 sql_data(Map) when is_map(Map) -> emqx_utils_json:encode(Map).
 
 -spec bin(term()) -> binary().
-bin(Val) -> emqx_plugin_libs_rule:bin(Val).
+bin(Val) -> emqx_utils_conv:bin(Val).
 
 -spec quote_sql(_Value) -> iolist().
 quote_sql(Str) ->
diff --git a/apps/emqx_utils/src/emqx_utils_conv.erl b/apps/emqx_utils/src/emqx_utils_conv.erl
new file mode 100644
index 000000000..9879d3a9d
--- /dev/null
+++ b/apps/emqx_utils/src/emqx_utils_conv.erl
@@ -0,0 +1,125 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_utils_conv).
+
+-export([bin/1]).
+-export([str/1]).
+-export([bool/1]).
+-export([int/1]).
+-export([float/1]).
+
+-compile({no_auto_import, [float/1]}).
+
+-type scalar() :: binary() | number() | atom() | string().
+
+-spec bin(Term) -> binary() when
+    Term :: scalar() | #{scalar() => Term} | [Term].
+bin(Bin) when is_binary(Bin) -> Bin;
+bin(Num) when is_number(Num) -> number_to_binary(Num);
+bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
+bin(Map) when is_map(Map) -> emqx_utils_json:encode(Map);
+bin(List) when is_list(List) ->
+    case io_lib:printable_list(List) of
+        true -> list_to_binary(List);
+        false -> emqx_utils_json:encode(List)
+    end;
+bin(Data) ->
+    error({invalid_bin, Data}).
+
+-spec str(Term) -> string() when
+    Term :: scalar() | #{scalar() => Term} | [Term].
+str(Bin) when is_binary(Bin) -> binary_to_list(Bin);
+str(Num) when is_number(Num) -> number_to_list(Num);
+str(Atom) when is_atom(Atom) -> atom_to_list(Atom);
+str(Map) when is_map(Map) -> binary_to_list(emqx_utils_json:encode(Map));
+str(List) when is_list(List) ->
+    case io_lib:printable_list(List) of
+        true -> List;
+        false -> binary_to_list(emqx_utils_json:encode(List))
+    end;
+str(Data) ->
+    error({invalid_str, Data}).
+
+-spec number_to_binary(number()) -> binary().
+number_to_binary(Int) when is_integer(Int) ->
+    integer_to_binary(Int);
+number_to_binary(Float) when is_float(Float) ->
+    float_to_binary(Float, [{decimals, 10}, compact]).
+
+-spec number_to_list(number()) -> string().
+number_to_list(Int) when is_integer(Int) ->
+    integer_to_list(Int);
+number_to_list(Float) when is_float(Float) ->
+    float_to_list(Float, [{decimals, 10}, compact]).
+
+-spec bool(Term) -> boolean() when
+    Term :: boolean() | binary() | 0..1.
+bool(true) -> true;
+bool(<<"true">>) -> true;
+bool(N) when N == 1 -> true;
+bool(false) -> false;
+bool(<<"false">>) -> false;
+bool(N) when N == 0 -> false;
+bool(Data) -> error(badarg, [Data]).
+
+-spec int(Term) -> integer() when
+    Term :: binary() | string() | number() | boolean().
+int(List) when is_list(List) ->
+    try
+        list_to_integer(List)
+    catch
+        error:badarg ->
+            int(list_to_float(List))
+    end;
+int(Bin) when is_binary(Bin) ->
+    try
+        binary_to_integer(Bin)
+    catch
+        error:badarg ->
+            int(binary_to_float(Bin))
+    end;
+int(Int) when is_integer(Int) ->
+    Int;
+int(Float) when is_float(Float) ->
+    erlang:floor(Float);
+int(true) ->
+    1;
+int(false) ->
+    0;
+int(Data) ->
+    error(badarg, [Data]).
+
+-spec float(Term) -> float() when
+    Term :: binary() | string() | number().
+float(List) when is_list(List) ->
+    try
+        list_to_float(List)
+    catch
+        error:badarg ->
+            float(list_to_integer(List))
+    end;
+float(Bin) when is_binary(Bin) ->
+    try
+        binary_to_float(Bin)
+    catch
+        error:badarg ->
+            float(binary_to_integer(Bin))
+    end;
+float(Num) when is_number(Num) ->
+    erlang:float(Num);
+float(Data) ->
+    error(badarg, [Data]).
diff --git a/apps/emqx_utils/test/emqx_utils_conv_tests.erl b/apps/emqx_utils/test/emqx_utils_conv_tests.erl
new file mode 100644
index 000000000..b60467264
--- /dev/null
+++ b/apps/emqx_utils/test/emqx_utils_conv_tests.erl
@@ -0,0 +1,44 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_utils_conv_tests).
+
+-import(emqx_utils_conv, [bin/1, str/1]).
+
+-include_lib("eunit/include/eunit.hrl").
+
+bin_test_() ->
+    [
+        ?_assertEqual(<<"abc">>, bin("abc")),
+        ?_assertEqual(<<"abc">>, bin(abc)),
+        ?_assertEqual(<<"{\"a\":1}">>, bin(#{a => 1})),
+        ?_assertEqual(<<"[{\"a\":1}]">>, bin([#{a => 1}])),
+        ?_assertEqual(<<"1">>, bin(1)),
+        ?_assertEqual(<<"2.0">>, bin(2.0)),
+        ?_assertEqual(<<"true">>, bin(true)),
+        ?_assertError(_, bin({a, v}))
+    ].
+
+str_test_() ->
+    [
+        ?_assertEqual("abc", str("abc")),
+        ?_assertEqual("abc", str(abc)),
+        ?_assertEqual("{\"a\":1}", str(#{a => 1})),
+        ?_assertEqual("1", str(1)),
+        ?_assertEqual("2.0", str(2.0)),
+        ?_assertEqual("true", str(true)),
+        ?_assertError(_, str({a, v}))
+    ].

From 8919a6ef93f78774e2e23c74903b635f4602c616 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Fri, 9 Jun 2023 00:21:39 +0300
Subject: [PATCH 27/33] refactor(pluglib): provide SQL related utils in
 `emqx_utils_sql`

---
 .../src/emqx_bridge_clickhouse_connector.erl  |   2 +-
 .../src/emqx_bridge_sqlserver_connector.erl   |  22 ++-
 .../src/emqx_bridge_tdengine_connector.erl    |  17 +-
 .../src/emqx_connector_mysql.erl              |  11 +-
 .../src/emqx_plugin_libs_rule.erl             |  49 +-----
 apps/emqx_utils/src/emqx_placeholder.erl      |  69 +-------
 apps/emqx_utils/src/emqx_utils_sql.erl        | 157 ++++++++++++++++++
 7 files changed, 199 insertions(+), 128 deletions(-)
 create mode 100644 apps/emqx_utils/src/emqx_utils_sql.erl

diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
index 002460c95..2d9af6bee 100644
--- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
+++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
@@ -212,7 +212,7 @@ prepare_sql_bulk_extend_template(Template, Separator) ->
     ExtendParamTemplate = iolist_to_binary([Separator, ValuesTemplate]),
     emqx_placeholder:preproc_tmpl(ExtendParamTemplate).
 
-%% This function is similar to emqx_plugin_libs_rule:split_insert_sql/1 but can
+%% This function is similar to emqx_utils_sql:parse_insert/1 but can
 %% also handle Clickhouse's SQL extension for INSERT statments that allows the
 %% user to specify different formats:
 %%
diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
index cb0c0e16e..9512d2f6b 100644
--- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
+++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl
@@ -443,11 +443,11 @@ parse_sql_template(Config) ->
     parse_sql_template(maps:to_list(RawSQLTemplates), BatchInsertTks).
 
 parse_sql_template([{Key, H} | T], BatchInsertTks) ->
-    case emqx_plugin_libs_rule:detect_sql_type(H) of
-        {ok, select} ->
+    case emqx_utils_sql:get_statement_type(H) of
+        select ->
             parse_sql_template(T, BatchInsertTks);
-        {ok, insert} ->
-            case emqx_plugin_libs_rule:split_insert_sql(H) of
+        insert ->
+            case emqx_utils_sql:parse_insert(H) of
                 {ok, {InsertSQL, Params}} ->
                     parse_sql_template(
                         T,
@@ -463,6 +463,9 @@ parse_sql_template([{Key, H} | T], BatchInsertTks) ->
                     ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}),
                     parse_sql_template(T, BatchInsertTks)
             end;
+        Type when is_atom(Type) ->
+            ?SLOG(error, #{msg => "detect sql type unsupported", sql => H, type => Type}),
+            parse_sql_template(T, BatchInsertTks);
         {error, Reason} ->
             ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}),
             parse_sql_template(T, BatchInsertTks)
@@ -488,10 +491,19 @@ apply_template(
         undefined ->
             BatchReqs;
         #{?BATCH_INSERT_PART := BatchInserts, ?BATCH_PARAMS_TOKENS := BatchParamsTks} ->
-            SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, BatchInserts, BatchParamsTks),
+            SQL = proc_batch_sql(BatchReqs, BatchInserts, BatchParamsTks),
             {Key, SQL}
     end;
 apply_template(Query, Templates) ->
     %% TODO: more detail infomatoin
     ?SLOG(error, #{msg => "apply sql template failed", query => Query, templates => Templates}),
     {error, failed_to_apply_sql_template}.
+
+proc_batch_sql(BatchReqs, BatchInserts, Tokens) ->
+    Values = erlang:iolist_to_binary(
+        lists:join($,, [
+            emqx_placeholder:proc_sql_param_str(Tokens, Msg)
+         || {_, Msg} <- BatchReqs
+        ])
+    ),
+    <<BatchInserts/binary, " values ", Values/binary>>.
diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
index 6c1d06429..e47a2d083 100644
--- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
+++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl
@@ -256,10 +256,10 @@ parse_prepare_sql(Config) ->
     parse_batch_prepare_sql(maps:to_list(SQL), #{}, #{}).
 
 parse_batch_prepare_sql([{Key, H} | T], InsertTksMap, BatchTksMap) ->
-    case emqx_plugin_libs_rule:detect_sql_type(H) of
-        {ok, select} ->
+    case emqx_utils_sql:get_statement_type(H) of
+        select ->
             parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap);
-        {ok, insert} ->
+        insert ->
             InsertTks = emqx_placeholder:preproc_tmpl(H),
             H1 = string:trim(H, trailing, ";"),
             case split_insert_sql(H1) of
@@ -275,6 +275,9 @@ parse_batch_prepare_sql([{Key, H} | T], InsertTksMap, BatchTksMap) ->
                     ?SLOG(error, #{msg => "split sql failed", sql => H, result => Result}),
                     parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap)
             end;
+        Type when is_atom(Type) ->
+            ?SLOG(error, #{msg => "detect sql type unsupported", sql => H, type => Type}),
+            parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap);
         {error, Reason} ->
             ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}),
             parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap)
@@ -289,7 +292,7 @@ to_bin(List) when is_list(List) ->
     unicode:characters_to_binary(List, utf8).
 
 split_insert_sql(SQL0) ->
-    SQL = emqx_plugin_libs_rule:formalize_sql(SQL0),
+    SQL = formalize_sql(SQL0),
     lists:filtermap(
         fun(E) ->
             case string:trim(E) of
@@ -301,3 +304,9 @@ split_insert_sql(SQL0) ->
         end,
         re:split(SQL, "(?i)(insert into)|(?i)(values)")
     ).
+
+formalize_sql(Input) ->
+    %% 1. replace all whitespaces like '\r' '\n' or spaces to a single space char.
+    SQL = re:replace(Input, "\\s+", " ", [global, {return, binary}]),
+    %% 2. trims the result
+    string:trim(SQL).
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index a46b21d92..9c40919f2 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -357,11 +357,11 @@ parse_prepare_sql([], Prepares, Tokens, BatchInserts, BatchTks) ->
     }.
 
 parse_batch_prepare_sql([{Key, H} | T], Prepares, Tokens, BatchInserts, BatchTks) ->
-    case emqx_plugin_libs_rule:detect_sql_type(H) of
-        {ok, select} ->
+    case emqx_utils_sql:get_statement_type(H) of
+        select ->
             parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks);
-        {ok, insert} ->
-            case emqx_plugin_libs_rule:split_insert_sql(H) of
+        insert ->
+            case emqx_utils_sql:parse_insert(H) of
                 {ok, {InsertSQL, Params}} ->
                     ParamsTks = emqx_placeholder:preproc_tmpl(Params),
                     parse_prepare_sql(
@@ -375,6 +375,9 @@ parse_batch_prepare_sql([{Key, H} | T], Prepares, Tokens, BatchInserts, BatchTks
                     ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}),
                     parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks)
             end;
+        Type when is_atom(Type) ->
+            ?SLOG(error, #{msg => "detect sql type unsupported", sql => H, type => Type}),
+            parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks);
         {error, Reason} ->
             ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}),
             parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks)
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
index aa9e38316..38dfb492f 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
@@ -19,10 +19,7 @@
 
 %% preprocess and process template string with place holders
 -export([
-    split_insert_sql/1,
-    detect_sql_type/1,
-    proc_batch_sql/3,
-    formalize_sql/1
+    proc_batch_sql/3
 ]).
 
 %% type converting
@@ -52,44 +49,6 @@
 
 -type tmpl_token() :: list({var, binary()} | {str, binary()}).
 
-%% SQL = <<"INSERT INTO \"abc\" (c1,c2,c3) VALUES (${1}, ${1}, ${1})">>
--spec split_insert_sql(binary()) -> {ok, {InsertSQL, Params}} | {error, atom()} when
-    InsertSQL :: binary(),
-    Params :: binary().
-split_insert_sql(SQL0) ->
-    SQL = formalize_sql(SQL0),
-    case re:split(SQL, "((?i)values)", [{return, binary}]) of
-        [Part1, _, Part3] ->
-            case string:trim(Part1, leading) of
-                <<"insert", _/binary>> = InsertSQL ->
-                    {ok, {InsertSQL, Part3}};
-                <<"INSERT", _/binary>> = InsertSQL ->
-                    {ok, {InsertSQL, Part3}};
-                _ ->
-                    {error, not_insert_sql}
-            end;
-        _ ->
-            {error, not_insert_sql}
-    end.
-
--spec detect_sql_type(binary()) -> {ok, Type} | {error, atom()} when
-    Type :: insert | select.
-detect_sql_type(SQL) ->
-    case re:run(SQL, "^\\s*([a-zA-Z]+)", [{capture, all_but_first, list}]) of
-        {match, [First]} ->
-            Types = [select, insert],
-            PropTypes = [{erlang:atom_to_list(Type), Type} || Type <- Types],
-            LowFirst = string:lowercase(First),
-            case proplists:lookup(LowFirst, PropTypes) of
-                {LowFirst, Type} ->
-                    {ok, Type};
-                _ ->
-                    {error, invalid_sql}
-            end;
-        _ ->
-            {error, invalid_sql}
-    end.
-
 -spec proc_batch_sql(
     BatchReqs :: list({atom(), map()}),
     InsertPart :: binary(),
@@ -104,12 +63,6 @@ proc_batch_sql(BatchReqs, InsertPart, Tokens) ->
     ),
     <<InsertPart/binary, " values ", ValuesPart/binary>>.
 
-formalize_sql(Input) ->
-    %% 1. replace all whitespaces like '\r' '\n' or spaces to a single space char.
-    SQL = re:replace(Input, "\\s+", " ", [global, {return, binary}]),
-    %% 2. trims the result
-    string:trim(SQL).
-
 unsafe_atom_key(Key) when is_atom(Key) ->
     Key;
 unsafe_atom_key(Key) when is_binary(Key) ->
diff --git a/apps/emqx_utils/src/emqx_placeholder.erl b/apps/emqx_utils/src/emqx_placeholder.erl
index 861548f93..fd40b906d 100644
--- a/apps/emqx_utils/src/emqx_placeholder.erl
+++ b/apps/emqx_utils/src/emqx_placeholder.erl
@@ -90,8 +90,6 @@
     | {tmpl, tmpl_token()}
     | {value, term()}.
 
--dialyzer({no_improper_lists, [quote_mysql/1, escape_mysql/4, escape_prepend/4]}).
-
 %%------------------------------------------------------------------------------
 %% APIs
 %%------------------------------------------------------------------------------
@@ -247,22 +245,15 @@ bin(Val) -> emqx_utils_conv:bin(Val).
 
 -spec quote_sql(_Value) -> iolist().
 quote_sql(Str) ->
-    quote_escape(Str, fun escape_sql/1).
+    emqx_utils_sql:to_sql_string(Str, #{escaping => sql}).
 
 -spec quote_cql(_Value) -> iolist().
 quote_cql(Str) ->
-    quote_escape(Str, fun escape_cql/1).
+    emqx_utils_sql:to_sql_string(Str, #{escaping => cql}).
 
 -spec quote_mysql(_Value) -> iolist().
-quote_mysql(Str) when is_binary(Str) ->
-    try
-        escape_mysql(Str)
-    catch
-        throw:invalid_utf8 ->
-            [<<"0x">> | binary:encode_hex(Str)]
-    end;
 quote_mysql(Str) ->
-    quote_escape(Str, fun escape_mysql/1).
+    emqx_utils_sql:to_sql_string(Str, #{escaping => mysql}).
 
 lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] ->
     Value;
@@ -370,57 +361,3 @@ unwrap(<<"\"${", Val/binary>>, _StripDoubleQuote = true) ->
     binary:part(Val, {0, byte_size(Val) - 2});
 unwrap(<<"${", Val/binary>>, _StripDoubleQuote) ->
     binary:part(Val, {0, byte_size(Val) - 1}).
-
--spec quote_escape(_Value, fun((binary()) -> iodata())) -> iodata().
-quote_escape(Str, EscapeFun) when is_binary(Str) ->
-    EscapeFun(Str);
-quote_escape(Str, EscapeFun) when is_list(Str) ->
-    case unicode:characters_to_binary(Str) of
-        Bin when is_binary(Bin) ->
-            EscapeFun(Bin);
-        Otherwise ->
-            error(Otherwise)
-    end;
-quote_escape(Str, EscapeFun) when is_atom(Str) orelse is_map(Str) ->
-    EscapeFun(bin(Str));
-quote_escape(Val, _EscapeFun) ->
-    bin(Val).
-
--spec escape_sql(binary()) -> iolist().
-escape_sql(S) ->
-    ES = binary:replace(S, [<<"\\">>, <<"'">>], <<"\\">>, [global, {insert_replaced, 1}]),
-    [$', ES, $'].
-
--spec escape_cql(binary()) -> iolist().
-escape_cql(S) ->
-    ES = binary:replace(S, <<"'">>, <<"'">>, [global, {insert_replaced, 1}]),
-    [$', ES, $'].
-
--spec escape_mysql(binary()) -> iolist().
-escape_mysql(S0) ->
-    % https://dev.mysql.com/doc/refman/8.0/en/string-literals.html
-    [$', escape_mysql(S0, 0, 0, S0), $'].
-
-%% NOTE
-%% This thing looks more complicated than needed because it's optimized for as few
-%% intermediate memory (re)allocations as possible.
-escape_mysql(<<$', Rest/binary>>, I, Run, Src) ->
-    escape_prepend(I, Run, Src, [<<"\\'">> | escape_mysql(Rest, I + Run + 1, 0, Src)]);
-escape_mysql(<<$\\, Rest/binary>>, I, Run, Src) ->
-    escape_prepend(I, Run, Src, [<<"\\\\">> | escape_mysql(Rest, I + Run + 1, 0, Src)]);
-escape_mysql(<<0, Rest/binary>>, I, Run, Src) ->
-    escape_prepend(I, Run, Src, [<<"\\0">> | escape_mysql(Rest, I + Run + 1, 0, Src)]);
-escape_mysql(<<_/utf8, Rest/binary>> = S, I, Run, Src) ->
-    CWidth = byte_size(S) - byte_size(Rest),
-    escape_mysql(Rest, I, Run + CWidth, Src);
-escape_mysql(<<>>, 0, _, Src) ->
-    Src;
-escape_mysql(<<>>, I, Run, Src) ->
-    binary:part(Src, I, Run);
-escape_mysql(_, _I, _Run, _Src) ->
-    throw(invalid_utf8).
-
-escape_prepend(_RunI, 0, _Src, Tail) ->
-    Tail;
-escape_prepend(I, Run, Src, Tail) ->
-    [binary:part(Src, I, Run) | Tail].
diff --git a/apps/emqx_utils/src/emqx_utils_sql.erl b/apps/emqx_utils/src/emqx_utils_sql.erl
new file mode 100644
index 000000000..3caed6b62
--- /dev/null
+++ b/apps/emqx_utils/src/emqx_utils_sql.erl
@@ -0,0 +1,157 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_utils_sql).
+
+-export([get_statement_type/1]).
+-export([parse_insert/1]).
+
+-export([to_sql_value/1]).
+-export([to_sql_string/2]).
+
+-export([escape_sql/1]).
+-export([escape_cql/1]).
+-export([escape_mysql/1]).
+
+-export_type([value/0]).
+
+-type statement_type() :: select | insert | delete.
+-type value() :: null | binary() | number() | boolean() | [value()].
+
+-dialyzer({no_improper_lists, [escape_mysql/4, escape_prepend/4]}).
+
+-spec get_statement_type(iodata()) -> statement_type() | {error, unknown}.
+get_statement_type(Query) ->
+    KnownTypes = #{
+        <<"select">> => select,
+        <<"insert">> => insert,
+        <<"delete">> => delete
+    },
+    case re:run(Query, <<"^\\s*([a-zA-Z]+)">>, [{capture, all_but_first, binary}]) of
+        {match, [Token]} ->
+            maps:get(string:lowercase(Token), KnownTypes, {error, unknown});
+        _ ->
+            {error, unknown}
+    end.
+
+%% @doc Parse an INSERT SQL statement into its INSERT part and the VALUES part.
+%% SQL = <<"INSERT INTO \"abc\" (c1, c2, c3) VALUES (${a}, ${b}, ${c.prop})">>
+%% {ok, {<<"INSERT INTO \"abc\" (c1, c2, c3)">>, <<"(${a}, ${b}, ${c.prop})">>}}
+-spec parse_insert(iodata()) ->
+    {ok, {_Statement :: binary(), _Rows :: binary()}} | {error, not_insert_sql}.
+parse_insert(SQL) ->
+    case re:split(SQL, "((?i)values)", [{return, binary}]) of
+        [Part1, _, Part3] ->
+            case string:trim(Part1, leading) of
+                <<"insert", _/binary>> = InsertSQL ->
+                    {ok, {InsertSQL, Part3}};
+                <<"INSERT", _/binary>> = InsertSQL ->
+                    {ok, {InsertSQL, Part3}};
+                _ ->
+                    {error, not_insert_sql}
+            end;
+        _ ->
+            {error, not_insert_sql}
+    end.
+
+%% @doc Convert an Erlang term to a value that can be used primarily in
+%% prepared SQL statements.
+-spec to_sql_value(term()) -> value().
+to_sql_value(undefined) -> null;
+to_sql_value(List) when is_list(List) -> List;
+to_sql_value(Bin) when is_binary(Bin) -> Bin;
+to_sql_value(Num) when is_number(Num) -> Num;
+to_sql_value(Bool) when is_boolean(Bool) -> Bool;
+to_sql_value(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
+to_sql_value(Map) when is_map(Map) -> emqx_utils_json:encode(Map).
+
+%% @doc Convert an Erlang term to a string that can be interpolated in literal
+%% SQL statements. The value is escaped if necessary.
+-spec to_sql_string(term(), Options) -> iodata() when
+    Options :: #{
+        escaping => cql | mysql | sql
+    }.
+to_sql_string(String, #{escaping := mysql}) when is_binary(String) ->
+    try
+        escape_mysql(String)
+    catch
+        throw:invalid_utf8 ->
+            [<<"0x">>, binary:encode_hex(String)]
+    end;
+to_sql_string(Term, #{escaping := mysql}) ->
+    maybe_escape(Term, fun escape_mysql/1);
+to_sql_string(Term, #{escaping := cql}) ->
+    maybe_escape(Term, fun escape_cql/1);
+to_sql_string(Term, #{}) ->
+    maybe_escape(Term, fun escape_sql/1).
+
+-spec maybe_escape(_Value, fun((binary()) -> iodata())) -> iodata().
+maybe_escape(Str, EscapeFun) when is_binary(Str) ->
+    EscapeFun(Str);
+maybe_escape(Str, EscapeFun) when is_list(Str) ->
+    case unicode:characters_to_binary(Str) of
+        Bin when is_binary(Bin) ->
+            EscapeFun(Bin);
+        Otherwise ->
+            error(Otherwise)
+    end;
+maybe_escape(Val, EscapeFun) when is_atom(Val) orelse is_map(Val) ->
+    EscapeFun(emqx_utils_conv:bin(Val));
+maybe_escape(Val, _EscapeFun) ->
+    emqx_utils_conv:bin(Val).
+
+-spec escape_sql(binary()) -> iodata().
+escape_sql(S) ->
+    % NOTE
+    % This is a bit misleading: currently, escaping logic in `escape_sql/1` likely
+    % won't work with pgsql since it does not support C-style escapes by default.
+    % https://www.postgresql.org/docs/14/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS
+    ES = binary:replace(S, [<<"\\">>, <<"'">>], <<"\\">>, [global, {insert_replaced, 1}]),
+    [$', ES, $'].
+
+-spec escape_cql(binary()) -> iodata().
+escape_cql(S) ->
+    ES = binary:replace(S, <<"'">>, <<"'">>, [global, {insert_replaced, 1}]),
+    [$', ES, $'].
+
+-spec escape_mysql(binary()) -> iodata().
+escape_mysql(S0) ->
+    % https://dev.mysql.com/doc/refman/8.0/en/string-literals.html
+    [$', escape_mysql(S0, 0, 0, S0), $'].
+
+%% NOTE
+%% This thing looks more complicated than needed because it's optimized for as few
+%% intermediate memory (re)allocations as possible.
+escape_mysql(<<$', Rest/binary>>, I, Run, Src) ->
+    escape_prepend(I, Run, Src, [<<"\\'">> | escape_mysql(Rest, I + Run + 1, 0, Src)]);
+escape_mysql(<<$\\, Rest/binary>>, I, Run, Src) ->
+    escape_prepend(I, Run, Src, [<<"\\\\">> | escape_mysql(Rest, I + Run + 1, 0, Src)]);
+escape_mysql(<<0, Rest/binary>>, I, Run, Src) ->
+    escape_prepend(I, Run, Src, [<<"\\0">> | escape_mysql(Rest, I + Run + 1, 0, Src)]);
+escape_mysql(<<_/utf8, Rest/binary>> = S, I, Run, Src) ->
+    CWidth = byte_size(S) - byte_size(Rest),
+    escape_mysql(Rest, I, Run + CWidth, Src);
+escape_mysql(<<>>, 0, _, Src) ->
+    Src;
+escape_mysql(<<>>, I, Run, Src) ->
+    binary:part(Src, I, Run);
+escape_mysql(_, _I, _Run, _Src) ->
+    throw(invalid_utf8).
+
+escape_prepend(_RunI, 0, _Src, Tail) ->
+    Tail;
+escape_prepend(I, Run, Src, Tail) ->
+    [binary:part(Src, I, Run) | Tail].

From e6fb0203b424eaf53ed559b0ba824c013203d2cd Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Fri, 9 Jun 2023 00:22:53 +0300
Subject: [PATCH 28/33] refactor(pluglib): move connectivity checks to
 `emqx_connector_lib`

---
 .../src/emqx_bridge_opents_connector.erl      |  4 +-
 .../emqx_connector/src/emqx_connector_lib.erl | 31 +++++++++++
 .../test/emqx_connector_lib_tests.erl         | 54 +++++++++++++++++++
 .../test/emqx_plugin_libs_rule_SUITE.erl      | 18 -------
 4 files changed, 88 insertions(+), 19 deletions(-)
 create mode 100644 apps/emqx_connector/test/emqx_connector_lib_tests.erl

diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl
index 71184e872..555921c93 100644
--- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl
+++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl
@@ -47,6 +47,8 @@ fields(config) ->
 %% `emqx_resource' API
 %%========================================================================================
 
+-define(HTTP_CONNECT_TIMEOUT, 1000).
+
 callback_mode() -> always_sync.
 
 is_buffer_supported() -> false.
@@ -171,7 +173,7 @@ opentsdb_connectivity(Server) ->
             <<"https://", _/binary>> -> Server;
             _ -> "http://" ++ Server
         end,
-    emqx_plugin_libs_rule:http_connectivity(SvrUrl).
+    emqx_connector_lib:http_connectivity(SvrUrl, ?HTTP_CONNECT_TIMEOUT).
 
 format_opentsdb_msg(Msg) ->
     maps:with(
diff --git a/apps/emqx_connector/src/emqx_connector_lib.erl b/apps/emqx_connector/src/emqx_connector_lib.erl
index d554ca5ab..8fbe7597a 100644
--- a/apps/emqx_connector/src/emqx_connector_lib.erl
+++ b/apps/emqx_connector/src/emqx_connector_lib.erl
@@ -15,8 +15,39 @@
 %%--------------------------------------------------------------------
 -module(emqx_connector_lib).
 
+%% connectivity check
+-export([
+    http_connectivity/2,
+    tcp_connectivity/3
+]).
+
 -export([resolve_dns/2]).
 
+-spec http_connectivity(uri_string:uri_string(), timeout()) ->
+    ok | {error, Reason :: term()}.
+http_connectivity(Url, Timeout) ->
+    case emqx_http_lib:uri_parse(Url) of
+        {ok, #{host := Host, port := Port}} ->
+            tcp_connectivity(Host, Port, Timeout);
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+-spec tcp_connectivity(
+    Host :: inet:socket_address() | inet:hostname(),
+    Port :: inet:port_number(),
+    timeout()
+) ->
+    ok | {error, Reason :: term()}.
+tcp_connectivity(Host, Port, Timeout) ->
+    case gen_tcp:connect(Host, Port, emqx_utils:ipv6_probe([]), Timeout) of
+        {ok, Sock} ->
+            gen_tcp:close(Sock),
+            ok;
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
 %% @doc Mostly for meck.
 resolve_dns(DNS, Type) ->
     inet_res:lookup(DNS, in, Type).
diff --git a/apps/emqx_connector/test/emqx_connector_lib_tests.erl b/apps/emqx_connector/test/emqx_connector_lib_tests.erl
new file mode 100644
index 000000000..5add35d31
--- /dev/null
+++ b/apps/emqx_connector/test/emqx_connector_lib_tests.erl
@@ -0,0 +1,54 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_connector_lib_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+http_connectivity_ok_test() ->
+    {ok, Socket} = gen_tcp:listen(0, []),
+    {ok, Port} = inet:port(Socket),
+    ?assertEqual(
+        ok,
+        emqx_connector_lib:http_connectivity("http://127.0.0.1:" ++ integer_to_list(Port), 1000)
+    ),
+    gen_tcp:close(Socket).
+
+http_connectivity_error_test() ->
+    {ok, Socket} = gen_tcp:listen(0, []),
+    {ok, Port} = inet:port(Socket),
+    ok = gen_tcp:close(Socket),
+    ?assertEqual(
+        {error, econnrefused},
+        emqx_connector_lib:http_connectivity("http://127.0.0.1:" ++ integer_to_list(Port), 1000)
+    ).
+
+tcp_connectivity_ok_test() ->
+    {ok, Socket} = gen_tcp:listen(0, []),
+    {ok, Port} = inet:port(Socket),
+    ?assertEqual(
+        ok,
+        emqx_connector_lib:tcp_connectivity("127.0.0.1", Port, 1000)
+    ),
+    ok = gen_tcp:close(Socket).
+
+tcp_connectivity_error_test() ->
+    {ok, Socket} = gen_tcp:listen(0, []),
+    {ok, Port} = inet:port(Socket),
+    ok = gen_tcp:close(Socket),
+    ?assertEqual(
+        {error, econnrefused},
+        emqx_connector_lib:tcp_connectivity("127.0.0.1", Port, 1000)
+    ).
diff --git a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl
index bb1ba838b..73b700974 100644
--- a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl
+++ b/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl
@@ -21,26 +21,8 @@
 
 -include_lib("eunit/include/eunit.hrl").
 
--define(PORT, 9876).
-
 all() -> emqx_common_test_helpers:all(?MODULE).
 
-t_http_connectivity(_) ->
-    {ok, Socket} = gen_tcp:listen(?PORT, []),
-    ok = emqx_plugin_libs_rule:http_connectivity(
-        "http://127.0.0.1:" ++ integer_to_list(?PORT), 1000
-    ),
-    gen_tcp:close(Socket),
-    {error, _} = emqx_plugin_libs_rule:http_connectivity(
-        "http://127.0.0.1:" ++ integer_to_list(?PORT), 1000
-    ).
-
-t_tcp_connectivity(_) ->
-    {ok, Socket} = gen_tcp:listen(?PORT, []),
-    ok = emqx_plugin_libs_rule:tcp_connectivity("127.0.0.1", ?PORT, 1000),
-    gen_tcp:close(Socket),
-    {error, _} = emqx_plugin_libs_rule:tcp_connectivity("127.0.0.1", ?PORT, 1000).
-
 t_atom_key(_) ->
     _ = erlang,
     _ = port,

From 830ba54721b20c1923d590150b55a1b2911087d8 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Fri, 9 Jun 2023 00:33:05 +0300
Subject: [PATCH 29/33] refactor(pluglib): remove `emqx_plugin_libs`
 application

---
 .github/CODEOWNERS                            |   1 -
 apps/emqx_machine/src/emqx_machine_boot.erl   |   1 -
 apps/emqx_machine/test/emqx_machine_SUITE.erl |   1 -
 apps/emqx_plugin_libs/rebar.config            |   8 -
 .../src/emqx_plugin_libs.app.src              |   8 -
 .../src/emqx_plugin_libs.appup.src            |  13 --
 .../emqx_plugin_libs/src/emqx_plugin_libs.erl |  17 --
 .../src/emqx_plugin_libs_rule.erl             | 202 ------------------
 .../test/emqx_plugin_libs_rule_SUITE.erl      |  47 ----
 mix.exs                                       |   1 -
 rebar.config.erl                              |   1 -
 11 files changed, 300 deletions(-)
 delete mode 100644 apps/emqx_plugin_libs/rebar.config
 delete mode 100644 apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
 delete mode 100644 apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src
 delete mode 100644 apps/emqx_plugin_libs/src/emqx_plugin_libs.erl
 delete mode 100644 apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
 delete mode 100644 apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index a45d9af59..4ad6049f3 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -11,7 +11,6 @@
 /apps/emqx_ft/             @emqx/emqx-review-board @savonarola @keynslug
 /apps/emqx_gateway/        @emqx/emqx-review-board @lafirest
 /apps/emqx_management/     @emqx/emqx-review-board @lafirest @sstrigler
-/apps/emqx_plugin_libs/    @emqx/emqx-review-board @lafirest
 /apps/emqx_plugins/        @emqx/emqx-review-board @JimMoen
 /apps/emqx_prometheus/     @emqx/emqx-review-board @JimMoen
 /apps/emqx_psk/            @emqx/emqx-review-board @lafirest
diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl
index 4c5740bad..b2b0823ce 100644
--- a/apps/emqx_machine/src/emqx_machine_boot.erl
+++ b/apps/emqx_machine/src/emqx_machine_boot.erl
@@ -137,7 +137,6 @@ basic_reboot_apps() ->
             emqx_resource,
             emqx_rule_engine,
             emqx_bridge,
-            emqx_plugin_libs,
             emqx_management,
             emqx_retainer,
             emqx_exhook,
diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl
index d8613a537..bd18c67aa 100644
--- a/apps/emqx_machine/test/emqx_machine_SUITE.erl
+++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl
@@ -51,7 +51,6 @@ init_per_suite(Config) ->
         emqx_resource,
         emqx_rule_engine,
         emqx_bridge,
-        emqx_plugin_libs,
         emqx_management,
         emqx_retainer,
         emqx_exhook,
diff --git a/apps/emqx_plugin_libs/rebar.config b/apps/emqx_plugin_libs/rebar.config
deleted file mode 100644
index dee2902a5..000000000
--- a/apps/emqx_plugin_libs/rebar.config
+++ /dev/null
@@ -1,8 +0,0 @@
-%% -*- mode: erlang -*-
-
-{deps, [
-    {emqx, {path, "../emqx"}},
-    {emqx_utils, {path, "../emqx_utils"}}
-]}.
-
-{project_plugins, [erlfmt]}.
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
deleted file mode 100644
index 6953acdf1..000000000
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
+++ /dev/null
@@ -1,8 +0,0 @@
-%% -*- mode: erlang -*-
-{application, emqx_plugin_libs, [
-    {description, "EMQX Plugin utility libs"},
-    {vsn, "4.3.12"},
-    {modules, []},
-    {applications, [kernel, stdlib]},
-    {env, []}
-]}.
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src
deleted file mode 100644
index 7ae0f0317..000000000
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src
+++ /dev/null
@@ -1,13 +0,0 @@
-%% -*- mode: erlang -*-
-{VSN,
- [ {"4.3.0",
-    [ {add_module, emqx_plugin_libs_pool}
-    ]},
-   {<<".*">>, []}
- ],
- [ {"4.3.0",
-    [ {delete_module, emqx_plugin_libs_pool}
-    ]},
-   {<<".*">>, []}
- ]
-}.
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs.erl
deleted file mode 100644
index e237ffe82..000000000
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.erl
+++ /dev/null
@@ -1,17 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_plugin_libs).
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
deleted file mode 100644
index 38dfb492f..000000000
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
+++ /dev/null
@@ -1,202 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_plugin_libs_rule).
--elvis([{elvis_style, god_modules, disable}]).
-
-%% preprocess and process template string with place holders
--export([
-    proc_batch_sql/3
-]).
-
-%% type converting
--export([
-    bool/1,
-    int/1,
-    float/1,
-    float2str/2,
-    map/1,
-    utf8_bin/1,
-    utf8_str/1,
-    atom_key/1,
-    unsafe_atom_key/1
-]).
-
-%% connectivity check
--export([
-    http_connectivity/1,
-    http_connectivity/2,
-    tcp_connectivity/2,
-    tcp_connectivity/3
-]).
-
--compile({no_auto_import, [float/1]}).
-
--type uri_string() :: iodata().
-
--type tmpl_token() :: list({var, binary()} | {str, binary()}).
-
--spec proc_batch_sql(
-    BatchReqs :: list({atom(), map()}),
-    InsertPart :: binary(),
-    Tokens :: tmpl_token()
-) -> InsertSQL :: binary().
-proc_batch_sql(BatchReqs, InsertPart, Tokens) ->
-    ValuesPart = erlang:iolist_to_binary(
-        lists:join($,, [
-            emqx_placeholder:proc_sql_param_str(Tokens, Msg)
-         || {_, Msg} <- BatchReqs
-        ])
-    ),
-    <<InsertPart/binary, " values ", ValuesPart/binary>>.
-
-unsafe_atom_key(Key) when is_atom(Key) ->
-    Key;
-unsafe_atom_key(Key) when is_binary(Key) ->
-    binary_to_atom(Key, utf8);
-unsafe_atom_key(Keys = [_Key | _]) ->
-    [unsafe_atom_key(SubKey) || SubKey <- Keys];
-unsafe_atom_key(Key) ->
-    error({invalid_key, Key}).
-
-atom_key(Key) when is_atom(Key) ->
-    Key;
-atom_key(Key) when is_binary(Key) ->
-    try
-        binary_to_existing_atom(Key, utf8)
-    catch
-        error:badarg -> error({invalid_key, Key})
-    end;
-%% nested keys
-atom_key(Keys = [_Key | _]) ->
-    [atom_key(SubKey) || SubKey <- Keys];
-atom_key(Key) ->
-    error({invalid_key, Key}).
-
--spec http_connectivity(uri_string()) -> ok | {error, Reason :: term()}.
-http_connectivity(Url) ->
-    http_connectivity(Url, 3000).
-
--spec http_connectivity(uri_string(), integer()) -> ok | {error, Reason :: term()}.
-http_connectivity(Url, Timeout) ->
-    case emqx_http_lib:uri_parse(Url) of
-        {ok, #{host := Host, port := Port}} ->
-            tcp_connectivity(Host, Port, Timeout);
-        {error, Reason} ->
-            {error, Reason}
-    end.
-
--spec tcp_connectivity(
-    Host :: inet:socket_address() | inet:hostname(),
-    Port :: inet:port_number()
-) ->
-    ok | {error, Reason :: term()}.
-tcp_connectivity(Host, Port) ->
-    tcp_connectivity(Host, Port, 3000).
-
--spec tcp_connectivity(
-    Host :: inet:socket_address() | inet:hostname(),
-    Port :: inet:port_number(),
-    Timeout :: integer()
-) ->
-    ok | {error, Reason :: term()}.
-tcp_connectivity(Host, Port, Timeout) ->
-    case gen_tcp:connect(Host, Port, emqx_utils:ipv6_probe([]), Timeout) of
-        {ok, Sock} ->
-            gen_tcp:close(Sock),
-            ok;
-        {error, Reason} ->
-            {error, Reason}
-    end.
-
-utf8_bin(Str) when is_binary(Str); is_list(Str) ->
-    unicode:characters_to_binary(Str);
-utf8_bin(Str) ->
-    unicode:characters_to_binary(emqx_utils_conv:bin(Str)).
-
-utf8_str(Str) when is_binary(Str); is_list(Str) ->
-    unicode:characters_to_list(Str);
-utf8_str(Str) ->
-    unicode:characters_to_list(emqx_utils_conv:str(Str)).
-
-int(List) when is_list(List) ->
-    try
-        list_to_integer(List)
-    catch
-        error:badarg ->
-            int(list_to_float(List))
-    end;
-int(Bin) when is_binary(Bin) ->
-    try
-        binary_to_integer(Bin)
-    catch
-        error:badarg ->
-            int(binary_to_float(Bin))
-    end;
-int(Int) when is_integer(Int) -> Int;
-int(Float) when is_float(Float) -> erlang:floor(Float);
-int(true) ->
-    1;
-int(false) ->
-    0;
-int(Data) ->
-    error({invalid_number, Data}).
-
-float(List) when is_list(List) ->
-    try
-        list_to_float(List)
-    catch
-        error:badarg ->
-            float(list_to_integer(List))
-    end;
-float(Bin) when is_binary(Bin) ->
-    try
-        binary_to_float(Bin)
-    catch
-        error:badarg ->
-            float(binary_to_integer(Bin))
-    end;
-float(Num) when is_number(Num) -> erlang:float(Num);
-float(Data) ->
-    error({invalid_number, Data}).
-
-float2str(Float, Precision) when is_float(Float) and is_integer(Precision) ->
-    float_to_binary(Float, [{decimals, Precision}, compact]).
-
-map(Bin) when is_binary(Bin) ->
-    case emqx_utils_json:decode(Bin, [return_maps]) of
-        Map = #{} -> Map;
-        _ -> error({invalid_map, Bin})
-    end;
-map(List) when is_list(List) -> maps:from_list(List);
-map(Map) when is_map(Map) -> Map;
-map(Data) ->
-    error({invalid_map, Data}).
-
-bool(Bool) when
-    Bool == true;
-    Bool == <<"true">>;
-    Bool == 1
-->
-    true;
-bool(Bool) when
-    Bool == false;
-    Bool == <<"false">>;
-    Bool == 0
-->
-    false;
-bool(Bool) ->
-    error({invalid_boolean, Bool}).
diff --git a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl
deleted file mode 100644
index 73b700974..000000000
--- a/apps/emqx_plugin_libs/test/emqx_plugin_libs_rule_SUITE.erl
+++ /dev/null
@@ -1,47 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_plugin_libs_rule_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
-
-all() -> emqx_common_test_helpers:all(?MODULE).
-
-t_atom_key(_) ->
-    _ = erlang,
-    _ = port,
-    ?assertEqual([erlang], emqx_plugin_libs_rule:atom_key([<<"erlang">>])),
-    ?assertEqual([erlang, port], emqx_plugin_libs_rule:atom_key([<<"erlang">>, port])),
-    ?assertEqual([erlang, port], emqx_plugin_libs_rule:atom_key([<<"erlang">>, <<"port">>])),
-    ?assertEqual(erlang, emqx_plugin_libs_rule:atom_key(<<"erlang">>)),
-    ?assertError({invalid_key, {a, v}}, emqx_plugin_libs_rule:atom_key({a, v})),
-    _ = xyz876gv123,
-    ?assertEqual([xyz876gv123, port], emqx_plugin_libs_rule:atom_key([<<"xyz876gv123">>, port])).
-
-t_unsafe_atom_key(_) ->
-    ?assertEqual([xyz876gv], emqx_plugin_libs_rule:unsafe_atom_key([<<"xyz876gv">>])),
-    ?assertEqual(
-        [xyz876gv33, port],
-        emqx_plugin_libs_rule:unsafe_atom_key([<<"xyz876gv33">>, port])
-    ),
-    ?assertEqual(
-        [xyz876gv331, port1221],
-        emqx_plugin_libs_rule:unsafe_atom_key([<<"xyz876gv331">>, <<"port1221">>])
-    ),
-    ?assertEqual(xyz876gv3312, emqx_plugin_libs_rule:unsafe_atom_key(<<"xyz876gv3312">>)).
diff --git a/mix.exs b/mix.exs
index fbd88e61d..5fdf5c565 100644
--- a/mix.exs
+++ b/mix.exs
@@ -352,7 +352,6 @@ defmodule EMQXUmbrella.MixProject do
       [
         mnesia: :load,
         ekka: :load,
-        emqx_plugin_libs: :load,
         esasl: :load,
         observer_cli: :permanent,
         tools: :permanent,
diff --git a/rebar.config.erl b/rebar.config.erl
index d265f53cd..81b5fba5f 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -405,7 +405,6 @@ relx_apps(ReleaseType, Edition) ->
         [
             {mnesia, load},
             {ekka, load},
-            {emqx_plugin_libs, load},
             {esasl, load},
             observer_cli,
             tools,

From c8360f86b78c1eff104033926934ce67ea4c0b53 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Fri, 9 Jun 2023 01:02:51 +0300
Subject: [PATCH 30/33] chore: bump application versions

---
 apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src | 2 +-
 apps/emqx_machine/src/emqx_machine.app.src             | 2 +-
 lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src
index 38da1e18b..1fda99700 100644
--- a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src
+++ b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src
@@ -1,6 +1,6 @@
 {application, emqx_gateway_stomp, [
     {description, "Stomp Gateway"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {applications, [kernel, stdlib, emqx, emqx_gateway]},
     {env, []},
diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src
index 5cfa80369..b9138d07d 100644
--- a/apps/emqx_machine/src/emqx_machine.app.src
+++ b/apps/emqx_machine/src/emqx_machine.app.src
@@ -3,7 +3,7 @@
     {id, "emqx_machine"},
     {description, "The EMQX Machine"},
     % strict semver, bump manually!
-    {vsn, "0.2.5"},
+    {vsn, "0.2.6"},
     {modules, []},
     {registered, []},
     {applications, [kernel, stdlib, emqx_ctl]},
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index 3ed460492..702b9ff09 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -1,6 +1,6 @@
 {application, emqx_ee_connector, [
     {description, "EMQX Enterprise connectors"},
-    {vsn, "0.1.13"},
+    {vsn, "0.1.14"},
     {registered, []},
     {applications, [
         kernel,

From 61488ec407fef3b34837277c4432a8224afa3eb1 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 9 Jun 2023 16:04:38 +0200
Subject: [PATCH 31/33] fix: lost messages when clickhouse closes while sending
 messages

Fixes:
https://emqx.atlassian.net/browse/EMQX-10217
---
 .../src/emqx_bridge_clickhouse_connector.erl          | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
index 211e41174..ceaf42a64 100644
--- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
+++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
@@ -465,8 +465,19 @@ transform_and_log_clickhouse_result(ClickhouseErrorResult, ResourceID, SQL) ->
         reason => ClickhouseErrorResult
     }),
     case ClickhouseErrorResult of
+        %% TODO: The hackeny errors that the clickhouse library forwards are
+        %% very loosely defined. We should try to make sure that the following
+        %% handles all error cases that we need to handle as recoverable_error
         {error, ecpool_empty} ->
             {error, {recoverable_error, ecpool_empty}};
+        {error, econnrefused} ->
+            {error, {recoverable_error, econnrefused}};
+        {error, closed} ->
+            {error, {recoverable_error, closed}};
+        {error, {closed, PartialBody}} ->
+            {error, {recoverable_error, {closed_partial_body, PartialBody}}};
+        {error, disconnected} ->
+            {error, {recoverable_error, disconnected}};
         _ ->
             {error, ClickhouseErrorResult}
     end.

From c6eb1af82ce99953a9c909cbce2f37f2bd080c08 Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 9 Jun 2023 16:17:13 +0200
Subject: [PATCH 32/33] docs: add changelog entry

---
 changes/ee/fix-10997.en.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/ee/fix-10997.en.md

diff --git a/changes/ee/fix-10997.en.md b/changes/ee/fix-10997.en.md
new file mode 100644
index 000000000..08ccfbe43
--- /dev/null
+++ b/changes/ee/fix-10997.en.md
@@ -0,0 +1 @@
+The ClickHouse bridge had a problem that could cause messages to be dropped when the ClickHouse server is closed while sending messages even when the request_ttl is set to infinity. This has been fixed by treating errors due to a closed connection as recoverable errors.

From 1b3af2ac91d7a57858dcd01cafc1bbf32956204f Mon Sep 17 00:00:00 2001
From: Kjell Winblad <kjellwinblad@gmail.com>
Date: Fri, 9 Jun 2023 17:22:16 +0200
Subject: [PATCH 33/33] refactor: nicer handling of errors and warning log for
 recoverable errors

---
 .../src/emqx_bridge_clickhouse_connector.erl  | 68 +++++++++++++++----
 1 file changed, 53 insertions(+), 15 deletions(-)

diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
index ceaf42a64..857a7c119 100644
--- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
+++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl
@@ -450,10 +450,14 @@ execute_sql_in_clickhouse_server_using_connection(Connection, SQL) ->
 
 %% This function transforms the result received from clickhouse to something
 %% that is a little bit more readable and creates approprieate log messages
-transform_and_log_clickhouse_result({ok, 200, <<"">>} = _ClickhouseResult, _, _) ->
+transform_and_log_clickhouse_result({ok, ResponseCode, <<"">>} = _ClickhouseResult, _, _) when
+    ResponseCode =:= 200; ResponseCode =:= 204
+->
     snabbkaffe_log_return(ok),
     ok;
-transform_and_log_clickhouse_result({ok, 200, Data}, _, _) ->
+transform_and_log_clickhouse_result({ok, ResponseCode, Data}, _, _) when
+    ResponseCode =:= 200; ResponseCode =:= 204
+->
     Result = {ok, Data},
     snabbkaffe_log_return(Result),
     Result;
@@ -464,24 +468,58 @@ transform_and_log_clickhouse_result(ClickhouseErrorResult, ResourceID, SQL) ->
         sql => SQL,
         reason => ClickhouseErrorResult
     }),
-    case ClickhouseErrorResult of
+    case is_recoverable_error(ClickhouseErrorResult) of
         %% TODO: The hackeny errors that the clickhouse library forwards are
         %% very loosely defined. We should try to make sure that the following
         %% handles all error cases that we need to handle as recoverable_error
-        {error, ecpool_empty} ->
-            {error, {recoverable_error, ecpool_empty}};
-        {error, econnrefused} ->
-            {error, {recoverable_error, econnrefused}};
-        {error, closed} ->
-            {error, {recoverable_error, closed}};
-        {error, {closed, PartialBody}} ->
-            {error, {recoverable_error, {closed_partial_body, PartialBody}}};
-        {error, disconnected} ->
-            {error, {recoverable_error, disconnected}};
-        _ ->
-            {error, ClickhouseErrorResult}
+        true ->
+            ?SLOG(warning, #{
+                msg => "clickhouse connector: sql query failed (recoverable)",
+                recoverable_error => true,
+                connector => ResourceID,
+                sql => SQL,
+                reason => ClickhouseErrorResult
+            }),
+            to_recoverable_error(ClickhouseErrorResult);
+        false ->
+            ?SLOG(error, #{
+                msg => "clickhouse connector: sql query failed (unrecoverable)",
+                recoverable_error => false,
+                connector => ResourceID,
+                sql => SQL,
+                reason => ClickhouseErrorResult
+            }),
+            to_error_tuple(ClickhouseErrorResult)
     end.
 
+to_recoverable_error({error, Reason}) ->
+    {error, {recoverable_error, Reason}};
+to_recoverable_error(Error) ->
+    {error, {recoverable_error, Error}}.
+
+to_error_tuple({error, Reason}) ->
+    {error, {unrecoverable_error, Reason}};
+to_error_tuple(Error) ->
+    {error, {unrecoverable_error, Error}}.
+
+is_recoverable_error({error, Reason}) ->
+    is_recoverable_error_reason(Reason);
+is_recoverable_error(_) ->
+    false.
+
+is_recoverable_error_reason(ecpool_empty) ->
+    true;
+is_recoverable_error_reason(econnrefused) ->
+    true;
+is_recoverable_error_reason(closed) ->
+    true;
+is_recoverable_error_reason({closed, _PartialBody}) ->
+    true;
+is_recoverable_error_reason(disconnected) ->
+    true;
+is_recoverable_error_reason(_) ->
+    false.
+
 snabbkaffe_log_return(_Result) ->
     ?tp(
         clickhouse_connector_query_return,